Skip to content

Commit

Permalink
New script executor (#135)
Browse files Browse the repository at this point in the history
* Enhanced the script executor (fixed bug for large trello board data) & fixed a defect in the events text structure

* removed commented working code

* update to user dashboard 'points alloc' to support separated event log format

* 'belt' wording & color to 'star'

* removed commented code
  • Loading branch information
matallen authored Feb 5, 2021
1 parent 00bed7c commit fa1ddb3
Show file tree
Hide file tree
Showing 7 changed files with 214 additions and 53 deletions.
18 changes: 9 additions & 9 deletions src/main/java/com/redhat/sso/ninja/Database2.java
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,13 @@ public void setVersion(String version){
public static String buildLinkMarkdown(Map<String,String> params){
if (!params.containsKey("linkId")) return "";
if (params.get("id").startsWith("TR")){
return "[Trello: "+params.get("linkId")+"/"+params.get("id")+"](https://trello.com/c/"+params.get("linkId");
return "[Trello: "+params.get("linkId")+"/"+params.get("id")+"](https://trello.com/c/"+params.get("linkId")+")";
}else if (params.get("id").startsWith("GH")){
if (params.get("pool").toLowerCase().contains("pull")){
return "[Github PR: "+params.get("linkId")+"](https://github.com/"+params.get("org")+"/"+params.get("board")+"/pull/"+params.get("linkId")+")";
}else{ // assume "issues"
return "[Github Issue: "+params.get("linkId")+"](https://github.com/"+params.get("org")+"/"+params.get("board")+"/issues/"+params.get("linkId")+")";
}
return "<https://github.com/"+params.get("org")+"/"+params.get("board")+"/pull/"+params.get("linkId")+">";
}else{ // assume "issues"
return "<https://github.com/"+params.get("org")+"/"+params.get("board")+"/issues/"+params.get("linkId")+">";
}
}
return "";
}
Expand Down Expand Up @@ -109,11 +109,11 @@ public Database2 increment(String poolId, String userId, Integer increment, Map<

if (params!=null && params.size()>1){ //because "id" is always added
// addEvent("Points Increment", userId, increment+" point"+(increment<=1?"":"s")+" added to "+poolId+" "+buildLink(params));
addEvent2("Points Increment", userId, increment, buildLinkMarkdown(params), poolId, "");
addEvent2("Points Increment", userId, increment, buildLinkMarkdown(params), poolId);
}else{
// no params & therefore no link
// addEvent("Points Increment", userId, increment+" point"+(increment<=1?"":"s")+" added to "+poolId+"");
addEvent2("Points Increment", userId, increment, "", poolId, "");
addEvent2("Points Increment", userId, increment, "", poolId);
}

}else{
Expand Down Expand Up @@ -172,12 +172,12 @@ public enum TASK_FIELDS{
}


public void addEvent2(String type, String user, Integer points, String source, String pool, String text){
public void addEvent2(String type, String user, Integer points, String source, String pool){
Map<String,String> event=new HashMap<String, String>();
event.put(EVENT_FIELDS.TIMESTAMP.v, sdf2.format(new Date()));
event.put(EVENT_FIELDS.TYPE.v, type);
event.put(EVENT_FIELDS.USER.v, user);
if (text!=null && !"".equals(text)) event.put(EVENT_FIELDS.TEXT.v, text);
// if (text!=null && !"".equals(text)) event.put(EVENT_FIELDS.TEXT.v, text);
event.put(EVENT_FIELDS.POINTS.v, String.valueOf(points));
event.put(EVENT_FIELDS.SOURCE.v, source);
event.put(EVENT_FIELDS.POOL.v, pool);
Expand Down
40 changes: 12 additions & 28 deletions src/main/java/com/redhat/sso/ninja/Heartbeat2.java
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,10 @@
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.mortbay.log.Log;

import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.redhat.sso.ninja.ChatNotification.ChatEvent;
import com.redhat.sso.ninja.ScriptRunner.ProcessResult;
import com.redhat.sso.ninja.controllers.EventsController;
import com.redhat.sso.ninja.user.UserService;
import com.redhat.sso.ninja.user.UserService.User;
Expand Down Expand Up @@ -464,35 +466,18 @@ public void run() {
command=convertLastRun(command, lastRun);
}


log.info("Script downloaded ("+version+"): "+originalCommand);
log.info("Script executing: "+command);

Process script_exec=Runtime.getRuntime().exec(command);
script_exec.waitFor();
if(script_exec.exitValue() != 0){

BufferedReader stdInput=new BufferedReader(new InputStreamReader(script_exec.getInputStream()));
StringBuffer sb=new StringBuffer();
String s;
while ((s=stdInput.readLine()) != null) sb.append(s).append("\n");
log.error("Error while executing script (stdout): "+sb.toString());

BufferedReader stdErr=new BufferedReader(new InputStreamReader(script_exec.getErrorStream()));
sb.setLength(0);
while ((s=stdErr.readLine()) != null) sb.append(s).append("\n");
log.error("Error while executing script (stderr): "+sb.toString());

db.addEvent("Script Execution FAILED", "", command+"\nERROR (stderr):\n"+sb.toString());

new ChatNotification().send(ChatEvent.onScriptError, name+" script failure occurred. Please investigate");

scriptFailure=true;

}else{
allocatePoints(db, script_exec.getInputStream(), script, scriptFolder, poolToUserIdMapper);
db.addEvent("Script Execution Succeeded", "", command +" (took "+(System.currentTimeMillis()-start)+"ms)");

PointsAllocation allocate=new PointsAllocation().database(db).mapper(poolToUserIdMapper).scriptName((String)script.get("name"));

ProcessResult process=new ScriptRunner().run(scriptFolder, command);
if (0==process.exitValue()){
for(String line:process.lines())
allocate.allocatePoints2(line);
}else{ // error
db.addEvent("Script Execution FAILED", "", command+"\nERROR (stderr):\n"+ Joiner.on("\n").join(process.lines()));
new ChatNotification().send(ChatEvent.onScriptError, name+" script failure occurred. Please investigate");
}

}catch (IOException e){
Expand Down Expand Up @@ -616,7 +601,6 @@ public void publishGraphsData(Database2 db, Config config){
try{
Map<String,String> filters=new MapBuilder<String,String>().put("daysOld", "180").put("events","User Promotion,Points Increment").put("asCSV", "true").build();
String events=(String)ec.getEventsV2(filters);
System.out.println("\n\n\n"+events);
if (200!=Http.post(url+"/events180", events).responseCode)
log.error("Error pushing 'events180' info to graphsProxy");
}catch (IOException e){ e.printStackTrace(); }
Expand Down Expand Up @@ -652,7 +636,7 @@ public void levelUpChecks(Database2 db){
db.addEvent("User Promotion", userInfo.get("username"), "Promoted to "+nextLevel.getRight());

String displayName=userInfo.containsKey("displayName")?userInfo.get("displayName"):userInfo.get("username");
String message="<https://mojo.redhat.com/people/"+userInfo.get("username")+"|"+displayName+"> promoted to "+nextLevel.getRight()+" belt";
String message=displayName +" promoted to "+nextLevel.getRight();
db.addTask(message, userInfo.get("username"));

// Notify everyone on the Ninja chat group of a new belt promotion
Expand Down
99 changes: 99 additions & 0 deletions src/main/java/com/redhat/sso/ninja/PointsAllocation.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package com.redhat.sso.ninja;

import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;

import com.redhat.sso.ninja.utils.ParamParser;

public class PointsAllocation{
private static final Logger log=Logger.getLogger(PointsAllocation.class);

private Database2 db;
private String scriptName;
private Map<String, String> mapper;

public PointsAllocation database(Database2 db){
this.db=db; return this;
}
public PointsAllocation scriptName(String scriptName){
this.scriptName=scriptName; return this;
}
public PointsAllocation mapper(Map<String, String> mapper){
this.mapper=mapper; return this;
}

private Pattern paramsPattern=Pattern.compile(".*(\\[.*\\]).*");
public void allocatePoints2(String line) throws UnsupportedEncodingException{

String s=line.trim();
// scriptLog.append(s).append("\n");
log.debug(s);

Map<String, String> params=new HashMap<String, String>();
// check for params here, extract them if present for use later on
if (s.matches(".* \\[.*\\]")){
Matcher m=paramsPattern.matcher(s);
if (m.find()){
String paramsExtract=m.group(1);
params.putAll(new ParamParser().splitParams(paramsExtract.replaceAll("\\[", "").replaceAll("\\]", "").trim()));
s=s.replaceAll("\\[.*\\]", "").trim();
}
}

if (s.startsWith("#")){ // Informational lines only, some may need to be added to event logging as reasons points were not awarded



}else if (s.contains("/")){ // ignore the line if it doesn't contain a slash
String[] split=s.split("/");

// take the last section of the script name as the pool id. so "trello" stays as "trello", but "trello.thoughtleadership" becomes "thoughtleadership" where the "trello" part is the source type/context
String pool=(String)scriptName;
String[] splitPool=pool.split("\\.");
pool=splitPool[splitPool.length-1];

String actionId;
String poolUserId;
Integer inc;

if (split.length==4){ //pool.sub
pool=pool+"."+split[0];
actionId=split[1];
poolUserId=split[2];
inc=Integer.valueOf(split[3]);

params.put("id", actionId);
params.put("pool", pool);

if (!db.getPointsDuplicateChecker().contains(actionId+"."+poolUserId)){
db.getPointsDuplicateChecker().add(actionId+"."+poolUserId);

String userId=mapper.get(poolUserId); // convert the channel (trello, github etc..) to the kerberos username

if (null!=userId){
db.increment(pool, userId, inc, params);//.save();
}else{
log.info("Unable to find '"+poolUserId+"' "+scriptName+" user - not registered? "+Database2.buildLink(params));
db.addEvent("Lost Points", poolUserId +"("+scriptName+")", scriptName+" user '"+poolUserId+"' was not found - not registered? "+Database2.buildLink(params));
}
}else{
// it's a duplicate increment for that actionId & user, so ignore it
log.warn(actionId+"."+poolUserId+" is a duplicate");
}

}else{
// dont increment because we dont know the structure of the script data
}


}

}


}
41 changes: 41 additions & 0 deletions src/main/java/com/redhat/sso/ninja/ScriptRunner.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package com.redhat.sso.ninja;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

public class ScriptRunner{

public class ProcessResult{
private List<String> lines=new ArrayList<String>();
private int exitValue;
public int exitValue(){
return this.exitValue;
}
public List<String> lines(){
return this.lines;
}
}

public ProcessResult run(File workingFolder, String command) throws IOException, InterruptedException{
boolean isWindows=false;
ProcessBuilder pBuilder=new ProcessBuilder(isWindows?"cmd.exe":"/bin/sh").directory(workingFolder).command(command.split(" "));
pBuilder.redirectErrorStream(true);
Process process=pBuilder.start();
String buf;
ProcessResult result=new ProcessResult();
BufferedReader reader=new BufferedReader(new InputStreamReader(process.getInputStream()));
while ((buf=reader.readLine())!=null) {
result.lines.add(buf);
}
process.waitFor(10, TimeUnit.SECONDS);
result.exitValue=process.exitValue();
return result;
}


}
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,6 @@ private List<Map<String, String>> getFilteredEvents1(Map<String,String> filters)
for(Map<String, String> v:result){
// add a generated "text" field if none exists - this is because on the events UI there is not enough space for all the separated fields
if ("Points Increment".equals(v.get(EVENT_FIELDS.TYPE.v)) && !v.containsKey(EVENT_FIELDS.TEXT.v)){
// System.out.println("adding a text field");
// if (!v.containsKey(EVENT_FIELDS.TEXT.v)){// && v.containsKey(EVENT_FIELDS.POINTS.v)){
Integer points=Integer.parseInt(v.get(EVENT_FIELDS.POINTS.v));
v.put(EVENT_FIELDS.TEXT.v, points+" point"+(points<=1?"":"s")+" added to "+v.get(EVENT_FIELDS.POOL.v)+" "+convertSourceToShowdown(v.get(EVENT_FIELDS.SOURCE.v)));
}
Expand Down Expand Up @@ -218,7 +216,7 @@ private List<Map<String, String>> getFilteredEvents2(Map<String,String> filters)
}
}

if (include) result.add(e);
if (include) result.add(new HashMap<>(e));
}
}
}
Expand Down
11 changes: 9 additions & 2 deletions src/main/webapp/mojo/igloo-ninjaWall.jsp
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@
users.push(username);
var belt = json['custom1'][i].split("|")[1];
var geo = json['custom1'][i].split("|")[2];
belt = (belt == "zero" ? "No" : "<span class='belt-" + belt + "'>" + uCase(belt)) + " Star<\/span>";
belt = (belt == "zero" ? "No" : "<span class='belt-" + belt + "'>" + beltToStar(uCase(belt))) + " Star<\/span>";
var NL = "<br/>";
var badges = "";
Expand Down Expand Up @@ -187,7 +187,14 @@
{
return string.charAt(0).toUpperCase() + string.slice(1);
}
function beltToStar(string)
{
if (string == "Black") return "Gold";
if (string == "Red") return "Green";
return string;
}
function getUserPics(users)
{
var jqxhr = jQuery.getJSON("/.api2/api/v1/communities/10/search/members?query=" + users.join("+OR+") + "&memberSearchType=email&limit=1000", function(data)
Expand Down
54 changes: 43 additions & 11 deletions src/main/webapp/mojo/mojo-dashboard-card.jsp
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
<script src="https://www.chartjs.org/dist/2.7.2/Chart.bundle.js"></script>
<script src="../js/http.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/showdown/1.6.4/showdown.min.js"></script>
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.7/css/all.css"/>
<script>
var ctx="https://ninja-graphs-ninja-graphs.6923.rh-us-east-1.openshiftapps.com/ninja-graphs/api/proxy";
Expand Down Expand Up @@ -552,32 +553,63 @@ function refresh(){
//refresh();
</script>


<script>
$(document).ready(function() {
var markdown;
$(document).ready(function(){
if (typeof showdown !== 'undefined')
markdown=new showdown.Converter();
});
function showPointsAllocations(){
$("#eventsBody").html("");
Http.httpGet("../api/events?user="+getUsername(), function(response,status){
console.log("status="+status);
response=JSON.parse(response);
for(i in response){
var item=response[i];
if (item["type"]=="Points Increment"){
var ts=item["timestamp"].split("T")[0];
var txt=item["text"].substring(0, item["text"].indexOf("added"));
txt+="- ";
var txt;
if (item["text"].indexOf("[")>=0){
var first=item["text"].substring(item["text"].indexOf("http"));
first=item["text"].substring(item["text"].indexOf("["));
first=first.substring(0, first.indexOf("]"))+"]";
// support newer split event field format (post Jan 2021). although there are 2 markdown formats to support. one <url> and the other [name](url)
if (item["points"] && item["source"]){
var source;
// display the links a bit nicer if we can, extracting bits using regex
if (item["source"].startsWith("<") && item["source"].endsWith(">")){
var name;
if (item["source"].includes("github")){
var match=item["source"].match(/http.*\/\/(.+)\/(.+)\/(.+)\/(.+)\/(.+)>/);
name=match[2]+"/"+match[3]+" "+match[4]+" "+match[5]; // 2=org, 3=board, 4=contribution type, 5=id
}
if (item["source"].includes("gitlab")){
var match=item["source"].match(/http.*\/\/(.+)\/(.+)\/(.+)\/(.+)\/(.+)\/(.+)>/);
name=match[2]+"/"+match[3]+" "+match[5]+" "+match[6]; // 2=org, 3=board, 5=contribution type, 6=id
}
if (item["source"].includes("trello")){
var match=item["source"].match(/http.*\/\/(.+)\.(.+)\/(.+)\/(.+)>/); // 1=trello, 4=trello card id
name=match[1]+"/"+match[4];
}
source="["+name+"]("+item["source"]+")";
}else // assume link has a descriptor, so use as is
source=item["source"]
txt=item["points"]+" point - "+source;
txt=markdown.makeHtml(txt);
}else{
first="Unknown";
// support for older event text messages (pre Jan 2021)
txt=item["text"].substring(0, item["text"].indexOf("added"));
txt+="- ";
// support pre showdown markup link in the following format [url](name)
if (item["text"].indexOf("[")>=0){
var first=item["text"].substring(item["text"].indexOf("http"));
first=item["text"].substring(item["text"].indexOf("["));
first=first.substring(0, first.indexOf("]"))+"]";
}else{
first="Unknown";
}
txt+=processText(first);
}
txt+=processText(first);
//txt=processText(item["text"]);
$("#eventsBody").append("<tr><td>"+ts+"</td><td>"+txt+"</td></tr>");
}
Expand Down

0 comments on commit fa1ddb3

Please sign in to comment.