Skip to content

Commit

Permalink
Minor improvements to verification crawler.
Browse files Browse the repository at this point in the history
  • Loading branch information
@davewichers
davewichers committed Dec 27, 2023
1 parent 845cfaa commit e51c50d
Show file tree
Hide file tree
Showing 6 changed files with 906 additions and 102 deletions.
21 changes: 11 additions & 10 deletions plugin/src/main/java/org/owasp/benchmarkutils/score/BenchmarkScore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -722,12 +722,12 @@ private static Map<String, TP_FN_TN_FP_Counts> calculateScores(TestSuiteResults
map.put(cat, c);
}
// real vulnerabilities
if (tcr.isReal() && tcr.isPassed()) c.tp++; // tp
else if (tcr.isReal() && !tcr.isPassed()) c.fn++; // fn
if (tcr.isTruePositive() && tcr.isPassed()) c.tp++; // tp
else if (tcr.isTruePositive() && !tcr.isPassed()) c.fn++; // fn

// fake vulnerabilities
else if (!tcr.isReal() && tcr.isPassed()) c.tn++; // tn
else if (!tcr.isReal() && !tcr.isPassed()) c.fp++; // fp
else if (!tcr.isTruePositive() && tcr.isPassed()) c.tn++; // tn
else if (!tcr.isTruePositive() && !tcr.isPassed()) c.fp++; // fp
}
return map;
}
Expand Down Expand Up @@ -801,7 +801,8 @@ private static TestSuiteResults analyze(
pass = compare(exp, act, rawToolResults.getToolName());

// helpful in debugging
// System.out.println( tc + ", " + exp.getCategory() + ", " + exp.isReal() + ", " +
// System.out.println( tc + ", " + exp.getCategory() + ", " + exp.isTruePositive() + ",
// " +
// exp.getCWE() + ", " + pass + "\n");

// fill the result into the "expected" results in case we need it later
Expand All @@ -828,7 +829,7 @@ private static TestSuiteResults analyze(
private static boolean compare(TestCaseResult exp, List<TestCaseResult> actList, String tool) {
// return true if there are no actual results and this was a false positive test
if (actList == null || actList.isEmpty()) {
return !exp.isReal();
return !exp.isTruePositive();
}

// otherwise check actual results
Expand Down Expand Up @@ -860,11 +861,11 @@ private static boolean compare(TestCaseResult exp, List<TestCaseResult> actList,

// return true if we find an exact match for a True Positive test
if (match) {
return exp.isReal();
return exp.isTruePositive();
}
}
// if we couldn't find a match, then return true if it's a False Positive test
return !exp.isReal();
return !exp.isTruePositive();
}

// Create a TestResults object that contains the expected results for this version
Expand Down Expand Up @@ -921,7 +922,7 @@ private static TestSuiteResults readExpectedResults(File file) {
TestCaseResult tcr = new TestCaseResult();
tcr.setTestCaseName(parts[0]);
tcr.setCategory(parts[1]);
tcr.setReal(Boolean.parseBoolean(parts[2]));
tcr.setTruePositive(Boolean.parseBoolean(parts[2]));
tcr.setCWE(Integer.parseInt(parts[3]));

tcr.setNumber(Reader.testNumber(parts[0]));
Expand Down Expand Up @@ -1005,7 +1006,7 @@ private static String produceResultsFile(TestSuiteResults actual, File scoreCard
ps.print("," + actualResult.getDataFlow());
ps.print("," + actualResult.getSink());
}
boolean isreal = actualResult.isReal();
boolean isreal = actualResult.isTruePositive();
ps.print(", " + isreal);
boolean passed = actualResult.isPassed();
boolean toolresult = !(isreal ^ passed);
Expand Down
62 changes: 44 additions & 18 deletions plugin/src/main/java/org/owasp/benchmarkutils/score/TestCaseResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
*/
package org.owasp.benchmarkutils.score;

import org.owasp.benchmarkutils.tools.AbstractTestCaseRequest;

/* This class represents a single test case result. It documents the expected result (real),
* and the actual result (result).
*/
Expand All @@ -25,8 +27,8 @@ public class TestCaseResult {

private String testCaseName = "";
private int number = 0;
private boolean real = false;
private boolean result = false;
private boolean truePositive = false; // Is this test case a true or false positive?
private boolean result = false; // Did a tool properly detect this as a true or false positive?
private int CWE = 0;
private String category = null;
private String evidence = null;
Expand All @@ -37,6 +39,29 @@ public class TestCaseResult {
private String dataflow = null;
private String sink = null;

public TestCaseResult() {
// By default, do nothing special.
}

/**
* Convert what we know about a TestCase Request description back into a TestCaseResult
* (expected or actual)
*
* @param request The request object used to access this test case.
*/
public TestCaseResult(AbstractTestCaseRequest request) {
this.testCaseName = request.getName();
this.number = request.getNumber();
this.truePositive = request.isVulnerability();
this.CWE = request.getCategory().getCWE();
this.category = request.getCategory().getName();

// fill in optional attributes since we have this data available
this.source = request.getSourceFile();
this.dataflow = request.getDataflowFile();
this.sink = request.getSinkFile();
}

/*
* Set the name of the test case (E.g., BenchmarkTest00001). This is frequently only used for
* expected results, not actual results. Expected to actual can be correlated by the test number.
Expand Down Expand Up @@ -68,12 +93,12 @@ public void setNumber(int number) {
this.number = number;
}

public boolean isReal() {
return real;
public boolean isTruePositive() {
return truePositive;
}

public void setReal(boolean real) {
this.real = real;
public void setTruePositive(boolean truePositive) {
this.truePositive = truePositive;
}

public boolean isPassed() {
Expand Down Expand Up @@ -134,18 +159,19 @@ public void setSink(String sink) {

@Override
public String toString() {
return getNumber()
+ ","
return "Testcase #: "
+ getNumber()
+ ", Category: "
+ getCategory()
+ ","
+ isReal()
+ ","
+ ", isVulnerable: "
+ isTruePositive()
+ ", CWE: "
+ getCWE()
+ ","
+ isPassed()
+ ","
+ getEvidence()
+ ","
+ getConfidence();
}
+ ", toolPassed: "
+ isPassed();
/* + ", evidence: "
+ getEvidence()
+ ", confidence: "
+ getConfidence();
*/ }
}
83 changes: 37 additions & 46 deletions plugin/src/main/java/org/owasp/benchmarkutils/tools/BenchmarkCrawler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,44 +68,23 @@
@Mojo(name = "run-crawler", requiresProject = false, defaultPhase = LifecyclePhase.COMPILE)
public class BenchmarkCrawler extends AbstractMojo {

// Intended to be a Singleton. So when instantiated, put it here:
static BenchmarkCrawler thisInstance = null;

static final long MAX_NETWORK_TIMEOUT = 15; // seconds
public static String proxyHost, proxyPort;

@Parameter(property = "crawlerFile")
String pluginFilenameParam;

/*
* Attaching the @Parameter property to the crawlerFile variable directly didn't work for some
* reason. So I attached it to a new String variable, and set it later. No clue why it doesn't
* work. But for now, leaving it this way because it works.
*
* If you run the mvn command with -X, when invoking this plugin, you'd see something like
* this at the end:
*
* [DEBUG] (s) crawlerFile = /Users/PATH/TO/BenchmarkJava/data/benchmark-crawler-http.xml
* [DEBUG] -- end configuration --
* but the crawlerFile variable would be null.
*
* When it should be:
* [DEBUG] (f) crawlerFile = data/benchmark-crawler-http.xml
* [DEBUG] -- end configuration --
*
* So after changing this, I now get:
* [DEBUG] (f) pluginFilenameParam = data/benchmark-crawler-http.xml
* [DEBUG] -- end configuration --
* and the pluginFilenameParam variable value is set properly.
* Attaching the @Parameter property to the crawlerFile variable allows you to set the value directly when invoking via maven.
* For example: -DcrawlerFile=data/benchmark-crawler-http.xml
*/
String crawlerFile;
@Parameter(property = "crawlerFile")
String crawlerFile = null;

File theCrawlerFile;
String selectedTestCaseName = null;
TestSuite testSuite;

BenchmarkCrawler() {
// A default constructor required to support Maven plugin API.
// The theCrawlerFile has to be instantiated before a crawl can be done.
}

/** Crawl the target test suite. */
protected void run() {
try {
Expand Down Expand Up @@ -148,8 +127,22 @@ void load() {
}
}

public void setCrawlerFile(File theCrawlerFile) {
this.theCrawlerFile = theCrawlerFile;
/**
* Load the crawler file that defines all the test cases, including their endpoints, and how to
* crawl them.
*
* @param targetFileName The crawler file name
* @throws RuntimeException If the file doesn't exist or can't be opened for some reason.
*/
public void setCrawlerFile(String targetFileName) throws RuntimeException {
File targetFile = new File(targetFileName);
if (targetFile.exists()) {
this.crawlerFile = targetFileName;
this.theCrawlerFile = targetFile;
} else {
throw new RuntimeException(
"Could not find crawler configuration file: '" + targetFileName + "'");
}
}

/**
Expand Down Expand Up @@ -309,7 +302,7 @@ static ResponseInfo sendRequest(CloseableHttpClient httpclient, HttpUriRequest r
* @param args - args passed to main().
* @return specified crawler file if valid command line arguments provided. Null otherwise.
*/
private void processCommandLineArgs(String[] args) {
protected void processCommandLineArgs(String[] args) {

// Create the command line parser
CommandLineParser parser = new DefaultParser();
Expand All @@ -336,14 +329,7 @@ private void processCommandLineArgs(String[] args) {
CommandLine line = parser.parse(options, args);

if (line.hasOption("f")) {
this.crawlerFile = line.getOptionValue("f");
File targetFile = new File(this.crawlerFile);
if (targetFile.exists()) {
setCrawlerFile(targetFile);
} else {
throw new RuntimeException(
"Could not find crawler configuration file '" + this.crawlerFile + "'");
}
setCrawlerFile(line.getOptionValue("f"));
}
if (line.hasOption("h")) {
formatter.printHelp("BenchmarkCrawlerVerification", options, true);
Expand All @@ -359,19 +345,24 @@ private void processCommandLineArgs(String[] args) {

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
if (null == this.pluginFilenameParam) {
if (thisInstance == null) thisInstance = this;

if (null == this.crawlerFile) {
System.out.println("ERROR: A crawlerFile parameter must be specified.");
} else {
String[] mainArgs = {"-f", this.pluginFilenameParam};
String[] mainArgs = {"-f", this.crawlerFile};
main(mainArgs);
}
}

public static void main(String[] args) {

BenchmarkCrawler crawler = new BenchmarkCrawler();
crawler.processCommandLineArgs(args);
crawler.load();
crawler.run();
// thisInstance can be set from execute() or here, depending on how this class is invoked
// (via maven or commmand line)
if (thisInstance == null) {
thisInstance = new BenchmarkCrawler();
}
thisInstance.processCommandLineArgs(args);
thisInstance.load();
thisInstance.run();
}
}
54 changes: 26 additions & 28 deletions plugin/src/main/java/org/owasp/benchmarkutils/tools/BenchmarkCrawlerVerification.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,6 @@ public class BenchmarkCrawlerVerification extends BenchmarkCrawler {
SimpleFileLogger eLogger;
SimpleFileLogger uLogger;

BenchmarkCrawlerVerification() {
// A default constructor required to support Maven plugin API.
// The theCrawlerFile has to be instantiated before a crawl can be done.
}

/**
* Overload the base crawl() method to send both attack and safe requests, and verify whether
* the test exploit worked or not based on the results that came back in both the attack
Expand Down Expand Up @@ -283,11 +278,15 @@ protected static void handleResponse(TestCaseVerificationResults result)
* @param args - args passed to main().
* @return specified crawler file if valid command line arguments provided. Null otherwise.
*/
private void processCommandLineArgs(String[] args) {
@Override
protected void processCommandLineArgs(String[] args) {

// Set default attack crawler file
String crawlerFileName = new File(Utils.DATA_DIR, "benchmark-attack-http.xml").getPath();
this.theCrawlerFile = new File(crawlerFileName);
// Set default attack crawler file, if it exists
// This value can be changed by the -f parameter for other test suites with different names
File defaultAttackCrawlerFile = new File(Utils.DATA_DIR, "benchmark-attack-http.xml");
if (defaultAttackCrawlerFile.exists()) {
setCrawlerFile(defaultAttackCrawlerFile.getPath());
}

RegressionTesting.isTestingEnabled = true;

Expand All @@ -301,7 +300,7 @@ private void processCommandLineArgs(String[] args) {
options.addOption(
Option.builder("f")
.longOpt("file")
.desc("a TESTSUITE-crawler-http.xml file")
.desc("a TESTSUITE-attack-http.xml file")
.hasArg()
.required()
.build());
Expand All @@ -325,16 +324,10 @@ private void processCommandLineArgs(String[] args) {
CommandLine line = parser.parse(options, args);

if (line.hasOption("f")) {
this.crawlerFile = line.getOptionValue("f");
File targetFile = new File(this.crawlerFile);
if (targetFile.exists()) {
setCrawlerFile(targetFile);
// Crawler output files go into the same directory as the crawler config file
CRAWLER_DATA_DIR = targetFile.getParent() + File.separator;
} else {
throw new RuntimeException(
"Could not find crawler configuration file '" + this.crawlerFile + "'");
}
// Following throws a RuntimeException if the target file doesn't exist
setCrawlerFile(line.getOptionValue("f"));
// Crawler output files go into the same directory as the crawler config file
CRAWLER_DATA_DIR = this.theCrawlerFile.getParent() + File.separator;
}
if (line.hasOption("h")) {
formatter.printHelp("BenchmarkCrawlerVerification", options, true);
Expand All @@ -353,19 +346,24 @@ private void processCommandLineArgs(String[] args) {

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
if (null == this.pluginFilenameParam) {
System.out.println("ERROR: A crawlerFile parameter must be specified.");
if (thisInstance == null) thisInstance = this;

if (null == this.crawlerFile) {
System.out.println("ERROR: An attack crawlerFile parameter must be specified.");
} else {
String[] mainArgs = {"-f", this.pluginFilenameParam};
String[] mainArgs = {"-f", this.crawlerFile};
main(mainArgs);
}
}

public static void main(String[] args) {

BenchmarkCrawlerVerification crawler = new BenchmarkCrawlerVerification();
crawler.processCommandLineArgs(args);
crawler.load();
crawler.run();
// thisInstance can be set from execute() or here, depending on how this class is invoked
// (via maven or commmand line)
if (thisInstance == null) {
thisInstance = new BenchmarkCrawlerVerification();
}
thisInstance.processCommandLineArgs(args);
thisInstance.load();
thisInstance.run();
}
}
Loading

0 comments on commit e51c50d

Please sign in to comment.