Add comment, fix jcstress result collection bug, run all tests with j…

…cstress
JetBrains-Research · Aug 4, 2024 · d8dd41f · d8dd41f
1 parent 29cfa94
commit d8dd41f
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 6 deletions.
diff --git a/.github/workflows/run-litmus.yml b/.github/workflows/run-litmus.yml
@@ -7,6 +7,9 @@ on:
       - development
       - main
 
+# Note: this CI run is an "integration"-test or "smoke"-test. It is intended to verify that
+# the basics of the tool work. It is NOT intended to be complete or to discover weak behaviors.
+
 jobs:
   linux-run:
     runs-on: ubuntu-latest
@@ -21,8 +24,9 @@ jobs:
         run: ./gradlew cli:linkReleaseExecutableLinuxX64
       - name: Run litmus tests via CLI
         run: ./cli/build/bin/linuxX64/releaseExecutable/cli.kexe -r pthread ".*"
-      - name: Run a single test with JCStress
-        run: ./gradlew :cli:jvmRun --args="-r jcstress -j '-m quick' StoreBuffering.Plain"
+      - name: Run litmus tests with JCStress
+        # takes ~10 mins
+        run: ./gradlew :cli:jvmRun --args="-r jcstress -j '-m sanity' .*"
 
   macos-run:
     runs-on: macos-latest
@@ -37,8 +41,9 @@ jobs:
         run: ./gradlew cli:linkReleaseExecutableMacosX64
       - name: Run litmus tests via CLI (x64 + release)
         run: ./cli/build/bin/macosX64/releaseExecutable/cli.kexe -r pthread ".*"
-      - name: Run a single test with JCStress
-        run: ./gradlew :cli:jvmRun --args="-r jcstress -j '-m quick' StoreBuffering.Plain"
+      - name: Run litmus tests with JCStress
+        # takes ~10 mins
+        run: ./gradlew :cli:jvmRun --args="-r jcstress -j '-m sanity' .*"
       - name: Assemble CLI binary (arm + release)
         run: ./gradlew cli:linkReleaseExecutableMacosArm64
       - name: Run litmus tests via CLI (arm + release)

diff --git a/jcstress-wrapper/src/main/kotlin/org/jetbrains/litmuskt/JCStressRunner.kt b/jcstress-wrapper/src/main/kotlin/org/jetbrains/litmuskt/JCStressRunner.kt
@@ -71,7 +71,11 @@ class JCStressRunner(
         return handle@{
             jcs.waitFor()
             if (jcs.exitValue() != 0) error("jcstress exited with code ${jcs.exitValue()}")
-            return@handle tests.associateWith { test -> parseJCStressResults(test) }
+            // not all tests might have generated wrappers
+            return@handle tests
+                .associateWith { test -> parseJCStressResults(test) }
+                .filterValues { it != null }
+                .mapValues { (_, result) -> result!! } // remove nullable type
         }
     }
 
@@ -100,8 +104,9 @@ class JCStressRunner(
      * </tr>    <-- these lines repeat per each configuration, so the results are summed in the end
      * ...
      */
-    private fun parseJCStressResults(test: LitmusTest<*>): LitmusResult {
+    private fun parseJCStressResults(test: LitmusTest<*>): LitmusResult? {
         val resultsFile = jcstressDirectory / "results" / "${test.javaFQN}.html"
+        if (Files.notExists(resultsFile)) return null
         var lines = Files.lines(resultsFile).asSequence()
 
         val allOutcomes = test.outcomeSpec.all