Merge pull request #20 from JetBrains-Research/dev-performance

Performance improvements (let each thread collect a portion of results)

cli/src/commonMain/kotlin/org/jetbrains/litmuskt/CliCommon.kt

@@ -85,11 +85,11 @@ abstract class CliCommon : CliktCommand(
             echo("running test ${test.alias}...")
             // TODO: handle exceptions
             // TODO: print ETA (later: calculate based on part of run)
-            paramsList.map { params ->
+            val result = paramsList.map { params ->
                 runTest(params, test)
-            }.mergeResults().let {
-                echo(it.generateTable())
-            }
+            }.mergeResults()
+            echo(result.generateTable(), false)
+            echo("total count: ${result.totalCount()}, overall status: ${result.overallStatus()}")
             echo()
         }
     }

core/src/commonMain/kotlin/org/jetbrains/litmuskt/LitmusOutcomeStats.kt

@@ -7,7 +7,7 @@ enum class LitmusOutcomeType { ACCEPTED, INTERESTING, FORBIDDEN }
 data class LitmusOutcomeStats(
     val outcome: LitmusOutcome,
     val count: Long,
-    val type: LitmusOutcomeType?,
+    val type: LitmusOutcomeType,
 )
 
 data class LitmusOutcomeSpec(
@@ -31,9 +31,7 @@ data class LitmusOutcomeSpec(
  * In other cases, use `accept(setOf(...))` to accept one or many values. Note that to accept an iterable,
  * it has to be wrapped in an extra `setOf()`. All of this applies as well to `interesting()` and `forbid()`.
  *
- * See [LitmusAutoOutcome] file for those extension functions. The generic <S> is used precisely for them.
- *
- * single values are handled differently !!!!!!!!!!!!! TODO
+ * See [LitmusAutoOutcome] file for `accept(r1, ...)` extension functions. The generic <S> is used precisely for them.
  */
 class LitmusOutcomeSpecScope<S : Any> {
     private val accepted = mutableSetOf<LitmusOutcome>()
@@ -63,32 +61,3 @@ class LitmusOutcomeSpecScope<S : Any> {
 
     fun build() = LitmusOutcomeSpec(accepted, interesting, forbidden, default ?: LitmusOutcomeType.FORBIDDEN)
 }
-
-typealias LitmusResult = List<LitmusOutcomeStats>
-
-fun LitmusResult.generateTable(): String {
-    val totalCount = sumOf { it.count }
-    val table = this.sortedByDescending { it.count }.map {
-        val freq = it.count.toDouble() / totalCount
-        listOf(
-            it.outcome.toString(),
-            it.type.toString(),
-            it.count.toString(),
-            if (freq < 1e-5) "<0.001%" else "${(freq * 100).toString().take(6)}%"
-        )
-    }
-    val tableHeader = listOf("outcome", "type", "count", "frequency")
-    return (listOf(tableHeader) + table).tableFormat(true)
-}
-
-fun List<LitmusResult>.mergeResults(): LitmusResult {
-    data class LTOutcomeStatTempData(var count: Long, var type: LitmusOutcomeType?)
-
-    val statMap = mutableMapOf<LitmusOutcome, LTOutcomeStatTempData>()
-    for (stat in this.flatten()) {
-        val tempData = statMap.getOrPut(stat.outcome) { LTOutcomeStatTempData(0L, stat.type) }
-        if (tempData.type != stat.type) error("merging conflicting stats: ${stat.outcome} is both ${stat.type} and ${tempData.type}")
-        tempData.count += stat.count
-    }
-    return statMap.map { (outcome, tempData) -> LitmusOutcomeStats(outcome, tempData.count, tempData.type) }
-}

core/src/commonMain/kotlin/org/jetbrains/litmuskt/LitmusResult.kt

@@ -0,0 +1,44 @@
+package org.jetbrains.litmuskt
+
+typealias LitmusResult = List<LitmusOutcomeStats>
+
+fun LitmusResult.generateTable(): String {
+    val totalCount = sumOf { it.count }
+    val table = this.sortedByDescending { it.count }.map {
+        val freq = it.count.toDouble() / totalCount
+        listOf(
+            it.outcome.toString(),
+            it.type.toString(),
+            it.count.toString(),
+            if (freq < 1e-5) "<0.001%" else "${(freq * 100).toString().take(6)}%"
+        )
+    }
+    val tableHeader = listOf("outcome", "type", "count", "frequency")
+    return (listOf(tableHeader) + table).tableFormat(true)
+}
+
+fun LitmusResult.totalCount() = sumOf { it.count }
+
+fun LitmusResult.overallStatus(): LitmusOutcomeType {
+    var isInteresting = false
+    for (stat in this) when (stat.type) {
+        LitmusOutcomeType.FORBIDDEN -> return LitmusOutcomeType.FORBIDDEN
+        LitmusOutcomeType.INTERESTING -> isInteresting = true
+        LitmusOutcomeType.ACCEPTED -> {} // ignore
+    }
+    return if (isInteresting) LitmusOutcomeType.INTERESTING else LitmusOutcomeType.ACCEPTED
+}
+
+fun List<LitmusResult>.mergeResults(): LitmusResult {
+    data class LTOutcomeStatsAccumulator(var count: Long, val type: LitmusOutcomeType)
+
+    val statMap = mutableMapOf<LitmusOutcome, LTOutcomeStatsAccumulator>()
+    for (stat in this.flatten()) {
+        val tempData = statMap.getOrPut(stat.outcome) { LTOutcomeStatsAccumulator(0L, stat.type) }
+        if (tempData.type != stat.type) {
+            error("merging conflicting stats: ${stat.outcome} is both ${stat.type} and ${tempData.type}")
+        }
+        tempData.count += stat.count
+    }
+    return statMap.map { (outcome, tempData) -> LitmusOutcomeStats(outcome, tempData.count, tempData.type) }
+}

core/src/commonMain/kotlin/org/jetbrains/litmuskt/LitmusRunner.kt

@@ -61,7 +61,7 @@ abstract class LitmusRunner {
     }
 
     protected fun <S : Any> calcStats(
-        states: Array<S>,
+        states: Iterable<S>,
         spec: LitmusOutcomeSpec,
         outcomeFinalizer: (S) -> LitmusOutcome
     ): LitmusResult {

core/src/commonMain/kotlin/org/jetbrains/litmuskt/Threadlike.kt

@@ -6,10 +6,11 @@ package org.jetbrains.litmuskt
  */
 interface Threadlike {
     /**
-     * Start running the function in a "thread". Returns a handle that will block when called until
-     * the "thread" has completed.
+     * Start running the function in a "thread". Note that the function should be non-capturing.
      *
      * This function should be only called once.
+     *
+     * @return a "future" handle that will block when called until the "thread" has completed.
      */
     fun <A : Any> start(args: A, function: (A) -> Unit): BlockingFuture
 

core/src/commonMain/kotlin/org/jetbrains/litmuskt/ThreadlikeRunner.kt

@@ -5,17 +5,23 @@ abstract class ThreadlikeRunner : LitmusRunner() {
     protected abstract fun threadlikeProducer(): Threadlike
 
     private fun <S : Any> threadFunction(threadContext: ThreadContext<S>) = with(threadContext) {
+        val testFunction = test.threadFunctions[threadIndex]
         for (i in states.indices) {
             if (i % syncPeriod == 0) barrier.await()
-            states[i].testThreadFunction()
+            states[i].testFunction()
         }
+        // performance optimization: each thread takes a portion of states and calculates stats for it
+        rangeResult = calcStats(states.view(resultCalcRange), test.outcomeSpec, test.outcomeFinalizer)
     }
 
     private data class ThreadContext<S : Any>(
         val states: Array<S>,
-        val testThreadFunction: S.() -> Unit,
+        val test: LitmusTest<S>,
+        val threadIndex: Int,
         val syncPeriod: Int,
         val barrier: Barrier,
+        val resultCalcRange: IntRange,
+        var rangeResult: LitmusResult? = null
     )
 
     override fun <S : Any> startTest(
@@ -29,8 +35,10 @@ abstract class ThreadlikeRunner : LitmusRunner() {
         val threads = List(test.threadCount) { threadlikeProducer() }
 
         val barrier = barrierProducer(test.threadCount)
+        val resultCalcRanges = states.indices.splitEqual(threads.size)
         val contexts = List(threads.size) { i ->
-            ThreadContext(states, test.threadFunctions[i], syncPeriod, barrier)
+            val range = resultCalcRanges[i]
+            ThreadContext(states, test, i, syncPeriod, barrier, range)
         }
 
         val futures = (threads zip contexts).map { (thread, context) ->
@@ -49,7 +57,7 @@ abstract class ThreadlikeRunner : LitmusRunner() {
         return {
             futures.forEach { it.await() } // await all results
             threads.forEach { it.dispose() } // stop all "threads"
-            calcStats(states, test.outcomeSpec, test.outcomeFinalizer)
+            contexts.map { it.rangeResult!! }.mergeResults()
         }
     }
 }

core/src/commonMain/kotlin/org/jetbrains/litmuskt/Utils.kt

@@ -27,3 +27,28 @@ expect fun cpuCount(): Int
 
 @Suppress("UNCHECKED_CAST")
 fun <S> TypedArray(size: Int, init: (Int) -> S): Array<S> = Array<Any?>(size, init) as Array<S>
+
+/**
+ * Returns a lazy iterable that iterates over a portion of the underlying array.
+ */
+fun <S> Array<S>.view(range: IntRange): Iterable<S> = sequence {
+    for (i in range) yield(this@view[i])
+}.asIterable()
+
+/**
+ * Split a range into [n] parts of equal (+/- 1) length.
+ */
+fun IntRange.splitEqual(n: Int): List<IntRange> {
+    val size = endInclusive - start + 1
+    val len = size / n // base length of each sub-range
+    val remainder = size % n
+    val delim = start + (len + 1) * remainder // delimiter between lengths (l+1) and l
+    return List(n) { i ->
+        if (i < remainder) {
+            (start + i * (len + 1))..<(start + (i + 1) * (len + 1))
+        } else {
+            val j = i - remainder
+            (delim + j * len)..<(delim + (j + 1) * len)
+        }
+    }
+}

core/src/commonTest/kotlin/org/jetbrains/litmuskt/UtilTest.kt

@@ -0,0 +1,28 @@
+package org.jetbrains.litmuskt
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class UtilTest {
+
+    @Test
+    fun testRangeSplitEqual() {
+        assertEquals(
+            listOf(0..<4, 4..<8, 8..<11),
+            (0..<11).splitEqual(3)
+        )
+        assertEquals(
+            listOf(0..<3, 3..<5, 5..<7, 7..<9, 9..<11),
+            (0..<11).splitEqual(5)
+        )
+        assertEquals(
+            listOf(0..<500, 500..<1000),
+            (0..<1000).splitEqual(2)
+        )
+        assertEquals(
+            listOf(1..1, 2..2),
+            (1..2).splitEqual(2)
+        )
+    }
+
+}

core/src/jvmMain/kotlin/org/jetbrains/litmuskt/JvmThreadRunner.kt

@@ -27,7 +27,7 @@ class JvmThreadRunner : LitmusRunner() {
 
         return {
             threads.forEach { it.join() }
-            calcStats(states, test.outcomeSpec, test.outcomeFinalizer)
+            calcStats(states.asIterable(), test.outcomeSpec, test.outcomeFinalizer)
         }
     }
 }