Fixed crashes on high-performance servers. Stabilization of variation tests.

src/main/java/com/github/chrisgleissner/benchmarks/AbstractBenchmark.java

@@ -7,12 +7,12 @@
 import org.openjdk.jmh.annotations.OutputTimeUnit;
 import org.openjdk.jmh.annotations.Warmup;
 
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
 
 @BenchmarkMode(Mode.Throughput)
-@OutputTimeUnit(MILLISECONDS)
-@Warmup(iterations = 5, time = 1000, timeUnit = MILLISECONDS)
-@Measurement(iterations = 10, time = 1000, timeUnit = MILLISECONDS)
+@OutputTimeUnit(SECONDS)
+@Warmup(iterations = 50, time = 1, timeUnit = SECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = SECONDS)
 @Fork(1)
 public abstract class AbstractBenchmark {
 

src/main/java/com/github/chrisgleissner/benchmarks/CounterBenchmark.java

@@ -2,19 +2,18 @@
 
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.commons.lang3.mutable.MutableLong;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.OperationsPerInvocation;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.*;
 import org.openjdk.jmh.infra.Blackhole;
 
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static com.github.chrisgleissner.benchmarks.Constants.OPERATIONS_PER_PER_INVOCATION;
+import static java.util.concurrent.TimeUnit.*;
 
 @OperationsPerInvocation(OPERATIONS_PER_PER_INVOCATION)
+@Warmup(iterations = 20, time = 1, timeUnit = SECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = SECONDS)
 public class CounterBenchmark extends AbstractBenchmark {
 
     @Benchmark

src/main/java/com/github/chrisgleissner/benchmarks/ObjectCacheBenchmark.java

@@ -1,14 +1,15 @@
 package com.github.chrisgleissner.benchmarks;
 
 import lombok.Value;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.OperationsPerInvocation;
-
+import org.openjdk.jmh.annotations.*;
+import static java.util.concurrent.TimeUnit.*;
 import java.util.stream.IntStream;
 
 import static com.github.chrisgleissner.benchmarks.Constants.OPERATIONS_PER_PER_INVOCATION;
 
 @OperationsPerInvocation(OPERATIONS_PER_PER_INVOCATION)
+@Warmup(iterations = 20, time = 1, timeUnit = SECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = SECONDS)
 public class ObjectCacheBenchmark extends AbstractBenchmark {
 
     private static Id[] idPool = IntStream.range(0, OPERATIONS_PER_PER_INVOCATION).mapToObj(Id::new).toArray(Id[]::new);

src/main/java/com/github/chrisgleissner/benchmarks/TimeBenchmark.java

@@ -10,8 +10,8 @@
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
-@Warmup(iterations = 5, time = 300, timeUnit = MILLISECONDS)
-@Measurement(iterations = 5, time = 300, timeUnit = MILLISECONDS)
+@Warmup(iterations = 20, time = 1000, timeUnit = MILLISECONDS)
+@Measurement(iterations = 10, time = 1000, timeUnit = MILLISECONDS)
 public class TimeBenchmark extends AbstractBenchmark {
 
     @Benchmark

src/main/java/com/github/chrisgleissner/benchmarks/collection/AbstractCollectionBenchmark.java

@@ -1,7 +1,7 @@
 package com.github.chrisgleissner.benchmarks.collection;
 
 import com.github.chrisgleissner.benchmarks.AbstractBenchmark;
-import org.openjdk.jmh.annotations.OperationsPerInvocation;
+import org.openjdk.jmh.annotations.*;
 
 import static com.github.chrisgleissner.benchmarks.Constants.OPERATIONS_PER_PER_INVOCATION;
 

src/main/java/com/github/chrisgleissner/benchmarks/collection/AbstractCollectionBenchmarkState.java

@@ -67,10 +67,10 @@ public abstract class AbstractCollectionBenchmarkState {
 
     @Setup(Level.Iteration)
     public void doSetup() {
-        abq = new ArrayBlockingQueue<>(MAX_OPERATIONS_PER_ITERATION);
+        abq = new ArrayBlockingQueue<>(MAX_OPERATIONS_PER_ITERATION * 2);
         ad = new ArrayDeque<>();
         al = new ArrayList<>();
-        alnr = new ArrayList<>(MAX_OPERATIONS_PER_ITERATION);
+        alnr = new ArrayList<>(MAX_OPERATIONS_PER_ITERATION * 2);
         chm = new ConcurrentHashMap<>();
         cld = new ConcurrentLinkedDeque<>();
         clq = new ConcurrentLinkedQueue<>();
@@ -92,7 +92,7 @@ public void doSetup() {
         tm = new TreeMap<>();
         ts = new TreeSet<>();
         v = new Vector<>();
-        vnr = new Vector<>(MAX_OPERATIONS_PER_ITERATION);
+        vnr = new Vector<>(MAX_OPERATIONS_PER_ITERATION * 2);
     }
 
     @TearDown(Level.Iteration)

src/main/java/com/github/chrisgleissner/benchmarks/collection/ArrayAddBenchmark.java

@@ -1,16 +1,15 @@
 package com.github.chrisgleissner.benchmarks.collection;
 
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.*;
 
 import java.util.Arrays;
 import java.util.stream.IntStream;
 import java.util.stream.LongStream;
+import static java.util.concurrent.TimeUnit.*;
 
 import static com.github.chrisgleissner.benchmarks.Constants.OPERATIONS_PER_PER_INVOCATION;
-
+@Warmup(iterations = 50, time = 1000, timeUnit = MILLISECONDS)
+@Measurement(iterations = 10, time = 1000, timeUnit = MILLISECONDS)
 public class ArrayAddBenchmark extends AbstractCollectionBenchmark {
 
     @Benchmark

src/main/java/com/github/chrisgleissner/benchmarks/collection/CollectionAddBenchmark.java

@@ -1,14 +1,18 @@
 package com.github.chrisgleissner.benchmarks.collection;
 
-import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.*;
 
 import java.util.Collection;
 import java.util.Map;
-
+import static java.util.concurrent.TimeUnit.*;
 import static com.github.chrisgleissner.benchmarks.Constants.MAX_OPERATIONS_PER_ITERATION;
 import static com.github.chrisgleissner.benchmarks.Constants.OPERATIONS_PER_PER_INVOCATION;
 import static java.util.stream.IntStream.range;
 
+@BenchmarkMode(Mode.Throughput)
+@Warmup(iterations = 50, time = 1000, timeUnit = MILLISECONDS)
+@Measurement(iterations = 10, time = 1000, timeUnit = MILLISECONDS)
+@Fork(value = 1, jvmArgsPrepend = { "-Xmx32g"})
 public class CollectionAddBenchmark extends AbstractCollectionBenchmark {
 
     static Integer[] integers = range(0, MAX_OPERATIONS_PER_ITERATION).boxed().toArray(Integer[]::new);
@@ -34,6 +38,7 @@ public Object ArrayListNoResize(MyState s) {
     }
 
     @Benchmark
+    @Measurement(iterations = 50, time = 1000, timeUnit = MILLISECONDS)
     public Object ConcurrentHashMap(MyState s) {
         return benchmarkPut(s, s.chm);
     }
@@ -49,6 +54,7 @@ public Object ConcurrentLinkedQueue(MyState s) {
     }
 
     @Benchmark
+    @Measurement(iterations = 100, time = 1000, timeUnit = MILLISECONDS)
     public Object ConcurrentSkipListMap(MyState s) {
         return benchmarkPut(s, s.cslm);
     }
@@ -69,6 +75,7 @@ public Object CopyOnWriteArraySet(MyState s) {
     }
 
     @Benchmark
+    @Measurement(iterations = 50, time = 1000, timeUnit = MILLISECONDS)
     public Object HashMap(MyState s) {
         return benchmarkPut(s, s.hm);
     }
@@ -94,6 +101,7 @@ public Object LinkedHashSet(MyState s) {
     }
 
     @Benchmark
+    @Measurement(iterations = 50, time = 1000, timeUnit = MILLISECONDS)
     public Object LinkedHashMap(MyState s) {
         return benchmarkPut(s, s.lhm);
     }

src/main/java/com/github/chrisgleissner/benchmarks/collection/CollectionIterateBenchmark.java

@@ -1,8 +1,6 @@
 package com.github.chrisgleissner.benchmarks.collection;
 
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.*;
 import org.openjdk.jmh.infra.Blackhole;
 
 import java.util.Collection;
@@ -13,7 +11,12 @@
 
 import static com.github.chrisgleissner.benchmarks.Constants.OPERATIONS_PER_PER_INVOCATION;
 import static java.util.function.Function.identity;
+import static java.util.concurrent.TimeUnit.*;
 
+@BenchmarkMode(Mode.Throughput)
+@Warmup(iterations = 50, time = 1000, timeUnit = MILLISECONDS)
+@Measurement(iterations = 10, time = 1000, timeUnit = MILLISECONDS)
+@Fork(value = 1, jvmArgsPrepend = { "-Xmx32g"})
 public class CollectionIterateBenchmark extends AbstractCollectionBenchmark {
 
     @Benchmark
@@ -23,8 +26,10 @@ public void ArrayBlockingQueue(MyState s, Blackhole bh) {
 
     private void iterate(Blackhole bh, Collection<Integer> target) {
         Iterator<Integer> iterator = target.iterator();
+        int sum = 0;
         while (iterator.hasNext())
-            bh.consume(iterator.next());
+            sum += iterator.next();
+        bh.consume(sum);
     }
 
     @Benchmark

src/main/java/com/github/chrisgleissner/benchmarks/collection/StreamBenchmark.java

@@ -5,7 +5,7 @@
 import org.openjdk.jmh.annotations.OperationsPerInvocation;
 import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.*;
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.infra.Blackhole;
 
@@ -17,12 +17,20 @@
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.TimeUnit;
 
 import static java.util.Comparator.comparingInt;
 import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.IntStream.range;
-
+import static java.util.concurrent.TimeUnit.*;
+
+@State(Scope.Thread)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 50, time = 1000, timeUnit = MILLISECONDS)
+@Measurement(iterations = 50, time = 1000, timeUnit = MILLISECONDS)
+@Fork(value = 1, jvmArgsPrepend = { "-Xmx32g"})
 @OperationsPerInvocation(1)
 public class StreamBenchmark extends AbstractCollectionBenchmark {
 
@@ -50,7 +58,6 @@ public Object integerStreamGroupByParallel(MyState state) {
     public int integerStreamReduceToSum(MyState state) {
         return state.integers.stream().reduce(0, (i, j) -> i + j);
     }
-
     @Benchmark
     public int integerStreamReduceToSumParallel(MyState state) {
         return state.integers.stream().parallel().reduce(0, (i, j) -> i + j);
@@ -68,59 +75,84 @@ public int integerStreamSumParallel(MyState state) {
 
     @Benchmark
     public void integerStreamForEach(MyState state, Blackhole blackhole) {
-        Collection<Integer> target = new ArrayList<>();
-        state.integers.stream().filter(i -> i < state.integerFilter).forEach(target::add);
+        final Collection<Integer> target = new ArrayList<>();
+        for (Integer threshold : state.range) {
+            state.integers.stream().filter(i -> i < threshold).forEach(target::add);
+        }
         blackhole.consume(target);
     }
 
     @Benchmark
     public void integerStreamForEachParallel(MyState state, Blackhole blackhole) {
-        Set<Integer> target = new ConcurrentSkipListSet<>();
-        state.integers.stream().parallel().filter(i -> i < state.integerFilter).forEach(target::add);
+        final Set<Integer> target = new ConcurrentSkipListSet<>();
+        for (Integer threshold : state.range) {
+            state.integers.stream().parallel().filter(i -> i < threshold).forEach(target::add);
+        }
         blackhole.consume(target);
     }
 
     @Benchmark
     public void integerStreamCollect(MyState state, Blackhole blackhole) {
-        blackhole.consume(state.integers.stream().filter(i -> i < state.integerFilter).collect(toList()));
+        Collection<Integer> target = new ArrayList<>();
+        for (Integer threshold : state.range) {
+            target = state.integers.stream().filter(i -> i < threshold).collect(toList());
+        }
+        blackhole.consume(target);
     }
 
     @Benchmark
     public void integerStreamCollectParallel(MyState state, Blackhole blackhole) {
-        blackhole.consume(state.integers.stream().parallel().filter(i -> i < state.integerFilter).collect(toList()));
+        Collection<Integer> target = new ArrayList<>();
+        for (Integer threshold : state.range) {
+           target = state.integers.stream().parallel().filter(i -> i < threshold).collect(toList());
+        }
+        blackhole.consume(target);
     }
 
     @Benchmark
     public void integerFor(MyState state, Blackhole blackhole) {
         Collection<Integer> source = state.integers;
         Collection<Integer> target = new ArrayList<>();
-        Integer filter = state.integerFilter;
-        for (Integer i : source)
-            if (i < filter)
-                target.add(i);
+        for (Integer threshold : state.range) {
+            Integer filter = threshold;
+            for (Integer i : source)
+                if (i < filter)
+                    target.add(i);
+        }
         blackhole.consume(target);
     }
 
     @Benchmark
     public void intStream(MyState state, Blackhole blackhole) {
-        blackhole.consume(Arrays.stream(state.ints).filter(i -> i < state.intFilter).toArray());
+        int[] target = new int[state.collectionSize];
+        for (Integer threshold : state.range) {
+            target = Arrays.stream(state.ints).filter(i -> i < threshold).toArray();
+        }
+        blackhole.consume(target);
     }
 
     @Benchmark
     public void intStreamParallel(MyState state, Blackhole blackhole) {
-        blackhole.consume(Arrays.stream(state.ints).parallel().filter(i -> i < state.intFilter).toArray());
+        int[] target = new int[state.collectionSize];
+        for (Integer threshold : state.range) {
+            Arrays.stream(state.ints).parallel().filter(i -> i < threshold).toArray();
+        }
+        blackhole.consume(target);
     }
 
     @Benchmark
     public void intFor(MyState state, Blackhole blackhole) {
         int[] source = state.ints;
         int[] target = new int[source.length];
         int targetIndex = 0;
-        int filter = state.intFilter;
-        for (int i = 0; i < source.length; i++) {
-            int candidate = source[i];
-            if (candidate < filter)
-                target[targetIndex++] = candidate;
+        for (Integer threshold : state.range) {
+            targetIndex = 0;
+            int filter = threshold;
+            for (int i = 0; i < source.length; i++) {
+                int candidate = source[i];
+                if (candidate < filter)
+                    target[targetIndex++] = candidate;
+            }
         }
         int[] trimmedTarget = new int[targetIndex];
         System.arraycopy(source, 0, trimmedTarget, 0, targetIndex);
@@ -131,20 +163,36 @@ public void intFor(MyState state, Blackhole blackhole) {
     public static class MyState {
         static Random random = new Random();
         public List<Integer> integers;
+        public List<Integer> range;
         public int[] ints;
-        public Integer integerFilter;
-        public int intFilter;
-        @Param({"1", "10", "100", "1000", "10000", "100000", "1000000", "10000000"})
+
+        @Param({"1", "10", "100", "1000", "10000", "100000","1000000", "10000000"})
         int collectionSize;
 
-        @Setup(Level.Iteration)
+        @Setup(Level.Trial)
         public void doSetup() {
-            integerFilter = random.nextInt(collectionSize);
-            intFilter = integerFilter;
-
+            if (collectionSize <= 100) {
+                range = range(0, collectionSize).boxed().collect(toList());
+            }
+            else if (collectionSize <= 100000) {
+                int step = collectionSize / 100;
+                range = range(0, 100).map(x -> x * step).boxed().collect(toList());
+            }
+            else {
+                int step = collectionSize / 2;
+                range = range(0, 2).map(x -> x * step).boxed().collect(toList());
+            }
             integers = range(0, collectionSize).boxed().collect(toList());
             Collections.shuffle(integers, random);
             ints = integers.stream().mapToInt(Integer::valueOf).toArray();
+
+            // Fix data layout so that all elements are plced continuously in memory (not in random order ad after shuffle)
+            List<Integer> old_int = integers;
+            integers = new ArrayList<Integer>(old_int.size());
+            for (Integer i: old_int) {
+                integers.add(Integer.valueOf(i.intValue()));
+            }
         }
     }
+
 }