Microbenchmarking using JBang and JMH

Loops.java

//DEPS org.openjdk.jmh:jmh-generator-annprocess:1.36
// --javaagent=ap-loader@maxandersen=start,event=cpu,file=profile.html
package com.grokhard.benchmarking;

import org.openjdk.jmh.*;
import org.openjdk.jmh.annotations.*;
import org.openjdk.jmh.infra.Blackhole;

import java.util.stream.*;
import java.util.*;
import java.util.concurrent.TimeUnit;

/**
* This is just an example of using Java Microbenchmark Harness (JHM) used in conjunction with JBang
* to demonstrate how simple it is to do that without heavy tool like Maven or Gradle.
* 
* RUN:
*    jbang Loops.java
*
* This chooses a scenario: sum 10,000 numbers from a list. There are 4 bechmarks of how to use the loops.
*
* The use of Blackhole is important to avoid Dead Code Elimination of Java Compiler.
*/


@State(Scope.Benchmark)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
public class Loops {

  public static void main(String[] args) throws java.io.IOException {
    org.openjdk.jmh.Main.main(args);
  }

  private List<Integer> numbers;

  @Setup(Level.Iteration)
  public void setup() {
    numbers = IntStream.range(0, 10_000).boxed().toList();
  }

  @Fork(value = 1)
  @Warmup(iterations = 5)
  @Benchmark
  @BenchmarkMode(Mode.AverageTime)
  public void sumUsingForEnhanceLoop(Blackhole bh) {
    int sum = 0;
    for (Integer number : numbers) {
      sum += number;
    }

    bh.consume(sum);
  }

  @Fork(value = 1)
  @Warmup(iterations = 5)
  @Benchmark
  @BenchmarkMode(Mode.AverageTime)
  public void sumUsingForLoopWithMethodCall(Blackhole bh) {
    int sum = 0;
    for (int i = 0; i < numbers.size(); i++) {
      sum += numbers.get(i);
    }
    bh.consume(sum);
  }

  @Fork(value = 1)
  @Warmup(iterations = 5)
  @Benchmark
  @BenchmarkMode(Mode.AverageTime)
  public void sumUsingForLoopWithSizeGotOutOfLoop(Blackhole bh) {
    int sum = 0;
    int size = numbers.size();
    for (int i = 0; i < size; i++) {
      sum += numbers.get(i);
    }
    bh.consume(sum);
  }

  @Fork(value = 1)
  @Warmup(iterations = 5)
  @Benchmark
  @BenchmarkMode(Mode.AverageTime)
  public void sumUsingReverseForLoopWithSizeGotOutOfLoop(Blackhole bh) {
    int sum = 0;
    int size = numbers.size();
    for (int i = size - 1; i >= 0; i--) {
      sum += numbers.get(i);
    }
    bh.consume(sum);
  }

}

Sample of result

# JMH version: 1.36
# VM version: JDK 22, OpenJDK 64-Bit Server VM, 22+36
# VM invoker: /Users/Genzer/.asdf/installs/java/temurin-22.0.0+36/bin/java
# VM options: <none>
# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable)
# Warmup: 5 iterations, 10 s each
# Measurement: 5 iterations, 10 s each
# Timeout: 10 min per iteration
# Threads: 1 thread, will synchronize iterations
# Benchmark mode: Average time, time/op
# Benchmark: com.grokhard.benchmarking.Loops.sumUsingForEnhanceLoop

# Run progress: 0.00% complete, ETA 00:06:40
# Fork: 1 of 1
# Warmup Iteration   1: 8987.796 ns/op
# Warmup Iteration   2: 9022.646 ns/op
# Warmup Iteration   3: 8942.568 ns/op
# Warmup Iteration   4: 8699.753 ns/op
# Warmup Iteration   5: 8865.366 ns/op
Iteration   1: 8662.103 ns/op
Iteration   2: 8726.804 ns/op
Iteration   3: 9205.181 ns/op
Iteration   4: 10019.170 ns/op
Iteration   5: 9471.325 ns/op


Result "com.grokhard.benchmarking.Loops.sumUsingForEnhanceLoop":
  9216.917 ±(99.9%) 2158.165 ns/op [Average]
  (min, avg, max) = (8662.103, 9216.917, 10019.170), stdev = 560.469
  CI (99.9%): [7058.752, 11375.081] (assumes normal distribution)


# JMH version: 1.36
# VM version: JDK 22, OpenJDK 64-Bit Server VM, 22+36
# VM invoker: /Users/Genzer/.asdf/installs/java/temurin-22.0.0+36/bin/java
# VM options: <none>
# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable)
# Warmup: 5 iterations, 10 s each
# Measurement: 5 iterations, 10 s each
# Timeout: 10 min per iteration
# Threads: 1 thread, will synchronize iterations
# Benchmark mode: Average time, time/op
# Benchmark: com.grokhard.benchmarking.Loops.sumUsingForLoopWithMethodCall

# Run progress: 25.00% complete, ETA 00:05:01
# Fork: 1 of 1
# Warmup Iteration   1: 9474.716 ns/op
# Warmup Iteration   2: 10147.232 ns/op
# Warmup Iteration   3: 9176.775 ns/op
# Warmup Iteration   4: 9342.937 ns/op
# Warmup Iteration   5: 9513.454 ns/op
Iteration   1: 9855.065 ns/op
Iteration   2: 9252.742 ns/op
Iteration   3: 9932.074 ns/op
Iteration   4: 9605.381 ns/op
Iteration   5: 10038.613 ns/op


Result "com.grokhard.benchmarking.Loops.sumUsingForLoopWithMethodCall":
  9736.775 ±(99.9%) 1209.730 ns/op [Average]
  (min, avg, max) = (9252.742, 9736.775, 10038.613), stdev = 314.163
  CI (99.9%): [8527.045, 10946.505] (assumes normal distribution)


# JMH version: 1.36
# VM version: JDK 22, OpenJDK 64-Bit Server VM, 22+36
# VM invoker: /Users/Genzer/.asdf/installs/java/temurin-22.0.0+36/bin/java
# VM options: <none>
# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable)
# Warmup: 5 iterations, 10 s each
# Measurement: 5 iterations, 10 s each
# Timeout: 10 min per iteration
# Threads: 1 thread, will synchronize iterations
# Benchmark mode: Average time, time/op
# Benchmark: com.grokhard.benchmarking.Loops.sumUsingForLoopWithSizeGotOutOfLoop

# Run progress: 50.00% complete, ETA 00:03:21
# Fork: 1 of 1
# Warmup Iteration   1: 10003.358 ns/op
# Warmup Iteration   2: 9156.474 ns/op
# Warmup Iteration   3: 8978.402 ns/op
# Warmup Iteration   4: 9137.532 ns/op
# Warmup Iteration   5: 9919.176 ns/op
Iteration   1: 9086.488 ns/op
Iteration   2: 8941.858 ns/op
Iteration   3: 9781.944 ns/op
Iteration   4: 9306.358 ns/op
Iteration   5: 9280.060 ns/op


Result "com.grokhard.benchmarking.Loops.sumUsingForLoopWithSizeGotOutOfLoop":
  9279.342 ±(99.9%) 1224.400 ns/op [Average]
  (min, avg, max) = (8941.858, 9279.342, 9781.944), stdev = 317.973
  CI (99.9%): [8054.941, 10503.742] (assumes normal distribution)


# JMH version: 1.36
# VM version: JDK 22, OpenJDK 64-Bit Server VM, 22+36
# VM invoker: /Users/Genzer/.asdf/installs/java/temurin-22.0.0+36/bin/java
# VM options: <none>
# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable)
# Warmup: 5 iterations, 10 s each
# Measurement: 5 iterations, 10 s each
# Timeout: 10 min per iteration
# Threads: 1 thread, will synchronize iterations
# Benchmark mode: Average time, time/op
# Benchmark: com.grokhard.benchmarking.Loops.sumUsingReverseForLoopWithSizeGotOutOfLoop

# Run progress: 75.00% complete, ETA 00:01:40
# Fork: 1 of 1
# Warmup Iteration   1: 9664.524 ns/op
# Warmup Iteration   2: 9379.475 ns/op
# Warmup Iteration   3: 9268.555 ns/op
# Warmup Iteration   4: 9277.025 ns/op
# Warmup Iteration   5: 9631.949 ns/op
Iteration   1: 9296.855 ns/op
Iteration   2: 9561.799 ns/op
Iteration   3: 10148.643 ns/op
Iteration   4: 9103.784 ns/op
Iteration   5: 9672.092 ns/op


Result "com.grokhard.benchmarking.Loops.sumUsingReverseForLoopWithSizeGotOutOfLoop":
  9556.635 ±(99.9%) 1535.908 ns/op [Average]
  (min, avg, max) = (9103.784, 9556.635, 10148.643), stdev = 398.871
  CI (99.9%): [8020.726, 11092.543] (assumes normal distribution)


# Run complete. Total time: 00:06:42

REMEMBER: The numbers below are just data. To gain reusable insights, you need to follow up on
why the numbers are the way they are. Use profilers (see -prof, -lprof), design factorial
experiments, perform baseline and negative tests that provide experimental control, make sure
the benchmarking environment is safe on JVM/OS/HW level, ask for reviews from the domain experts.
Do not assume the numbers tell you what you want them to tell.

NOTE: Current JVM experimentally supports Compiler Blackholes, and they are in use. Please exercise
extra caution when trusting the results, look into the generated code to check the benchmark still
works, and factor in a small probability of new VM bugs. Additionally, while comparisons between
different JVMs are already problematic, the performance difference caused by different Blackhole
modes can be very significant. Please make sure you use the consistent Blackhole mode for comparisons.

Benchmark                                         Mode  Cnt     Score      Error  Units
Loops.sumUsingForEnhanceLoop                      avgt    5  9216.917 ± 2158.165  ns/op
Loops.sumUsingForLoopWithMethodCall               avgt    5  9736.775 ± 1209.730  ns/op
Loops.sumUsingForLoopWithSizeGotOutOfLoop         avgt    5  9279.342 ± 1224.400  ns/op
Loops.sumUsingReverseForLoopWithSizeGotOutOfLoop  avgt    5  9556.635 ± 1535.908  ns/op