feat: Setup JMH benchmarking

.github/workflows/pullrequest.yml

@@ -16,7 +16,7 @@ jobs:
                 - java: 11
                   profile: java11
     name: with Java ${{ matrix.build.java }}
-    runs-on: ${{ matrix.os}}
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Check out the code
         uses: actions/checkout@09d2acae674a48949e3602304ab46fd20ae0c42f
@@ -54,5 +54,9 @@ jobs:
           fail_ci_if_error: true # optional (default = false)
           verbose: true # optional (default = false)
 
+      - if: matrix.build.java == '17'
+        name: Run JMH benchmark
+        run: mvn --activate-profiles benchmark jmh:benchmark "-Djmh.rf=json" "-Djmh.rff=benchmark/results/results.json"
+
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@624d0bca90f761ffa7ce50c41875a1a226969a02

pom.xml

@@ -167,7 +167,12 @@
             <version>1.37</version>
             <scope>test</scope>
         </dependency>
-
+        <!--<dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-generator-annprocess</artifactId>
+            <version>1.37</version>
+            <scope>test</scope>
+        </dependency>-->
     </dependencies>
 
     <dependencyManagement>
@@ -298,7 +303,6 @@
                     </archive>
                 </configuration>
             </plugin>
-
         </plugins>
     </build>
 
@@ -703,6 +707,24 @@
                             </execution>
                         </executions>
                     </plugin>
+                    <!--<plugin>
+                        <artifactId>maven-compiler-plugin</artifactId>
+                        <version>3.8.1</version>
+                        <configuration>
+                            <annotationProcessorPaths>
+                                <path>
+                                    <groupId>org.openjdk.jmh</groupId>
+                                    <artifactId>jmh-generator-annprocess</artifactId>
+                                    <version>1.37</version>
+                                </path>
+                                <path>
+                                    <groupId>org.projectlombok</groupId>
+                                    <artifactId>lombok</artifactId>
+                                    <version>1.18.38</version>
+                                </path>
+                            </annotationProcessorPaths>
+                        </configuration>
+                    </plugin>-->
                 </plugins>
             </build>
         </profile>

src/test/java/dev/openfeature/sdk/benchmark/FlagEvaluationBenchmark.java

@@ -0,0 +1,32 @@
+package dev.openfeature.sdk.benchmark;
+
+import dev.openfeature.sdk.benchmark.state.FlagEvaluationState;
+import dev.openfeature.sdk.benchmark.state.HooksState;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.Throughput)
+@Warmup(time = 1, timeUnit = TimeUnit.SECONDS, iterations = 1)
+// @Warmup(time = 1, timeUnit = TimeUnit.SECONDS, iterations = 2)
+@Measurement(time = 5, timeUnit = TimeUnit.SECONDS, iterations = 1)
+// @Measurement(time = 5, timeUnit = TimeUnit.SECONDS, iterations = 4)
+@Fork(1)
+public class FlagEvaluationBenchmark {
+
+    @Benchmark
+    public String flagEvaluations(FlagEvaluationState state) {
+        return state.client
+                .getStringDetails(FlagEvaluationState.FLAG_KEY, "default")
+                .getValue();
+    }
+
+    @Benchmark
+    public String hookExecution(HooksState state) {
+        return state.client.getStringDetails(HooksState.FLAG_KEY, "default").getValue();
+    }
+}

src/test/java/dev/openfeature/sdk/benchmark/state/FlagEvaluationState.java

@@ -0,0 +1,40 @@
+package dev.openfeature.sdk.benchmark.state;
+
+import dev.openfeature.sdk.Client;
+import dev.openfeature.sdk.ImmutableMetadata;
+import dev.openfeature.sdk.OpenFeatureAPI;
+import dev.openfeature.sdk.providers.memory.Flag;
+import dev.openfeature.sdk.providers.memory.InMemoryProvider;
+import java.util.Map;
+import org.openjdk.jmh.annotations.*;
+
+@State(Scope.Benchmark)
+public class FlagEvaluationState {
+    public static final String FLAG_KEY = "flag-key";
+    public static final String DOMAIN = "jmh-domain";
+
+    public InMemoryProvider provider;
+    public Client client;
+
+    @Setup(Level.Trial)
+    public void setup() {
+        provider = new InMemoryProvider(Map.of(
+                FLAG_KEY,
+                Flag.builder()
+                        .variant("a", "a-value")
+                        .variant("b", "b-value")
+                        .defaultVariant("b")
+                        .flagMetadata(ImmutableMetadata.builder()
+                                .addString("meta", "data")
+                                .build())
+                        .build()));
+        OpenFeatureAPI.getInstance().setProviderAndWait(DOMAIN, provider);
+        client = OpenFeatureAPI.getInstance().getClient(DOMAIN);
+    }
+
+    @TearDown(Level.Trial)
+    public void teardown() {
+        OpenFeatureAPI.getInstance().shutdown();
+        OpenFeatureAPI.getInstance().clearHooks();
+    }
+}

src/test/java/dev/openfeature/sdk/benchmark/state/HooksState.java

@@ -0,0 +1,47 @@
+package dev.openfeature.sdk.benchmark.state;
+
+import dev.openfeature.sdk.Client;
+import dev.openfeature.sdk.Hook;
+import dev.openfeature.sdk.ImmutableMetadata;
+import dev.openfeature.sdk.OpenFeatureAPI;
+import dev.openfeature.sdk.StringHook;
+import dev.openfeature.sdk.providers.memory.Flag;
+import dev.openfeature.sdk.providers.memory.InMemoryProvider;
+import java.util.Map;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+
+@State(Scope.Benchmark)
+public class HooksState {
+    public static final String FLAG_KEY = "flag-key";
+    public static final String DOMAIN = "jmh-domain";
+
+    public InMemoryProvider provider;
+    public Client client;
+
+    @Setup(Level.Trial)
+    public void setup() {
+        provider = new InMemoryProvider(Map.of(
+                FLAG_KEY,
+                Flag.builder()
+                        .variant("a", "a-value")
+                        .variant("b", "b-value")
+                        .defaultVariant("b")
+                        .flagMetadata(ImmutableMetadata.builder()
+                                .addString("meta", "data")
+                                .build())
+                        .build()));
+        OpenFeatureAPI.getInstance().setProviderAndWait(DOMAIN, provider);
+        client = OpenFeatureAPI.getInstance().getClient(DOMAIN);
+        client.addHooks(new StringHook() {}, new Hook() {});
+    }
+
+    @TearDown(Level.Trial)
+    public void teardown() {
+        OpenFeatureAPI.getInstance().shutdown();
+        OpenFeatureAPI.getInstance().clearHooks();
+    }
+}