diff --git a/build.gradle b/build.gradle index ff3a88204..9ff17b5c8 100644 --- a/build.gradle +++ b/build.gradle @@ -34,6 +34,7 @@ buildscript { classpath 'com.palantir.javaformat:gradle-palantir-java-format:2.83.0' classpath 'com.palantir.suppressible-error-prone:gradle-suppressible-error-prone:2.26.0' classpath 'org.revapi:gradle-revapi:1.8.0' + classpath 'me.champeau.jmh:jmh-gradle-plugin:0.7.3' } } @@ -48,7 +49,7 @@ apply plugin: 'com.palantir.jdks.latest' javaVersions { libraryTarget = 17 - runtime = 25 + runtime = 21 } allprojects { diff --git a/conjure-jmh/build.gradle b/conjure-jmh/build.gradle new file mode 100644 index 000000000..a956c8a11 --- /dev/null +++ b/conjure-jmh/build.gradle @@ -0,0 +1,21 @@ +apply plugin: 'me.champeau.jmh' +apply plugin: 'org.revapi.revapi-gradle-plugin' + +jmh { + // Use profilers to collect additional data. Supported profilers: + // [cl, comp, gc, stack, perf, perfnorm, perfasm, xperf, xperfasm, hs_cl, hs_comp, hs_gc, hs_rt, hs_thr] + profilers = ['gc'] +} + +dependencies { + jmh 'org.openjdk.jmh:jmh-core' + + jmh 'com.fasterxml.jackson.core:jackson-annotations' + jmh 'com.fasterxml.jackson.core:jackson-databind' + jmh 'com.google.code.findbugs:jsr305' + jmh 'com.google.guava:guava' + jmh 'com.palantir.conjure.java.runtime:conjure-java-jackson-serialization' + jmh 'com.palantir.safe-logging:preconditions' + + jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess' +} diff --git a/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java b/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java new file mode 100644 index 000000000..547501c39 --- /dev/null +++ b/conjure-jmh/src/jmh/java/com/palantir/conjure/ConjureBenchmarks.java @@ -0,0 +1,297 @@ +/* + * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.conjure; + +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.annotation.Nulls; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.google.common.collect.ImmutableMap; +import com.google.errorprone.annotations.CheckReturnValue; +import com.palantir.conjure.java.serialization.ObjectMappers; +import com.palantir.logsafe.Preconditions; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; +import org.jspecify.annotations.NullMarked; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +@SuppressWarnings({"NullAway", "designforextension", "checkstyle:RegexpSinglelineJava", "checkstyle:VisibilityModifier" +}) +public class ConjureBenchmarks { + + private static final ObjectMapper mapper = ObjectMappers.newClientJsonMapper(); + + @Setup + public void before() { + // Ensure we have enough retained capacity to avoid measuring resizing costs. + // Each operation takes at least ~100-200ns, so 10 million ensures we have enough for all benchmark iterations. + MemoryProfiler.ensureRetainedCapacity(10_000_000); + } + + @SuppressWarnings("ImmutableEnumChecker") + public enum RawJson { + NO_MAP("{}"), + EMPTY(generateMap(0)), + SINGLETON(generateMap(1)), + THREE(generateMap(3)), + SIX(generateMap(6)), + TEN(generateMap(10)), + HUNDRED(generateMap(100)), + THOUSAND(generateMap(1000)), + ; + + private static String generateMap(int count) { + StringBuilder sb = new StringBuilder(); + sb.append("{\"map\":{"); + for (int i = 1; i <= count; i++) { + sb.append("\"key").append(i).append("\":\"value").append(i).append("\""); + if (i < count) { + sb.append(","); + } + } + sb.append("}}"); + return sb.toString(); + } + + private final byte[] json; + + RawJson(String jsonString) { + this.json = jsonString.getBytes(StandardCharsets.UTF_8); + } + } + + public enum MapImplementation { + NORMAL(NormalMap.class), + SINGLETON(SingletonMap.class), + GUAVA_IMMUTABLE(GuavaImMap.class); + + private final Class clazz; + + MapImplementation(Class clazz) { + this.clazz = clazz; + } + } + + @Param + public RawJson json; + + @Param + public MapImplementation mapImpl; + + @Benchmark + public void testAllocatingBenchmark() throws IOException { + MemoryProfiler.addRetained(mapper.readValue(json.json, mapImpl.clazz)); + } + + @JsonDeserialize(builder = NormalMap.Builder.class) + public static final class NormalMap { + private final Map map; + + private NormalMap(Map map) { + this.map = Collections.unmodifiableMap(map); + } + + public Map getMap() { + return map; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private boolean buildInvoked; + + private Map map = new LinkedHashMap<>(); + + private Builder() {} + + @JsonSetter(value = "map", nulls = Nulls.SKIP, contentNulls = Nulls.FAIL) + public Builder map(Map newMap) { + checkNotBuilt(); + this.map = new LinkedHashMap<>(Preconditions.checkNotNull(newMap, "map cannot be null")); + return this; + } + + @CheckReturnValue + public NormalMap build() { + checkNotBuilt(); + this.buildInvoked = true; + return new NormalMap(map); + } + + private void checkNotBuilt() { + Preconditions.checkState(!buildInvoked, "Build has already been called"); + } + } + } + + private static Map specialUnmodifiableMap(Map in) { + if (in.isEmpty()) { + return Map.of(); + } + if (in.size() == 1) { + Iterator> itr = in.entrySet().iterator(); + if (itr.hasNext()) { + Entry entry = itr.next(); + if (!itr.hasNext()) { + return Map.of(entry.getKey(), entry.getValue()); + } + } + } + return Collections.unmodifiableMap(in); + } + + @JsonDeserialize(builder = SingletonMap.Builder.class) + public static final class SingletonMap { + private final Map map; + + private SingletonMap(Map map) { + this.map = specialUnmodifiableMap(map); + } + + public Map getMap() { + return map; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private boolean buildInvoked; + + private Map map = new LinkedHashMap<>(); + + private Builder() {} + + @JsonSetter(value = "map", nulls = Nulls.SKIP, contentNulls = Nulls.FAIL) + public Builder map(Map newMap) { + checkNotBuilt(); + this.map = new LinkedHashMap<>(Preconditions.checkNotNull(newMap, "map cannot be null")); + return this; + } + + @CheckReturnValue + public SingletonMap build() { + checkNotBuilt(); + this.buildInvoked = true; + return new SingletonMap(map); + } + + private void checkNotBuilt() { + Preconditions.checkState(!buildInvoked, "Build has already been called"); + } + } + } + + @JsonDeserialize(builder = GuavaImMap.Builder.class) + @NullMarked + public static final class GuavaImMap { + private final Map map; + + private GuavaImMap(ImmutableMap map) { + this.map = map; + } + + public Map getMap() { + return map; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private boolean buildInvoked; + + @Nullable + private Object map; + + private Builder() {} + + @JsonSetter(value = "map", nulls = Nulls.SKIP, contentNulls = Nulls.FAIL) + @JsonDeserialize(as = ImmutableMap.class) + public Builder map(Map newMap) { + checkNotBuilt(); + this.map = newMap instanceof ImmutableMap im + ? im + : ImmutableMap.builder().putAll(newMap); + return this; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + @CheckReturnValue + public GuavaImMap build() { + checkNotBuilt(); + this.buildInvoked = true; + ImmutableMap finalMap; + if (map == null) { + finalMap = ImmutableMap.of(); + } else if (map instanceof ImmutableMap im) { + finalMap = im; + } else if (map instanceof ImmutableMap.Builder builder) { + finalMap = builder.buildOrThrow(); + } else { + throw new IllegalStateException("Unexpected map type: " + map.getClass()); + } + return new GuavaImMap(finalMap); + } + + private void checkNotBuilt() { + Preconditions.checkState(!buildInvoked, "Build has already been called"); + } + } + } + + public static void main(String[] _args) throws Exception { + Options opt = new OptionsBuilder() + .include(ConjureBenchmarks.class.getSimpleName()) + .addProfiler(GCProfiler.class) + .addProfiler(MemoryProfiler.class) + .build(); + + new Runner(opt).run(); + } +} diff --git a/conjure-jmh/src/jmh/java/com/palantir/conjure/MemoryProfiler.java b/conjure-jmh/src/jmh/java/com/palantir/conjure/MemoryProfiler.java new file mode 100644 index 000000000..87cba8fa6 --- /dev/null +++ b/conjure-jmh/src/jmh/java/com/palantir/conjure/MemoryProfiler.java @@ -0,0 +1,174 @@ +/* + * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.conjure; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.IterationParams; +import org.openjdk.jmh.profile.InternalProfiler; +import org.openjdk.jmh.results.AggregationPolicy; +import org.openjdk.jmh.results.IterationResult; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.ScalarResult; + +public final class MemoryProfiler implements InternalProfiler { + private static final List retained = new ArrayList<>(); + private static final MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); + + private long beforeUsedMemory = 0L; + + /** + * Used to pre-allocate retained objects positions in the list to avoid resizing overhead during the benchmark. + */ + public static void ensureRetainedCapacity(int capacity) { + ((ArrayList) retained).ensureCapacity(capacity); + } + + public static void addRetained(Object obj) { + retained.add(obj); + } + + private static void clearRetained() { + retained.clear(); + } + + @Override + public void beforeIteration(BenchmarkParams _benchmarkParams, IterationParams _iterationParams) { + clearRetained(); + runSystemGC(); + beforeUsedMemory = getUsedMemory(); + } + + @Override + public Collection afterIteration( + BenchmarkParams _benchmarkParams, IterationParams _iterationParams, IterationResult result) { + runSystemGC(); + long afterUsedMemory = getUsedMemory(); + long retainedMemory = afterUsedMemory - beforeUsedMemory; + + List results = new ArrayList<>(); + results.add( + new ScalarResult("mem.retained.total", retainedMemory / 1024.0 / 1024.0, "MB", AggregationPolicy.AVG)); + results.add(new ScalarResult( + "mem.retained.total.norm", + (1.0 * retainedMemory) / result.getMetadata().getAllOps(), + "B/op", + AggregationPolicy.AVG)); + results.add(new ScalarResult("mem.retained.count", retained.size(), "obj", AggregationPolicy.AVG)); + return results; + } + + @Override + public String getDescription() { + return "Measures retained memory after each iteration"; + } + + private static final int MAX_WAIT_MSEC = 20 * 1000; + + @SuppressWarnings("checkstyle:CyclomaticComplexity") + // Same as BaseRunner#runSystemGC + private boolean runSystemGC() { + List enabledBeans = new ArrayList<>(); + + long beforeGcCount = 0; + for (GarbageCollectorMXBean bean : ManagementFactory.getGarbageCollectorMXBeans()) { + long count = bean.getCollectionCount(); + if (count != -1) { + enabledBeans.add(bean); + } + } + + for (GarbageCollectorMXBean bean : enabledBeans) { + beforeGcCount += bean.getCollectionCount(); + } + + // Run the GC twice, and force finalization before each GCs. + System.runFinalization(); + System.gc(); + System.runFinalization(); + System.gc(); + + // Now make sure GC actually happened. We have to wait for two things: + // a) That at least two collections happened, indicating GC work. + // b) That counter updates have not happened for a while, indicating GC work had ceased. + // + // Note there is an opportunity window for a concurrent GC to happen before the first + // System.gc() call, which would get counted towards our GCs. This race is unresolvable + // unless we have GC-specific information about the collection cycles, and verify those + // were indeed GCs triggered by us. + + if (enabledBeans.isEmpty()) { + System.out.println("WARNING: MXBeans can not report GC info. System.gc() invoked, pessimistically waiting " + + MAX_WAIT_MSEC + " msecs"); + try { + TimeUnit.MILLISECONDS.sleep(MAX_WAIT_MSEC); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return true; + } + + boolean gcHappened = false; + + long start = System.nanoTime(); + while (TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start) < MAX_WAIT_MSEC) { + try { + TimeUnit.MILLISECONDS.sleep(200); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + long afterGcCount = 0; + for (GarbageCollectorMXBean bean : enabledBeans) { + afterGcCount += bean.getCollectionCount(); + } + + if (!gcHappened) { + if (afterGcCount - beforeGcCount >= 2) { + gcHappened = true; + } + } else { + if (afterGcCount == beforeGcCount) { + // Stable! + return true; + } + beforeGcCount = afterGcCount; + } + } + + if (gcHappened) { + System.out.println("WARNING: System.gc() was invoked but unable to wait while GC stopped, is GC too" + + " asynchronous?"); + } else { + System.out.println("WARNING: System.gc() was invoked but couldn't detect a GC occurring, is System.gc()" + + " disabled?"); + } + return false; + } + + private long getUsedMemory() { + MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage(); + return heapUsage.getUsed(); + } +} diff --git a/settings.gradle b/settings.gradle index 298ea859b..677e67fb1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,6 +17,7 @@ include 'conjure-java-client-verifier:verification-server-api' include 'conjure-java-server-verifier' include 'conjure-java-server-verifier:verification-client-api' include 'conjure-java-undertow-runtime' +include 'conjure-jmh' include 'conjure-lib' include 'conjure-undertow-annotations' include 'conjure-undertow-lib' diff --git a/versions.lock b/versions.lock index 7cd4a27d2..68191f8ce 100644 --- a/versions.lock +++ b/versions.lock @@ -314,8 +314,12 @@ net.bytebuddy:byte-buddy:1.17.7 (2 constraints: 9f161b3f) net.bytebuddy:byte-buddy-agent:1.17.7 (1 constraints: 4a0b4ede) +net.sf.jopt-simple:jopt-simple:5.0.4 (1 constraints: be0ad6cc) + net.sourceforge.argparse4j:argparse4j:0.9.0 (2 constraints: da1b0890) +org.apache.commons:commons-math3:3.6.1 (1 constraints: bf0adbcc) + org.apache.commons:commons-text:1.13.0 (1 constraints: 3e1153ce) org.apiguardian:apiguardian-api:1.1.2 (6 constraints: 5366ce6e) @@ -408,9 +412,17 @@ org.mockito:mockito-junit-jupiter:5.20.0 (1 constraints: 3905453b) org.objenesis:objenesis:3.3 (1 constraints: b20a14bd) +org.openjdk.jmh:jmh-core:1.37 (4 constraints: 2d349791) + +org.openjdk.jmh:jmh-generator-asm:1.37 (1 constraints: 2c107598) + +org.openjdk.jmh:jmh-generator-bytecode:1.37 (1 constraints: de04fb30) + +org.openjdk.jmh:jmh-generator-reflection:1.37 (2 constraints: 491e3064) + org.opentest4j:opentest4j:1.3.0 (2 constraints: cf209249) -org.ow2.asm:asm:9.8 (1 constraints: 0b0aaea4) +org.ow2.asm:asm:9.8 (2 constraints: f617ab6a) org.slf4j:jcl-over-slf4j:2.0.17 (1 constraints: b20e835e) diff --git a/versions.props b/versions.props index 14faf4ea2..8edbc55c1 100644 --- a/versions.props +++ b/versions.props @@ -40,6 +40,7 @@ org.slf4j:* = 2.0.17 org.wildfly.common:wildfly-common = 2.0.1 jakarta.ws.rs:jakarta.ws.rs-api = 4.0.0 jakarta.validation:jakarta.validation-api = 3.0.0 +org.openjdk.jmh:* = 1.37 # dependency-upgrader:OFF # Generator should be compatible with java 8