Skip to content

Commit 12439ee

Browse files
authored
ci: use ap-loader for perf testing (#58)
Allows for nice things: - No longer need to set up our own async profiler. - We can finally upgrade to latest async profiler. - Which brings many new visualizations options, which we also enable.
1 parent 5c474da commit 12439ee

File tree

9 files changed

+121
-104
lines changed

9 files changed

+121
-104
lines changed

.github/dependabot.yml

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
version: 2
22
updates:
3-
- package-ecosystem: "maven" # See documentation for possible values
4-
directory: "/" # Location of package manifests
3+
- package-ecosystem: maven
4+
directory: "/"
55
schedule:
6-
interval: "weekly"
6+
interval: weekly
7+
time: '05:00' # Otherwise it picks a random time.
8+
open-pull-requests-limit: 10
9+
target-branch: "main"
10+
commit-message:
11+
prefix: "deps: "
12+
- package-ecosystem: "github-actions"
13+
directory: "/"
14+
schedule:
15+
interval: weekly
16+
time: '05:00' # Otherwise it picks a random time.

.github/workflows/performance_score_director.yml

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ on:
2222
required: true
2323
baseline:
2424
description: 'Timefold Solver release'
25-
default: '1.20.1'
25+
default: '1.21.0'
2626
required: true
2727
jdk_branch:
2828
description: 'JDK version'
@@ -36,10 +36,6 @@ on:
3636
description: 'User owning the branch'
3737
default: 'TimefoldAI'
3838
required: true
39-
async_profiler_version:
40-
description: 'async-profiler version'
41-
default: '3.0'
42-
required: true
4339

4440
run-name: "Timefold Solver v${{ github.event.inputs.baseline }} vs. ${{ github.event.inputs.branch_owner }}/${{ github.event.inputs.branch }} (Java ${{ github.event.inputs.jdk_baseline }} vs. ${{ github.event.inputs.jdk_branch }})"
4541

@@ -75,7 +71,7 @@ jobs:
7571
working-directory: ./timefold-solver-benchmarks
7672
shell: bash
7773
run: |
78-
mvn clean install -B -Dquickly -Dversion.ai.timefold.solver=${{ github.event.inputs.baseline }} -Dversion.tools.provider="${{ github.event.inputs.async_profiler_version }}"
74+
mvn clean install -B -Dquickly -Dversion.ai.timefold.solver=${{ github.event.inputs.baseline }}
7975
mv target/benchmarks.jar ../benchmarks-baseline.jar
8076
8177
- name: (SUT) Checkout timefold-solver
@@ -128,7 +124,7 @@ jobs:
128124
working-directory: ./timefold-solver-benchmarks
129125
shell: bash
130126
run: |
131-
mvn clean install -B -Dquickly -Dversion.tools.provider="${{ github.event.inputs.async_profiler_version }}"
127+
mvn clean install -B -Dquickly
132128
mv target/benchmarks.jar ../benchmarks-sut.jar
133129
134130
- name: Upload the binaries
@@ -178,15 +174,6 @@ jobs:
178174
name: binaries-${{ matrix.example }}
179175
path: ./timefold-solver-benchmarks/
180176

181-
- name: Setup Async Profiler
182-
working-directory: ./timefold-solver-benchmarks
183-
run: |
184-
export FILENAME=async-profiler-${{ github.event.inputs.async_profiler_version }}-linux-x64.tar.gz
185-
wget https://github.com/async-profiler/async-profiler/releases/download/v${{ github.event.inputs.async_profiler_version }}/$FILENAME
186-
tar -xzf $FILENAME
187-
mkdir target
188-
ls -l
189-
190177
# Fine-tuned for stability on GHA.
191178
- name: Configure the benchmark
192179
working-directory: ./timefold-solver-benchmarks
@@ -250,11 +237,9 @@ jobs:
250237
name: results-${{ matrix.example }}-${{ env.SANITIZED_BASELINE }}_vs_${{ env.SANITIZED_BRANCH }}
251238
path: |
252239
./timefold-solver-benchmarks/scoredirector-benchmark.properties
253-
./timefold-solver-benchmarks/${{ env.SANITIZED_BASELINE }}/*.log
254240
./timefold-solver-benchmarks/${{ env.SANITIZED_BASELINE }}/*combined.jfr
255241
./timefold-solver-benchmarks/${{ env.SANITIZED_BASELINE }}/*.html
256242
./timefold-solver-benchmarks/${{ env.SANITIZED_BASELINE }}/*.json
257-
./timefold-solver-benchmarks/${{ env.SANITIZED_BRANCH }}/*.log
258243
./timefold-solver-benchmarks/${{ env.SANITIZED_BRANCH }}/*combined.jfr
259244
./timefold-solver-benchmarks/${{ env.SANITIZED_BRANCH }}/*.html
260245
./timefold-solver-benchmarks/${{ env.SANITIZED_BRANCH }}/*.json

README.adoc

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,6 @@ and they will include flame graphs if Async Profiler is available.
2424
In the default configuration, the benchmark will run for many hours and fully occupy 1 CPU.
2525
Only run the benchmark on a quiet machine, otherwise results will be skewed.
2626

27-
==== Async profiler
28-
29-
The benchmark can optionally produce flame graphs using https://github.com/async-profiler/async-profiler[Async Profiler].
30-
On the first run, if Async Profiler is not found, a message will be logged.
31-
Let the message point you in the right direction.
32-
3327
=== Configuring the benchmark
3428

3529
The benchmark is configured using the `coldstart-benchmark.properties` and `scoredirector-benchmark.properties` file.
@@ -58,5 +52,4 @@ mvn clean install
5852
The benchmark results will be published as a CSV file in the `results` directory.
5953

6054
In the default configuration, the benchmark will run for many hours and fully occupy 4 CPUs.
61-
Only run the benchmark on a quiet machine, otherwise results will be skewed.
62-
55+
Only run the benchmark on a quiet machine, otherwise results will be skewed.

pom.xml

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
<properties>
1717
<version.ai.timefold.solver>999-SNAPSHOT</version.ai.timefold.solver>
18+
<version.me.bechberger>4.0-10</version.me.bechberger>
1819
<version.org.jdom>2.0.6.1</version.org.jdom>
1920
<version.org.junit>5.10.2</version.org.junit>
2021
<version.org.mockito>5.11.0</version.org.mockito>
@@ -26,8 +27,6 @@
2627
<jmh.version>1.37</jmh.version>
2728
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
2829
<uberjar.name>benchmarks</uberjar.name>
29-
<!-- Used to set a property in buildtime.properties, that will later be read by the JMH benchmark. -->
30-
<version.tools.profiler>3.0</version.tools.profiler>
3130
</properties>
3231

3332
<dependencyManagement>
@@ -55,9 +54,9 @@
5554
<scope>provided</scope>
5655
</dependency>
5756
<dependency>
58-
<groupId>tools.profiler</groupId>
59-
<artifactId>async-profiler-converter</artifactId>
60-
<version>${version.tools.profiler}</version>
57+
<groupId>me.bechberger</groupId>
58+
<artifactId>ap-loader-all</artifactId>
59+
<version>${version.me.bechberger}</version>
6160
</dependency>
6261
<dependency>
6362
<groupId>ai.timefold.solver.enterprise</groupId>

src/assembly/jar-with-dependencies-and-services.xml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
<filtered>true</filtered>
3737
<includes>
3838
<include>logback.xml</include>
39-
<include>buildtime.properties</include>
4039
</includes>
4140
</fileSet>
4241
</fileSets>

src/main/java/ai/timefold/solver/benchmarks/micro/coldstart/Main.java

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,9 @@
4141

4242
import org.openjdk.jmh.runner.RunnerException;
4343
import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
44-
import org.slf4j.Logger;
45-
import org.slf4j.LoggerFactory;
4644

4745
public final class Main extends AbstractMain<Configuration> {
4846

49-
private static final Logger LOGGER = LoggerFactory.getLogger(Main.class);
50-
5147
public Main() {
5248
super("coldstart");
5349
}
@@ -62,16 +58,19 @@ protected Configuration getDefaultConfiguration() {
6258
return Configuration.getDefault();
6359
}
6460

65-
public static void main(String[] args) throws RunnerException, IOException {
66-
var main = new Main();
67-
var configuration = main.readConfiguration();
68-
var options = main.getBaseJmhConfig(configuration);
61+
public static void main(String[] args) throws RunnerException {
62+
new Main().run(args);
63+
}
64+
65+
public void run(String[] args) throws RunnerException {
66+
var configuration = readConfiguration();
67+
var options = getBaseJmhConfig(configuration);
6968
options = processBenchmark(options, configuration);
70-
options = main.initAsyncProfiler(options);
69+
options = initAsyncProfiler(options);
7170

72-
var runner = new ResultCapturingJMHRunner(main.resultsDirectory, options.build());
71+
var runner = new ResultCapturingJMHRunner(resultsDirectory, options.build());
7372
var runResults = runner.run();
74-
main.convertJfrToFlameGraphs();
73+
visualizeJfr();
7574

7675
var relativeScoreErrorThreshold = configuration.getRelativeScoreErrorThreshold();
7776
var thresholdForPrint = ((int) Math.round(relativeScoreErrorThreshold * 10_000)) / 100.0D;
@@ -94,7 +93,7 @@ public static void main(String[] args) throws RunnerException, IOException {
9493
});
9594
}
9695

97-
private static ChainedOptionsBuilder processBenchmark(ChainedOptionsBuilder options, Configuration configuration) {
96+
private ChainedOptionsBuilder processBenchmark(ChainedOptionsBuilder options, Configuration configuration) {
9897
var supportedExampleNames = getSupportedExampleNames(configuration);
9998
if (supportedExampleNames.length > 0) {
10099
options = options.include(TimeToFirstScoreBenchmark.class.getSimpleName())
@@ -104,7 +103,7 @@ private static ChainedOptionsBuilder processBenchmark(ChainedOptionsBuilder opti
104103
return options;
105104
}
106105

107-
private static String[] getSupportedExampleNames(Configuration configuration) {
106+
private String[] getSupportedExampleNames(Configuration configuration) {
108107
var examples = configuration.getEnabledExamples()
109108
.stream()
110109
.map(Enum::name)

src/main/java/ai/timefold/solver/benchmarks/micro/common/AbstractMain.java

Lines changed: 77 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
import java.util.Arrays;
4141
import java.util.Objects;
4242
import java.util.Optional;
43-
import java.util.Properties;
43+
import java.util.stream.Stream;
4444

4545
import org.openjdk.jmh.profile.AsyncProfiler;
4646
import org.openjdk.jmh.results.format.ResultFormatType;
@@ -49,11 +49,13 @@
4949
import org.slf4j.Logger;
5050
import org.slf4j.LoggerFactory;
5151

52-
public abstract class AbstractMain<C extends AbstractConfiguration> {
52+
import one.convert.Arguments;
53+
import one.profiler.AsyncProfilerLoader;
5354

54-
private static final Logger STATIC_LOGGER = LoggerFactory.getLogger(AbstractMain.class);
55+
public abstract class AbstractMain<C extends AbstractConfiguration> {
5556

5657
protected final Logger LOGGER = LoggerFactory.getLogger(getClass());
58+
5759
private final String subpackage;
5860
protected final Path resultsDirectory;
5961

@@ -81,21 +83,11 @@ protected static String getTimestamp() {
8183
return year + month + day + "_" + hour + minute + second;
8284
}
8385

84-
protected static Optional<Path> getAsyncProfilerPath() {
86+
protected Optional<Path> getAsyncProfilerPath() {
8587
try {
86-
var properties = new Properties();
87-
properties.load(AbstractMain.class.getResourceAsStream("/buildtime.properties"));
88-
var asyncProfilerPath = Path.of(
89-
properties.getProperty("async.profiler.path").trim(),
90-
"lib",
91-
"libasyncProfiler.so")
92-
.toAbsolutePath();
93-
if (!asyncProfilerPath.toFile().exists()) {
94-
return Optional.empty();
95-
}
96-
return Optional.of(asyncProfilerPath);
97-
} catch (IOException e) {
98-
STATIC_LOGGER.error("Failed reading buildtime.properties from the benchmarks JAR.", e);
88+
return Optional.of(AsyncProfilerLoader.getAsyncProfilerPath());
89+
} catch (Exception e) {
90+
LOGGER.error("Failed loading AsyncProfiler.", e);
9991
return Optional.empty();
10092
}
10193
}
@@ -115,24 +107,34 @@ protected ChainedOptionsBuilder initAsyncProfiler(ChainedOptionsBuilder options)
115107
});
116108
}
117109

118-
protected void convertJfrToFlameGraphs() {
110+
protected void visualizeJfr() {
119111
if (getAsyncProfilerPath().isPresent()) {
120112
var combinedJfr = resultsDirectory.resolve("combined.jfr");
121-
var jfrAssembleLog = resultsDirectory.resolve("jfr-assemble.log");
113+
// JFR will create the combined file before it finishes combining all of the inputs.
114+
// Because the command looks at the entire directory, this will make the combined file part of the inputs.
115+
// And this will cause an unkillable JFR process on MacOS.
116+
// Therefore we combine the JFR files to a file outside the directory,
117+
// and only when fully combined we move it over to the results folder.
118+
var tmpCombinedJfr = resultsDirectory.getParent().resolve("combined.jfr");
122119
try {
123120
// Merge all the JFR files that were generated by JMH and copied over by our custom JMH runner.
124121
var process = new ProcessBuilder()
125122
.command("jfr", "assemble", resultsDirectory.toAbsolutePath().toString(),
126-
combinedJfr.toAbsolutePath().toString())
127-
.redirectOutput(jfrAssembleLog.toFile())
123+
tmpCombinedJfr.toAbsolutePath().toString())
124+
.inheritIO()
128125
.start();
129126
if (process.waitFor() != 0) {
130-
LOGGER.error("Failed combining JFR files. See '{}' for details.", jfrAssembleLog);
127+
LOGGER.error("Failed combining JFR files.");
131128
return;
132129
}
133-
// From the combined JMH file, create flame graphs.
134-
generateFlameGraphsFromJfr(combinedJfr, null);
135-
generateFlameGraphsFromJfr(combinedJfr, "alloc");
130+
Files.move(tmpCombinedJfr, combinedJfr);
131+
LOGGER.error("Combined JFR files to {}.", combinedJfr);
132+
// From the combined JMH file, create visualizations.
133+
for (var visualizationType : VisualizationType.values()) {
134+
for (var dataType : DataType.values()) {
135+
visualizeJfr(combinedJfr, visualizationType, dataType);
136+
}
137+
}
136138
} catch (Exception e) {
137139
LOGGER.error("Failed converting JFR to flame graphs.", e);
138140
}
@@ -141,25 +143,31 @@ protected void convertJfrToFlameGraphs() {
141143
}
142144
}
143145

144-
private static void generateFlameGraphsFromJfr(Path jfrFilePath, String type) {
145-
var args = type == null ? new String[] {
146-
"--simple",
147-
jfrFilePath.toString(),
148-
Path.of(jfrFilePath.toAbsolutePath().getParent().toString(), "cpu.html").toString()
149-
}
150-
: new String[] {
151-
"--simple",
152-
"--" + type,
153-
jfrFilePath.toString(),
154-
Path.of(jfrFilePath.toAbsolutePath().getParent().toString(), type + ".html").toString()
155-
};
156-
try { // Converter is stupidly in the default package.
157-
var fooClass = Class.forName("jfr2flame");
158-
var fooMethod = fooClass.getMethod("main", String[].class);
159-
fooMethod.invoke(null, (Object) args);
160-
STATIC_LOGGER.info("Generating flame graph succeeded: {}.", Arrays.toString(args));
146+
private void visualizeJfr(Path jfrFilePath, VisualizationType visualizationType, DataType dataType) {
147+
var inputPath = jfrFilePath.toAbsolutePath();
148+
var filename = String.format("%s-%s.html", dataType.name, visualizationType.name);
149+
var output = Path.of(inputPath.getParent().toString(), filename);
150+
var argStream = Stream.of(
151+
"--simple", // Shorter names.
152+
// "--norm" removes random strings from lambdas;
153+
// allows to merge different frames which are only different
154+
// because they use a different instance of the same lambda.
155+
"--norm",
156+
// "--skip 15" removes bottom frames which come from JMH; they are unnecessary clutter.
157+
"--skip", "15",
158+
"--" + dataType.name);
159+
var args = argStream.toArray(String[]::new);
160+
try {
161+
if (visualizationType == VisualizationType.FLAME_GRAPH) {
162+
one.convert.JfrToFlame.convert(inputPath.toString(), output.toString(), new Arguments(args));
163+
} else if (visualizationType == VisualizationType.HEAT_MAP) {
164+
one.convert.JfrToHeatmap.convert(inputPath.toString(), output.toString(), new Arguments(args));
165+
} else {
166+
throw new IllegalArgumentException("Unsupported visualization: " + visualizationType);
167+
}
168+
LOGGER.info("{} Generation succeeded: {}.", visualizationType, Arrays.toString(args));
161169
} catch (Exception ex) {
162-
STATIC_LOGGER.error("Generating flame graph failed: {}.", Arrays.toString(args), ex);
170+
LOGGER.error("{} Generation failed: {}.", visualizationType, Arrays.toString(args), ex);
163171
}
164172
}
165173

@@ -193,4 +201,30 @@ public ChainedOptionsBuilder getBaseJmhConfig(C configuration) {
193201
.shouldDoGC(true);
194202
}
195203

204+
private enum VisualizationType {
205+
206+
FLAME_GRAPH("flamegraph"),
207+
HEAT_MAP("heatmap");
208+
209+
private final String name;
210+
211+
VisualizationType(String name) {
212+
this.name = Objects.requireNonNull(name);
213+
}
214+
215+
}
216+
217+
private enum DataType {
218+
219+
CPU("cpu"),
220+
MEM("alloc");
221+
222+
private final String name;
223+
224+
DataType(String name) {
225+
this.name = Objects.requireNonNull(name);
226+
}
227+
228+
}
229+
196230
}

0 commit comments

Comments
 (0)