Improve test runner performance by restoring from CRaC checkpoint

.dockerignore

@@ -1,15 +1,12 @@
 .appends
 .git
 .github
-.gradle
 .idea
-gradle
 tests
 .dockerignore
 .gitattributes
 .gitignore
 bin/run-in-docker.sh
 bin/run-tests.sh
 bin/run-tests-in-docker.sh
-gradlew
 gradlew.bat

.github/workflows/ci.yml

@@ -15,6 +15,13 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 21
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
 
       - name: Run Tests in Docker
         run: bin/run-tests-in-docker.sh

.github/workflows/deploy.yml

@@ -9,14 +9,44 @@ on:
 permissions:
   contents: write
 
+env:
+  IMAGE_TAG: exercism/java-test-runner-crac-checkpoint
+
 jobs:
   build-and-push-image:
     if: github.repository_owner == 'exercism' # Stops this job from running on forks.
-    uses: exercism/github-actions/.github/workflows/docker-build-push-image.yml@main
-    secrets:
-      AWS_ACCOUNT_ID: ${{secrets.AWS_ACCOUNT_ID}}
-      AWS_REGION: ${{secrets.AWS_REGION}}
-      AWS_ECR_ACCESS_KEY_ID: ${{secrets.AWS_ECR_ACCESS_KEY_ID}}
-      AWS_ECR_SECRET_ACCESS_KEY: ${{secrets.AWS_ECR_SECRET_ACCESS_KEY}}
-      DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}}
-      DOCKERHUB_PASSWORD: ${{secrets.DOCKERHUB_PASSWORD}}
+    steps:
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 21
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+      - name: Build with Gradle
+        run: ./gradlew build
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and export to Docker
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4
+        with:
+          context: .
+          file: ./Dockerfile.createCheckpoint
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          #         build-args: ${{ secrets.DOCKER_BUILD_ARGS }}
+          provenance: false
+          platforms: linux/amd64
+          tags: ${{ env.IMAGE_TAG }}
+      - name: Create CRaC checkpoint
+        run: bin/create-checkpoint.sh ${{ env.IMAGE_TAG }}
+      - name: Build and push Docker image
+        uses: exercism/github-actions/.github/workflows/docker-build-push-image.yml@main
+        secrets:
+          AWS_ACCOUNT_ID: ${{secrets.AWS_ACCOUNT_ID}}
+          AWS_REGION: ${{secrets.AWS_REGION}}
+          AWS_ECR_ACCESS_KEY_ID: ${{secrets.AWS_ECR_ACCESS_KEY_ID}}
+          AWS_ECR_SECRET_ACCESS_KEY: ${{secrets.AWS_ECR_SECRET_ACCESS_KEY}}
+          DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}}
+          DOCKERHUB_PASSWORD: ${{secrets.DOCKERHUB_PASSWORD}}

Dockerfile

@@ -1,13 +1,8 @@
-FROM gradle:8.12-jdk21 AS build
-
-WORKDIR /app
-COPY --chown=gradle:gradle . /app
-RUN gradle -i --stacktrace clean build
-
-FROM eclipse-temurin:21
+FROM bellsoft/liberica-runtime-container:jdk-21-crac-musl AS build
 
 WORKDIR /opt/test-runner
-COPY bin/run.sh bin/run.sh
-COPY --from=build /app/build/libs/java-test-runner.jar .
+COPY --link build/libs/java-test-runner.jar /opt/test-runner/java-test-runner.jar
+COPY --link bin/run-restore-from-checkpoint.sh bin/run-restore-from-checkpoint.sh
+COPY --link build/cr /opt/test-runner/crac-checkpoint
 
-ENTRYPOINT ["sh", "/opt/test-runner/bin/run.sh"]
+ENTRYPOINT ["sh", "/opt/test-runner/bin/run-restore-from-checkpoint.sh"]

Dockerfile.createCheckpoint

@@ -0,0 +1,7 @@
+FROM bellsoft/liberica-runtime-container:jdk-21-crac-musl
+
+WORKDIR /opt/test-runner
+COPY --link build/libs/java-test-runner.jar /opt/test-runner/java-test-runner.jar
+COPY --link bin/run-to-create-crac-checkpoint.sh bin/run-to-create-crac-checkpoint.sh
+
+ENTRYPOINT ["sh", "/opt/test-runner/bin/run-to-create-crac-checkpoint.sh"]

README.md

@@ -4,6 +4,10 @@ An [Exercism test runner][test-runner-docs] automatically verifies if a submissi
 
 This repository contains the Java test runner, which implements the V3 spec of the [test runner interface][test-runner-interface-docs].
 
+This test runner uses [CRaC (Coordinated Restore at Checkpoint)](https://crac.org/) to speed up execution.
+The build logic is implemented and documented in [bin/build-crac-checkpoint-image.sh](bin/build-crac-checkpoint-image.sh).
+If you change this logic you might also have to adjust the GitHub [deploy action](.github/workflows/deploy.yml).
+
 ## Run the test runner
 
 To run the tests of an arbitrary exercise, do the following:
@@ -31,11 +35,11 @@ To run the tests to verify the behavior of the test runner, do the following:
 1. Open a terminal in the project's root
 2. Run `./bin/run-tests.sh`
 
-These are [golden tests][golden] that compare the `results.json` generated by running the current state of the code 
+These are [golden tests][golden] that compare the `results.json` generated by running the current state of the code
 against the "known good" `tests/<test-name>/expected_results.json`.
 All files created during the test run itself are discarded.
 
-When you've made modifications to the code that will result in a new "golden" state, 
+When you've made modifications to the code that will result in a new "golden" state,
 you'll need to generate and commit a new `tests/<test-name>/expected_results.json` file.
 
 ## Run the tests using Docker
@@ -47,11 +51,11 @@ To run the tests to verify the behavior of the test runner using the Docker imag
 1. Open a terminal in the project's root
 2. Run `./bin/run-tests-in-docker.sh`
 
-These are [golden tests][golden] that compare the `results.json` generated by running the current state of the code 
-against the "known good" `tests/<test-name>/expected_results.json`. 
+These are [golden tests][golden] that compare the `results.json` generated by running the current state of the code
+against the "known good" `tests/<test-name>/expected_results.json`.
 All files created during the test run itself are discarded.
 
-When you've made modifications to the code that will result in a new "golden" state, 
+When you've made modifications to the code that will result in a new "golden" state,
 you'll need to generate and commit a new `tests/<test-name>/expected_results.json` file.
 
 [test-runner-docs]: https://exercism.org/docs/building/tooling/test-runners

bin/build-crac-checkpoint-image.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Build a Docker image containing a CraC checkpoint to restart from.
+# An initial image is built. A container is created from that image
+# and tests are run to warm up the JVM. Once the runner has finished running
+# all the tests it creates a checkpoint of the VM process before exiting.
+# This checkpoint is written to the host file system, so it can be copied
+# into the final image.
+# Then the final image is created, which restores from the checkpoint
+# instead of starting a new JVM
+# This avoids slow JVM start up and JVM warm up costs.
+#
+# Example:
+# ./bin/build-crac-checkpoint-image.sh
+
+create_checkpoint() {
+    # Copy all tests into one merged project, so we can warm up the JVM
+    # TODO(FAP): this is missing some tests as most tests use the same filenames
+    mkdir -p tests/merged
+    for dir in tests/*; do
+        if [ -d "$dir" ] && [ "$dir" != "tests/merged/" ]; then
+            rsync -a "$dir"/ tests/merged/
+        fi
+    done
+
+    real_path() {
+        echo "$(cd "$(dirname -- "$1")" >/dev/null; pwd -P)/$(basename -- "$1")";
+    }
+
+    mkdir -p build/cr
+
+    image_tag="$1"
+    slug="merged"
+    solution_dir=$(realpath "tests/merged/")
+    output_dir=$(realpath "tests/merged/")
+
+    docker run --cap-add CHECKPOINT_RESTORE \
+           --cap-add SYS_PTRACE \
+           --name java-test-runner-crac \
+           --network none \
+           --mount type=bind,src="${solution_dir}",dst=/solution \
+           --mount type=bind,src="${output_dir}",dst=/output \
+           --mount type=bind,src="$(real_path build/cr)",dst=/opt/test-runner/crac-checkpoint \
+           --mount type=tmpfs,dst=/tmp \
+           "${image_tag}" "${slug}" /solution /output
+
+    docker rm -f java-test-runner-crac
+    rm -rf tests/merged/
+}
+
+# 1. Build jar outside of Docker, so we can copy the jar into both images
+echo "build-crac-checkpoint-image.sh: Building jar with Gradle"
+./gradlew build
+
+image_tag="exercism/java-test-runner-crac-checkpoint"
+
+# 2. Build first image with an entrypoint that will create a CraC checkpoint
+echo "build-crac-checkpoint-image.sh: Building image that creates CraC checkpoint"
+docker build -t "$image_tag" -f Dockerfile.createCheckpoint .
+
+# 3. Run a container from image created in step 2 and create CraC checkpoint
+echo "build-crac-checkpoint-image.sh: Creating CraC checkpoint"
+create_checkpoint "$image_tag"
+
+# Build final test runner image, includes the jar and the checkpoint created in step 1 and 3
+echo "build-crac-checkpoint-image.sh: Building final test runner image"
+docker build -t exercism/java-test-runner -f Dockerfile .

bin/run-in-docker.sh

@@ -8,6 +8,7 @@
 # $1: exercise slug
 # $2: path to solution folder
 # $3: path to output directory
+# $4: [--no-build]: Don't run docker build
 
 # Output:
 # Writes the test results to a results.json file in the passed-in output directory.
@@ -17,8 +18,10 @@
 # ./bin/run-in-docker.sh two-fer path/to/solution/folder/ path/to/output/directory/
 
 # If any required arguments is missing, print the usage and exit
-if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
-    echo "usage: ./bin/run-in-docker.sh exercise-slug path/to/solution/folder/ path/to/output/directory/"
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ "${4:---no-build}" != "--no-build" ]; then
+    echo "usage: ./bin/run-in-docker.sh exercise-slug path/to/solution/folder/ path/to/output/directory/ [--no-build]"
+    echo "All arguments are positional, including the optional --no-build flag"
+    echo "Pass in --no-build as fourth argument to stop the Docker build from running"
     exit 1
 fi
 
@@ -29,8 +32,10 @@ output_dir=$(realpath "${3%/}")
 # Create the output directory if it doesn't exist
 mkdir -p "${output_dir}"
 
-# Build the Docker image
-docker build --rm -t exercism/java-test-runner .
+if [ "$4" != "--no-build" ]; then
+    # Build the Docker image
+    bin/build-crac-checkpoint-image.sh
+fi
 
 # Run the Docker image using the settings mimicking the production environment
 docker run \
@@ -40,4 +45,4 @@ docker run \
     --mount type=bind,src="${solution_dir}",dst=/solution \
     --mount type=bind,src="${output_dir}",dst=/output \
     --mount type=tmpfs,dst=/tmp \
-    exercism/java-test-runner "${slug}" /solution /output 
+    exercism/java-test-runner "${slug}" /solution /output

bin/run-restore-from-checkpoint.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Run the test runner on a solution using the test runner Docker image.
+# The test runner Docker image is built automatically.
+
+# Arguments:
+# $1: exercise slug
+# $2: path to solution folder
+# $3: path to output directory
+
+# Output:
+# Writes the test results to a results.json file in the passed-in output directory.
+# The test results are formatted according to the specifications at https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
+
+# Example:
+# ./bin/run-restore-from-checkpoint.sh two-fer path/to/solution/folder/ path/to/output/directory/
+
+if [ $# -lt 3 ]
+then
+    echo "Usage:"
+    echo "./bin/run-restore-from-checkpoint.sh two-fer ~/input/ ~/output/"
+    exit 1
+fi
+
+problem_slug="$1"
+input_folder="$2"
+output_folder="$3"
+tmp_folder="/tmp/solution"
+
+mkdir -p $output_folder
+
+rm -rf $tmp_folder
+mkdir -p $tmp_folder
+
+cd $tmp_folder
+cp -R $input_folder/* .
+
+find . -mindepth 1 -type f | grep 'Test.java' | xargs -I file sed -i "s/@Ignore(.*)//g;s/@Ignore//g;s/@Disabled(.*)//g;s/@Disabled//g;" file
+
+java -XX:CRaCRestoreFrom=/opt/test-runner/crac-checkpoint com.exercism.Restore $problem_slug . $output_folder

bin/run-tests-in-docker.sh

@@ -1,7 +1,7 @@
 #! /bin/bash -e
 
 # Synopsis:
-# Test the test runner Docker image by running it against a predefined set of 
+# Test the test runner Docker image by running it against a predefined set of
 # solutions with an expected output.
 # The test runner Docker image is built automatically.
 
@@ -13,16 +13,28 @@
 # ./bin/run-tests-in-docker.sh
 
 # Build the Docker image
-docker build --rm -t exercism/java-test-runner .
-
-# Run the Docker image using the settings mimicking the production environment
-docker run \
-    --rm \
-    --network none \
-    --read-only \
-    --mount type=bind,src="${PWD}/tests",dst=/opt/test-runner/tests \
-    --mount type=tmpfs,dst=/tmp \
-    --volume "${PWD}/bin/run-tests.sh:/opt/test-runner/bin/run-tests.sh" \
-    --workdir /opt/test-runner \
-    --entrypoint /opt/test-runner/bin/run-tests.sh \
-    exercism/java-test-runner
+bin/build-crac-checkpoint-image.sh
+
+exit_code=0
+
+# Iterate over all test directories
+for test_dir in tests/*; do
+    test_dir_name=$(basename "${test_dir}")
+    test_dir_path=$(realpath "${test_dir}")
+    results_file_path="${test_dir_path}/results.json"
+    expected_results_file_path="${test_dir_path}/expected_results.json"
+
+    bin/run-in-docker.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}" --no-build
+
+    # Normalize the results file
+    sed -i "s~${test_dir_path}~/solution~g" "${results_file_path}"
+
+    echo "${test_dir_name}: comparing results.json to expected_results.json"
+    diff "${results_file_path}" "${expected_results_file_path}"
+
+    if [ $? -ne 0 ]; then
+        exit_code=1
+    fi
+done
+
+exit ${exit_code}

bin/run-to-create-crac-checkpoint.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Synopsis:
+# Run the test runner on a solution using the test runner Docker image.
+# The test runner Docker image is built automatically.
+
+# Arguments:
+# $1: exercise slug
+# $2: path to solution folder
+# $3: path to output directory
+
+# Output:
+# Writes the test results to a results.json file in the passed-in output directory.
+# The test results are formatted according to the specifications at https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
+
+# Example:
+# ./bin/run-create-crac-checkpoint.sh two-fer path/to/solution/folder/ path/to/output/directory/
+
+if [ $# -lt 3 ]
+then
+    echo "Usage:"
+    echo "./bin/run-create-crac-checkpoint.sh two-fer ~/input/ ~/output/"
+    exit 1
+fi
+
+problem_slug="$1"
+input_folder="$2"
+output_folder="$3"
+tmp_folder="/tmp/solution"
+
+mkdir -p $output_folder
+
+rm -rf $tmp_folder
+mkdir -p $tmp_folder
+
+cd $tmp_folder
+cp -R $input_folder/* .
+
+find . -mindepth 1 -type f | grep 'Test.java' | xargs -I file sed -i "s/@Ignore(.*)//g;s/@Ignore//g;s/@Disabled(.*)//g;s/@Disabled//g;" file
+
+# -XX:-UsePerfData option worked outside of Docker, but inside of Docker the restore would fail
+# See https://docs.azul.com/core/crac/crac-debugging#restore-conflict-of-pids and https://docs.azul.com/core/crac/crac-debugging#using-cracminpid-option
+# for info about -XX:CRaCMinPid
+java -XX:CRaCMinPid=128 -XX:CRaCCheckpointTo=/opt/test-runner/crac-checkpoint -Xshare:off -jar /opt/test-runner/java-test-runner.jar $problem_slug . $output_folder