diff --git a/.dockerignore b/.dockerignore index 74322ca..31bdcf5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,7 @@ .appends .git .github -.gradle .idea -gradle tests .dockerignore .gitattributes @@ -11,5 +9,4 @@ tests bin/run-in-docker.sh bin/run-tests.sh bin/run-tests-in-docker.sh -gradlew gradlew.bat diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f843523..fa7f095 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,13 @@ jobs: steps: - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 21 + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 - name: Run Tests in Docker run: bin/run-tests-in-docker.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 43c1cc9..f8a95be 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -9,14 +9,44 @@ on: permissions: contents: write +env: + IMAGE_TAG: exercism/java-test-runner-crac-checkpoint + jobs: build-and-push-image: if: github.repository_owner == 'exercism' # Stops this job from running on forks. - uses: exercism/github-actions/.github/workflows/docker-build-push-image.yml@main - secrets: - AWS_ACCOUNT_ID: ${{secrets.AWS_ACCOUNT_ID}} - AWS_REGION: ${{secrets.AWS_REGION}} - AWS_ECR_ACCESS_KEY_ID: ${{secrets.AWS_ECR_ACCESS_KEY_ID}} - AWS_ECR_SECRET_ACCESS_KEY: ${{secrets.AWS_ECR_SECRET_ACCESS_KEY}} - DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}} - DOCKERHUB_PASSWORD: ${{secrets.DOCKERHUB_PASSWORD}} + steps: + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 21 + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + - name: Build with Gradle + run: ./gradlew build + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and export to Docker + uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 + with: + context: . + file: ./Dockerfile.createCheckpoint + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + # build-args: ${{ secrets.DOCKER_BUILD_ARGS }} + provenance: false + platforms: linux/amd64 + tags: ${{ env.IMAGE_TAG }} + - name: Create CRaC checkpoint + run: bin/create-checkpoint.sh ${{ env.IMAGE_TAG }} + - name: Build and push Docker image + uses: exercism/github-actions/.github/workflows/docker-build-push-image.yml@main + secrets: + AWS_ACCOUNT_ID: ${{secrets.AWS_ACCOUNT_ID}} + AWS_REGION: ${{secrets.AWS_REGION}} + AWS_ECR_ACCESS_KEY_ID: ${{secrets.AWS_ECR_ACCESS_KEY_ID}} + AWS_ECR_SECRET_ACCESS_KEY: ${{secrets.AWS_ECR_SECRET_ACCESS_KEY}} + DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}} + DOCKERHUB_PASSWORD: ${{secrets.DOCKERHUB_PASSWORD}} diff --git a/Dockerfile b/Dockerfile index 49826d8..7f895bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,8 @@ -FROM gradle:8.12-jdk21 AS build - -WORKDIR /app -COPY --chown=gradle:gradle . /app -RUN gradle -i --stacktrace clean build - -FROM eclipse-temurin:21 +FROM bellsoft/liberica-runtime-container:jdk-21-crac-musl AS build WORKDIR /opt/test-runner -COPY bin/run.sh bin/run.sh -COPY --from=build /app/build/libs/java-test-runner.jar . +COPY --link build/libs/java-test-runner.jar /opt/test-runner/java-test-runner.jar +COPY --link bin/run-restore-from-checkpoint.sh bin/run-restore-from-checkpoint.sh +COPY --link build/cr /opt/test-runner/crac-checkpoint -ENTRYPOINT ["sh", "/opt/test-runner/bin/run.sh"] +ENTRYPOINT ["sh", "/opt/test-runner/bin/run-restore-from-checkpoint.sh"] diff --git a/Dockerfile.createCheckpoint b/Dockerfile.createCheckpoint new file mode 100644 index 0000000..5df2252 --- /dev/null +++ b/Dockerfile.createCheckpoint @@ -0,0 +1,7 @@ +FROM bellsoft/liberica-runtime-container:jdk-21-crac-musl + +WORKDIR /opt/test-runner +COPY --link build/libs/java-test-runner.jar /opt/test-runner/java-test-runner.jar +COPY --link bin/run-to-create-crac-checkpoint.sh bin/run-to-create-crac-checkpoint.sh + +ENTRYPOINT ["sh", "/opt/test-runner/bin/run-to-create-crac-checkpoint.sh"] diff --git a/README.md b/README.md index e09321f..4f42a36 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ An [Exercism test runner][test-runner-docs] automatically verifies if a submissi This repository contains the Java test runner, which implements the V3 spec of the [test runner interface][test-runner-interface-docs]. +This test runner uses [CRaC (Coordinated Restore at Checkpoint)](https://crac.org/) to speed up execution. +The build logic is implemented and documented in [bin/build-crac-checkpoint-image.sh](bin/build-crac-checkpoint-image.sh). +If you change this logic you might also have to adjust the GitHub [deploy action](.github/workflows/deploy.yml). + ## Run the test runner To run the tests of an arbitrary exercise, do the following: @@ -31,11 +35,11 @@ To run the tests to verify the behavior of the test runner, do the following: 1. Open a terminal in the project's root 2. Run `./bin/run-tests.sh` -These are [golden tests][golden] that compare the `results.json` generated by running the current state of the code +These are [golden tests][golden] that compare the `results.json` generated by running the current state of the code against the "known good" `tests//expected_results.json`. All files created during the test run itself are discarded. -When you've made modifications to the code that will result in a new "golden" state, +When you've made modifications to the code that will result in a new "golden" state, you'll need to generate and commit a new `tests//expected_results.json` file. ## Run the tests using Docker @@ -47,11 +51,11 @@ To run the tests to verify the behavior of the test runner using the Docker imag 1. Open a terminal in the project's root 2. Run `./bin/run-tests-in-docker.sh` -These are [golden tests][golden] that compare the `results.json` generated by running the current state of the code -against the "known good" `tests//expected_results.json`. +These are [golden tests][golden] that compare the `results.json` generated by running the current state of the code +against the "known good" `tests//expected_results.json`. All files created during the test run itself are discarded. -When you've made modifications to the code that will result in a new "golden" state, +When you've made modifications to the code that will result in a new "golden" state, you'll need to generate and commit a new `tests//expected_results.json` file. [test-runner-docs]: https://exercism.org/docs/building/tooling/test-runners diff --git a/bin/build-crac-checkpoint-image.sh b/bin/build-crac-checkpoint-image.sh new file mode 100755 index 0000000..6f7c949 --- /dev/null +++ b/bin/build-crac-checkpoint-image.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env sh + +# Synopsis: +# Build a Docker image containing a CraC checkpoint to restart from. +# An initial image is built. A container is created from that image +# and tests are run to warm up the JVM. Once the runner has finished running +# all the tests it creates a checkpoint of the VM process before exiting. +# This checkpoint is written to the host file system, so it can be copied +# into the final image. +# Then the final image is created, which restores from the checkpoint +# instead of starting a new JVM +# This avoids slow JVM start up and JVM warm up costs. +# +# Example: +# ./bin/build-crac-checkpoint-image.sh + +create_checkpoint() { + # Copy all tests into one merged project, so we can warm up the JVM + # TODO(FAP): this is missing some tests as most tests use the same filenames + mkdir -p tests/merged + for dir in tests/*; do + if [ -d "$dir" ] && [ "$dir" != "tests/merged/" ]; then + rsync -a "$dir"/ tests/merged/ + fi + done + + real_path() { + echo "$(cd "$(dirname -- "$1")" >/dev/null; pwd -P)/$(basename -- "$1")"; + } + + mkdir -p build/cr + + image_tag="$1" + slug="merged" + solution_dir=$(realpath "tests/merged/") + output_dir=$(realpath "tests/merged/") + + docker run --cap-add CHECKPOINT_RESTORE \ + --cap-add SYS_PTRACE \ + --name java-test-runner-crac \ + --network none \ + --mount type=bind,src="${solution_dir}",dst=/solution \ + --mount type=bind,src="${output_dir}",dst=/output \ + --mount type=bind,src="$(real_path build/cr)",dst=/opt/test-runner/crac-checkpoint \ + --mount type=tmpfs,dst=/tmp \ + "${image_tag}" "${slug}" /solution /output + + docker rm -f java-test-runner-crac + rm -rf tests/merged/ +} + +# 1. Build jar outside of Docker, so we can copy the jar into both images +echo "build-crac-checkpoint-image.sh: Building jar with Gradle" +./gradlew build + +image_tag="exercism/java-test-runner-crac-checkpoint" + +# 2. Build first image with an entrypoint that will create a CraC checkpoint +echo "build-crac-checkpoint-image.sh: Building image that creates CraC checkpoint" +docker build -t "$image_tag" -f Dockerfile.createCheckpoint . + +# 3. Run a container from image created in step 2 and create CraC checkpoint +echo "build-crac-checkpoint-image.sh: Creating CraC checkpoint" +create_checkpoint "$image_tag" + +# Build final test runner image, includes the jar and the checkpoint created in step 1 and 3 +echo "build-crac-checkpoint-image.sh: Building final test runner image" +docker build -t exercism/java-test-runner -f Dockerfile . diff --git a/bin/run-in-docker.sh b/bin/run-in-docker.sh index 188caef..0e8b38a 100755 --- a/bin/run-in-docker.sh +++ b/bin/run-in-docker.sh @@ -8,6 +8,7 @@ # $1: exercise slug # $2: path to solution folder # $3: path to output directory +# $4: [--no-build]: Don't run docker build # Output: # Writes the test results to a results.json file in the passed-in output directory. @@ -17,8 +18,10 @@ # ./bin/run-in-docker.sh two-fer path/to/solution/folder/ path/to/output/directory/ # If any required arguments is missing, print the usage and exit -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then - echo "usage: ./bin/run-in-docker.sh exercise-slug path/to/solution/folder/ path/to/output/directory/" +if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ "${4:---no-build}" != "--no-build" ]; then + echo "usage: ./bin/run-in-docker.sh exercise-slug path/to/solution/folder/ path/to/output/directory/ [--no-build]" + echo "All arguments are positional, including the optional --no-build flag" + echo "Pass in --no-build as fourth argument to stop the Docker build from running" exit 1 fi @@ -29,8 +32,10 @@ output_dir=$(realpath "${3%/}") # Create the output directory if it doesn't exist mkdir -p "${output_dir}" -# Build the Docker image -docker build --rm -t exercism/java-test-runner . +if [ "$4" != "--no-build" ]; then + # Build the Docker image + bin/build-crac-checkpoint-image.sh +fi # Run the Docker image using the settings mimicking the production environment docker run \ @@ -40,4 +45,4 @@ docker run \ --mount type=bind,src="${solution_dir}",dst=/solution \ --mount type=bind,src="${output_dir}",dst=/output \ --mount type=tmpfs,dst=/tmp \ - exercism/java-test-runner "${slug}" /solution /output + exercism/java-test-runner "${slug}" /solution /output diff --git a/bin/run-restore-from-checkpoint.sh b/bin/run-restore-from-checkpoint.sh new file mode 100755 index 0000000..b1fd36b --- /dev/null +++ b/bin/run-restore-from-checkpoint.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env sh + +# Synopsis: +# Run the test runner on a solution using the test runner Docker image. +# The test runner Docker image is built automatically. + +# Arguments: +# $1: exercise slug +# $2: path to solution folder +# $3: path to output directory + +# Output: +# Writes the test results to a results.json file in the passed-in output directory. +# The test results are formatted according to the specifications at https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md + +# Example: +# ./bin/run-restore-from-checkpoint.sh two-fer path/to/solution/folder/ path/to/output/directory/ + +if [ $# -lt 3 ] +then + echo "Usage:" + echo "./bin/run-restore-from-checkpoint.sh two-fer ~/input/ ~/output/" + exit 1 +fi + +problem_slug="$1" +input_folder="$2" +output_folder="$3" +tmp_folder="/tmp/solution" + +mkdir -p $output_folder + +rm -rf $tmp_folder +mkdir -p $tmp_folder + +cd $tmp_folder +cp -R $input_folder/* . + +find . -mindepth 1 -type f | grep 'Test.java' | xargs -I file sed -i "s/@Ignore(.*)//g;s/@Ignore//g;s/@Disabled(.*)//g;s/@Disabled//g;" file + +java -XX:CRaCRestoreFrom=/opt/test-runner/crac-checkpoint com.exercism.Restore $problem_slug . $output_folder diff --git a/bin/run-tests-in-docker.sh b/bin/run-tests-in-docker.sh index 6cceeb6..0abea45 100755 --- a/bin/run-tests-in-docker.sh +++ b/bin/run-tests-in-docker.sh @@ -1,7 +1,7 @@ #! /bin/bash -e # Synopsis: -# Test the test runner Docker image by running it against a predefined set of +# Test the test runner Docker image by running it against a predefined set of # solutions with an expected output. # The test runner Docker image is built automatically. @@ -13,16 +13,28 @@ # ./bin/run-tests-in-docker.sh # Build the Docker image -docker build --rm -t exercism/java-test-runner . - -# Run the Docker image using the settings mimicking the production environment -docker run \ - --rm \ - --network none \ - --read-only \ - --mount type=bind,src="${PWD}/tests",dst=/opt/test-runner/tests \ - --mount type=tmpfs,dst=/tmp \ - --volume "${PWD}/bin/run-tests.sh:/opt/test-runner/bin/run-tests.sh" \ - --workdir /opt/test-runner \ - --entrypoint /opt/test-runner/bin/run-tests.sh \ - exercism/java-test-runner +bin/build-crac-checkpoint-image.sh + +exit_code=0 + +# Iterate over all test directories +for test_dir in tests/*; do + test_dir_name=$(basename "${test_dir}") + test_dir_path=$(realpath "${test_dir}") + results_file_path="${test_dir_path}/results.json" + expected_results_file_path="${test_dir_path}/expected_results.json" + + bin/run-in-docker.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}" --no-build + + # Normalize the results file + sed -i "s~${test_dir_path}~/solution~g" "${results_file_path}" + + echo "${test_dir_name}: comparing results.json to expected_results.json" + diff "${results_file_path}" "${expected_results_file_path}" + + if [ $? -ne 0 ]; then + exit_code=1 + fi +done + +exit ${exit_code} diff --git a/bin/run-to-create-crac-checkpoint.sh b/bin/run-to-create-crac-checkpoint.sh new file mode 100755 index 0000000..97af004 --- /dev/null +++ b/bin/run-to-create-crac-checkpoint.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Synopsis: +# Run the test runner on a solution using the test runner Docker image. +# The test runner Docker image is built automatically. + +# Arguments: +# $1: exercise slug +# $2: path to solution folder +# $3: path to output directory + +# Output: +# Writes the test results to a results.json file in the passed-in output directory. +# The test results are formatted according to the specifications at https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md + +# Example: +# ./bin/run-create-crac-checkpoint.sh two-fer path/to/solution/folder/ path/to/output/directory/ + +if [ $# -lt 3 ] +then + echo "Usage:" + echo "./bin/run-create-crac-checkpoint.sh two-fer ~/input/ ~/output/" + exit 1 +fi + +problem_slug="$1" +input_folder="$2" +output_folder="$3" +tmp_folder="/tmp/solution" + +mkdir -p $output_folder + +rm -rf $tmp_folder +mkdir -p $tmp_folder + +cd $tmp_folder +cp -R $input_folder/* . + +find . -mindepth 1 -type f | grep 'Test.java' | xargs -I file sed -i "s/@Ignore(.*)//g;s/@Ignore//g;s/@Disabled(.*)//g;s/@Disabled//g;" file + +# -XX:-UsePerfData option worked outside of Docker, but inside of Docker the restore would fail +# See https://docs.azul.com/core/crac/crac-debugging#restore-conflict-of-pids and https://docs.azul.com/core/crac/crac-debugging#using-cracminpid-option +# for info about -XX:CRaCMinPid +java -XX:CRaCMinPid=128 -XX:CRaCCheckpointTo=/opt/test-runner/crac-checkpoint -Xshare:off -jar /opt/test-runner/java-test-runner.jar $problem_slug . $output_folder diff --git a/build.gradle b/build.gradle index a50c877..6fde5fa 100644 --- a/build.gradle +++ b/build.gradle @@ -24,6 +24,8 @@ dependencies { implementation "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$jacksonVersion" implementation 'com.github.javaparser:javaparser-core:3.26.3' + implementation 'org.crac:crac:1.5.0' + implementation 'org.assertj:assertj-core:3.25.3' implementation 'org.apiguardian:apiguardian-api:1.1.2' // https://github.com/exercism/java-test-runner/issues/79 implementation platform('org.junit:junit-bom:5.11.3') diff --git a/src/main/java/com/exercism/Restore.java b/src/main/java/com/exercism/Restore.java new file mode 100644 index 0000000..e99d3c3 --- /dev/null +++ b/src/main/java/com/exercism/Restore.java @@ -0,0 +1,11 @@ +package com.exercism; + +import java.io.IOException; + +public class Restore { + + public static void main(String[] args) throws IOException { + new TestRunner(args[0], args[1], args[2]).run(); + } + +} diff --git a/src/main/java/com/exercism/TestRunner.java b/src/main/java/com/exercism/TestRunner.java index 06c144c..e1b666c 100644 --- a/src/main/java/com/exercism/TestRunner.java +++ b/src/main/java/com/exercism/TestRunner.java @@ -17,6 +17,10 @@ import java.util.List; import java.util.stream.Stream; +import org.crac.CheckpointException; +import org.crac.Core; +import org.crac.RestoreException; + public final class TestRunner { private final JUnitTestParser testParser; @@ -33,14 +37,16 @@ public TestRunner(String slug, String inputDirectory, String outputDirectory) { this.inputDirectory = inputDirectory; } - public static void main(String[] args) throws IOException { + public static void main(String[] args) throws IOException, CheckpointException, RestoreException { if (args.length < 3) { throw new IllegalArgumentException("Not enough arguments, need "); } new TestRunner(args[0], args[1], args[2]).run(); + + Core.checkpointRestore(); } - private void run() throws IOException { + void run() throws IOException { var sourceFiles = resolveSourceFiles(); var testFiles = resolveTestFiles();