Repository navigation
diff: add -S and --ignore-file-name-case #344
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: GnuTests | |
| # Run GNU diffutils testsuite against the Rust diffutils implementation | |
| # and compare results against the main branch to catch regressions | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - '*' | |
| permissions: | |
| contents: write # Publish diffutils instead of discarding | |
| # End the current execution if there is a new changeset in the PR | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| env: | |
| DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} | |
| TEST_FULL_SUMMARY_FILE: 'diffutils-gnu-full-result.json' | |
| jobs: | |
| native: | |
| name: Run GNU diffutils testsuite | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| with: | |
| persist-credentials: false | |
| - uses: Swatinem/rust-cache@v2 | |
| ### Build | |
| - name: Build Rust diffutils binary | |
| shell: bash | |
| run: | | |
| ## Build Rust diffutils binary | |
| cargo build --config=profile.release.strip=true --profile=release | |
| zstd -19 target/release/diffutils -o diffutils-x86_64-unknown-linux-gnu.zst | |
| - name: Publish latest commit | |
| uses: softprops/action-gh-release@v3 | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| with: | |
| tag_name: latest-commit | |
| body: | | |
| commit: ${{ github.sha }} | |
| draft: false | |
| prerelease: true | |
| files: | | |
| diffutils-x86_64-unknown-linux-gnu.zst | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ### Run tests | |
| - name: Run GNU diffutils testsuite | |
| shell: bash | |
| run: | | |
| ## Run GNU diffutils testsuite | |
| # Exit code 1 means some tests failed, which is expected and handled | |
| # by the comparison below; 2 means the suite couldn't be run at all | |
| # 'shell: bash' runs with -e, so don't let the expected exit code 1 | |
| # abort the step before the exit code is inspected | |
| result=0 | |
| ./tests/run-upstream-testsuite.sh release || result=$? | |
| if [[ $result -ge 2 ]]; then | |
| echo "::error ::The GNU testsuite could not be run (exit code $result); see the log above" | |
| exit 1 | |
| fi | |
| env: | |
| TERM: xterm | |
| - name: Upload full json results | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: diffutils-gnu-full-result | |
| path: tests/test-results.json | |
| if-no-files-found: warn | |
| aggregate: | |
| needs: [native] | |
| permissions: | |
| actions: read | |
| contents: read | |
| pull-requests: read | |
| name: Aggregate GNU test results | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Initialize workflow variables | |
| id: vars | |
| shell: bash | |
| run: | | |
| ## VARs setup | |
| outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } | |
| TEST_SUMMARY_FILE='diffutils-gnu-result.json' | |
| outputs TEST_SUMMARY_FILE | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| with: | |
| persist-credentials: false | |
| - name: Retrieve reference artifacts | |
| uses: dawidd6/action-download-artifact@v28 | |
| continue-on-error: true | |
| with: | |
| workflow: GnuTests.yml | |
| branch: "${{ env.DEFAULT_BRANCH }}" | |
| workflow_conclusion: completed | |
| path: "reference" | |
| if_no_artifact_found: warn | |
| - name: Download full json results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: diffutils-gnu-full-result | |
| path: results | |
| - name: Extract/summarize testing info | |
| id: summary | |
| shell: bash | |
| run: | | |
| ## Extract/summarize testing info | |
| outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } | |
| RESULT_FILE="results/test-results.json" | |
| if [[ ! -f "$RESULT_FILE" ]]; then | |
| echo "::error ::Missing test results at $RESULT_FILE" | |
| exit 1 | |
| fi | |
| if ! jq -e . "$RESULT_FILE" > /dev/null; then | |
| echo "::error ::Test results at $RESULT_FILE are not valid JSON" | |
| exit 1 | |
| fi | |
| TOTAL=$(jq '[.tests[]] | length' "$RESULT_FILE") | |
| PASS=$(jq '[.tests[] | select(.result=="PASS")] | length' "$RESULT_FILE") | |
| FAIL=$(jq '[.tests[] | select(.result=="FAIL")] | length' "$RESULT_FILE") | |
| SKIP=$(jq '[.tests[] | select(.result=="SKIP")] | length' "$RESULT_FILE") | |
| ERROR=0 | |
| if [[ "$TOTAL" -eq 0 ]]; then | |
| echo "::error ::No test was run; refusing to report or compare an empty test run" | |
| exit 1 | |
| fi | |
| output="GNU diffutils tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP" | |
| echo "${output}" | |
| if [[ "$FAIL" -gt 0 ]]; then | |
| echo "::warning ::${output}" | |
| fi | |
| jq -n \ | |
| --arg date "$(date --rfc-email)" \ | |
| --arg sha "$GITHUB_SHA" \ | |
| --arg total "$TOTAL" \ | |
| --arg pass "$PASS" \ | |
| --arg skip "$SKIP" \ | |
| --arg fail "$FAIL" \ | |
| --arg error "$ERROR" \ | |
| '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, error: $error }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | |
| HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) | |
| outputs HASH TOTAL PASS FAIL SKIP | |
| - name: Upload SHA1/ID of 'test-summary' | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: "${{ steps.summary.outputs.HASH }}" | |
| path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" | |
| - name: Upload test results summary | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-summary | |
| path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" | |
| - name: Compare test failures VS reference | |
| shell: bash | |
| run: | | |
| ## Compare test failures VS reference | |
| REF_SUMMARY_FILE='reference/diffutils-gnu-full-result/test-results.json' | |
| CURRENT_SUMMARY_FILE="results/test-results.json" | |
| IGNORE_INTERMITTENT=".github/workflows/ignore-intermittent.txt" | |
| # Build the comment in a directory of its own: 'reference/' holds the | |
| # artifacts downloaded from the reference run, including its own | |
| # 'comment' artifact, and re-uploading those would post a comparison | |
| # belonging to another run (see https://github.com/uutils/diffutils/pull/262) | |
| COMMENT_DIR="comment" | |
| rm -rf ${COMMENT_DIR} | |
| mkdir -p ${COMMENT_DIR} | |
| echo ${{ github.event.number }} > ${COMMENT_DIR}/NR | |
| COMMENT_LOG="${COMMENT_DIR}/result.txt" | |
| : > "${COMMENT_LOG}" | |
| COMPARISON_RESULT=0 | |
| if test -f "${CURRENT_SUMMARY_FILE}"; then | |
| if test -s "${REF_SUMMARY_FILE}"; then | |
| echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" | |
| echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")" | |
| python3 util/compare_test_results.py \ | |
| --ignore-file "${IGNORE_INTERMITTENT}" \ | |
| --output "${COMMENT_LOG}" \ | |
| "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" | |
| COMPARISON_RESULT=$? | |
| else | |
| echo "::warning ::Skipping test comparison; no usable reference summary is available at '${REF_SUMMARY_FILE}'." | |
| fi | |
| else | |
| echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early" | |
| exit 1 | |
| fi | |
| if [ ${COMPARISON_RESULT} -eq 1 ]; then | |
| echo "::error ::Found new non-intermittent test failures" | |
| exit 1 | |
| elif [ ${COMPARISON_RESULT} -ge 2 ]; then | |
| # The comparison itself failed (e.g. an unusable reference summary). | |
| # Don't post a comment rather than post a misleading one. | |
| : > "${COMMENT_LOG}" | |
| echo "::warning ::Could not compare the test results against the reference" | |
| else | |
| echo "::notice ::No new test failures detected" | |
| fi | |
| - name: Upload comparison log (for GnuComment workflow) | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: comment | |
| path: comment/ | |
| - name: Report test results | |
| if: success() || failure() | |
| shell: bash | |
| run: | | |
| ## Report final results | |
| echo "::notice ::GNU diffutils testsuite results:" | |
| echo "::notice :: Total tests: ${{ steps.summary.outputs.TOTAL }}" | |
| echo "::notice :: Passed: ${{ steps.summary.outputs.PASS }}" | |
| echo "::notice :: Failed: ${{ steps.summary.outputs.FAIL }}" | |
| echo "::notice :: Skipped: ${{ steps.summary.outputs.SKIP }}" | |
| if [[ "${{ steps.summary.outputs.FAIL }}" -gt 0 ]]; then | |
| PASS_RATE=$(( ${{ steps.summary.outputs.PASS }} * 100 / (${{ steps.summary.outputs.PASS }} + ${{ steps.summary.outputs.FAIL }}) )) | |
| echo "::notice :: Pass rate: ${PASS_RATE}%" | |
| fi |