Repository navigation
chore(ci): add two-stage secret gate #5
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Stage 1 of 2. Untrusted: runs the PR's own copy of this file with a read-only | |
| # token and no secrets. | |
| # Blocks on: betterleaks errors, added scanner-suppression files or inline allow | |
| # annotations, semgrep failing to install. Advisory: betterleaks and semgrep | |
| # findings; the blocking secret verdict is stage 2's `AI secret verdict` status. | |
| # The PR can edit this file, its config and scripts, so CODEOWNERS review of the | |
| # gate files is required. | |
| # The artifact is human-facing only. Stage 2 re-scans and must never consume it. | |
| name: PR Security Scan | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| permissions: | |
| contents: read | |
| concurrency: | |
| # Keyed on the head repo so forks cannot share a group. A PR can rewrite this | |
| # in its own copy, so it is friction, not a boundary. | |
| group: pr-security-scan-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.repo.full_name }} | |
| cancel-in-progress: true | |
| env: | |
| # Bump together with BETTERLEAKS_SHA256. | |
| BETTERLEAKS_VERSION: "1.8.1" | |
| BETTERLEAKS_SHA256: "efa407244e1ea8e35f582b8a42becdeac08bdead04f68eb752adda722d583c2a" | |
| # Must be a version published on PyPI (`pip index versions semgrep`). | |
| SEMGREP_VERSION: "1.177.0" | |
| jobs: | |
| scan: | |
| # Required check name in the branch ruleset; keep in sync with README. | |
| name: Secret & vulnerability scan | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Check out PR merge ref with full history | |
| uses: actions/checkout@v7 | |
| with: | |
| # A secret added and later deleted within the PR is still in history. | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - uses: actions/setup-python@v7 | |
| with: | |
| python-version: "3.12" | |
| - name: Resolve the commit range | |
| id: range | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| run: | | |
| set -euo pipefail | |
| git fetch --no-tags origin "$BASE_SHA" 2>/dev/null || true | |
| git fetch --no-tags origin "$HEAD_SHA" 2>/dev/null || true | |
| MERGE_BASE="$(git merge-base "$BASE_SHA" "$HEAD_SHA" 2>/dev/null || echo "$BASE_SHA")" | |
| echo "merge_base=$MERGE_BASE" >> "$GITHUB_OUTPUT" | |
| echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" | |
| echo "Scanning range ${MERGE_BASE}..${HEAD_SHA}" | |
| # Ignore files and the inline allow annotation live outside .github/, so | |
| # CODEOWNERS never sees them, and betterleaks reads a .gitleaksignore from | |
| # the scan target regardless of --gitleaks-ignore-path. Both are rejected. | |
| - name: Reject scanner-suppression additions | |
| env: | |
| MERGE_BASE: ${{ steps.range.outputs.merge_base }} | |
| HEAD_SHA: ${{ steps.range.outputs.head_sha }} | |
| run: | | |
| set -euo pipefail | |
| FAIL=0 | |
| # `grep -c`, not `grep -q`: -q exits early, SIGPIPEs the producer, and | |
| # pipefail then makes the `if` false. `|| true` because no match exits 1. | |
| IGNORE_FILES="$(git diff --name-only "$MERGE_BASE" "$HEAD_SHA" \ | |
| | grep -cE '(^|/)\.(better)?leaksignore$|(^|/)\.gitleaksignore$' || true)" | |
| if [ "${IGNORE_FILES:-0}" -gt 0 ]; then | |
| echo "::error title=Suppression file::This PR adds or modifies a scanner ignore file, which can silence the secret gate. Not permitted." | |
| FAIL=1 | |
| fi | |
| # Added lines only. Matches the annotation text anywhere, prose included, | |
| # so repo docs describe it rather than spell it out. | |
| INLINE_ALLOWS="$(git diff --unified=0 "$MERGE_BASE" "$HEAD_SHA" \ | |
| | grep -E '^\+' | grep -ciE '(git|better)leaks[: ]*allow' || true)" | |
| if [ "${INLINE_ALLOWS:-0}" -gt 0 ]; then | |
| echo "::error title=Inline suppression::This PR adds an inline scanner-allow annotation, which silences the secret gate for that line. Not permitted." | |
| FAIL=1 | |
| fi | |
| exit $FAIL | |
| - name: Install betterleaks (checksum-pinned) | |
| run: | | |
| set -euo pipefail | |
| curl -sSfL --proto '=https' --tlsv1.2 -o /tmp/betterleaks.tar.gz \ | |
| "https://github.com/betterleaks/betterleaks/releases/download/v${BETTERLEAKS_VERSION}/betterleaks_${BETTERLEAKS_VERSION}_linux_x64.tar.gz" | |
| # Refuse to run an unverifiable binary. | |
| if ! printf '%s' "$BETTERLEAKS_SHA256" | grep -qE '^[0-9a-f]{64}$'; then | |
| echo "::error title=Unpinned binary::BETTERLEAKS_SHA256 is not a sha256 digest, so the scanner binary cannot be verified. Refusing to run it. Set it from the checksums file for v${BETTERLEAKS_VERSION}." | |
| exit 1 | |
| fi | |
| echo "${BETTERLEAKS_SHA256} /tmp/betterleaks.tar.gz" | sha256sum -c - | |
| tar -xzf /tmp/betterleaks.tar.gz -C /tmp betterleaks | |
| sudo install -m 0755 /tmp/betterleaks /usr/local/bin/betterleaks | |
| betterleaks version | |
| # The `gitleaks` step id and file names stay; betterleaks reads the same config. | |
| - name: Run betterleaks over the PR commits | |
| id: gitleaks | |
| env: | |
| MERGE_BASE: ${{ steps.range.outputs.merge_base }} | |
| HEAD_SHA: ${{ steps.range.outputs.head_sha }} | |
| run: | | |
| set -uo pipefail | |
| # Empty dir, so the ignore-file lookup does not default to the PR-controlled repo root. | |
| mkdir -p /tmp/gl-noignore | |
| # --exit-code=0: findings exit 0, so any non-zero exit is a scanner error. | |
| # --max-target-megabytes caps decode/archive recursion over PR-supplied bytes. | |
| set +e | |
| betterleaks git . \ | |
| --config=.github/security/gitleaks-runpod.toml \ | |
| --log-opts="${MERGE_BASE}..${HEAD_SHA}" \ | |
| --gitleaks-ignore-path=/tmp/gl-noignore \ | |
| --ignore-gitleaks-allow \ | |
| --report-format=json \ | |
| --report-path=/tmp/gitleaks.raw.json \ | |
| --exit-code=0 \ | |
| --max-target-megabytes=25 \ | |
| --no-banner | |
| GL_RC=$? | |
| set -e | |
| # Fail closed: a crashed scanner must not look like a clean PR. | |
| if [ "$GL_RC" -ne 0 ]; then | |
| echo "::error title=Secret scan errored::betterleaks exited $GL_RC. Treating as FAILURE, not as a pass." | |
| echo "count=error" >> "$GITHUB_OUTPUT" | |
| exit 1 | |
| fi | |
| [ -f /tmp/gitleaks.raw.json ] || echo '[]' > /tmp/gitleaks.raw.json | |
| # betterleaks writes `null` for zero findings. Null or empty counts as 0; | |
| # any other non-list shape is a parse failure. | |
| COUNT="" | |
| COUNT="$(python3 -c 'import json,os,sys; p="/tmp/gitleaks.raw.json"; d=json.load(open(p)) if os.path.getsize(p) else []; d=[] if d is None else d; sys.exit("not a JSON array") if not isinstance(d,list) else print(len(d))')" || true | |
| if ! [[ "$COUNT" =~ ^[0-9]+$ ]]; then | |
| echo "::error title=Secret scan unreadable::could not parse the betterleaks report. Treating as FAILURE." | |
| echo "count=error" >> "$GITHUB_OUTPUT" | |
| exit 1 | |
| fi | |
| echo "count=$COUNT" >> "$GITHUB_OUTPUT" | |
| echo "betterleaks findings: $COUNT" | |
| # No path exclusions: only the stage-2 prompt skips noise files, so hiding a | |
| # file from the model never hides it from a scanner. | |
| - name: Run semgrep on changed files | |
| # Findings are advisory; a failed install is fatal. | |
| env: | |
| MERGE_BASE: ${{ steps.range.outputs.merge_base }} | |
| HEAD_SHA: ${{ steps.range.outputs.head_sha }} | |
| run: | | |
| set -uo pipefail | |
| # NUL-delimited so odd filenames cannot be split or read as flags. | |
| git diff -z --name-only --diff-filter=ACMR "$MERGE_BASE" "$HEAD_SHA" \ | |
| > /tmp/changed.z || true | |
| python3 -m venv /tmp/sgvenv | |
| # Retry transient PyPI errors; still failing after 3 attempts means a bad pin. | |
| installed=0 | |
| for attempt in 1 2 3; do | |
| if /tmp/sgvenv/bin/pip install --quiet "semgrep==${SEMGREP_VERSION}"; then | |
| installed=1 | |
| break | |
| fi | |
| echo "semgrep install attempt ${attempt} failed" | |
| [ "$attempt" -lt 3 ] && sleep $((attempt * 5)) | |
| done | |
| if [ "$installed" -ne 1 ]; then | |
| echo "::error title=semgrep unavailable::semgrep==${SEMGREP_VERSION} would not install after 3 attempts. If that version is not published on PyPI this is a bad pin — fix SEMGREP_VERSION. Static analysis is NOT being reported as clean." | |
| exit 1 | |
| fi | |
| if [ -s /tmp/changed.z ]; then | |
| # xargs splits a long file list into several semgrep runs, so each run | |
| # writes its own report and they are merged below. An empty report | |
| # means that run did not complete (e.g. registry unreachable); it marks | |
| # the whole scan as not scanned rather than as zero findings. Trailing | |
| # `--` so a file named "--config=evil.py" is a path, not a flag. | |
| # shellcheck disable=SC2016 # the inner sh expands $out and $@ | |
| xargs -0 -a /tmp/changed.z sh -c ' | |
| out=$(mktemp /tmp/semgrep.part.XXXXXX) | |
| /tmp/sgvenv/bin/semgrep scan \ | |
| --config=p/secrets --config=p/security-audit --config=p/owasp-top-ten \ | |
| --json --output="$out" --metrics=off --quiet --timeout=120 -- "$@" || true | |
| [ -s "$out" ] || echo "{\"results\":[],\"__not_scanned\":\"scan did not complete\"}" > "$out" | |
| ' sh || true | |
| jq -s '{results: (map(.results // []) | add)} | |
| + (map(.__not_scanned // empty) | if length > 0 then {__not_scanned: .[0]} else {} end)' \ | |
| /tmp/semgrep.part.* > /tmp/semgrep.raw.json \ | |
| || echo '{"results":[],"__not_scanned":"scan did not complete"}' > /tmp/semgrep.raw.json | |
| else | |
| echo '{"results":[]}' > /tmp/semgrep.raw.json | |
| fi | |
| # Raw reports stay in /tmp; only sanitised findings reach artifact/. No diff is | |
| # uploaded: its base-branch deleted and context lines are never redacted. | |
| - name: Sanitise findings for upload | |
| if: always() | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| HEAD_SHA: ${{ steps.range.outputs.head_sha }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p artifact | |
| [ -f /tmp/gitleaks.raw.json ] || echo '[]' > /tmp/gitleaks.raw.json | |
| [ -f /tmp/semgrep.raw.json ] \ | |
| || echo '{"results":[],"__not_scanned":"report missing"}' \ | |
| > /tmp/semgrep.raw.json | |
| python3 .github/security/sanitize_findings.py \ | |
| --gitleaks-report /tmp/gitleaks.raw.json \ | |
| --semgrep /tmp/semgrep.raw.json \ | |
| --out-findings artifact/gitleaks.sanitized.json \ | |
| --out-semgrep artifact/semgrep.json \ | |
| --out-meta artifact/meta.json | |
| - name: Upload sanitised findings | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: pr-security-context | |
| path: artifact/ | |
| retention-days: 7 | |
| if-no-files-found: warn | |
| - name: Report secret findings (advisory) | |
| if: steps.gitleaks.outputs.count != '0' | |
| env: | |
| COUNT: ${{ steps.gitleaks.outputs.count }} | |
| run: | | |
| echo "::warning title=Secret detected::betterleaks reported ${COUNT} finding(s) in this PR's commits." | |
| echo "" | |
| echo "Values are intentionally not printed — CI logs for a public repo are public." | |
| echo "Redacted locations are in the 'pr-security-context' artifact." | |
| echo "" | |
| echo "This step does not fail the build; the 'AI secret verdict' status does." | |
| echo "If any of these is real:" | |
| echo "" | |
| echo " 1. ROTATE the credential first. It is already in git history and on" | |
| echo " GitHub's servers; removing the line does not un-leak it." | |
| echo " 2. Then rewrite the branch history to drop the blob." | |
| echo " 3. Re-push. This check will clear." |