diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 406a9d3..fef07c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -218,8 +218,11 @@ jobs: # ---------------------------------------------------------------------- # No-system-deps guarantee + packaging check. Fails if any forbidden - # numerical/BLAS crate has crept into the dependency tree, then dry-runs - # the publishable crates to catch packaging problems before a real release. + # numerical/BLAS crate has crept into the dependency tree, then checks the + # publishable crate artifacts before a real release. The lockstep + # ordvec-manifest crate cannot run a registry-backed dry-run for a bumped + # version until ordvec itself is published, so CI verifies the pre-publish + # package artifact that release.yml later byte-compares after the core publish. # ---------------------------------------------------------------------- deps: name: deps (no-system-deps + publish dry-run) @@ -258,21 +261,22 @@ jobs: # from the current directory. The Python binding remains publish = false # and ships to PyPI via maturin. run: cargo publish -p ordvec --dry-run --locked - - name: cargo publish --dry-run -p ordvec-manifest - run: cargo publish -p ordvec-manifest --dry-run --locked + - name: cargo package --no-verify -p ordvec-manifest + run: cargo package -p ordvec-manifest --locked --no-verify # ---------------------------------------------------------------------- # Pin the release-publish invariants. release.yml is tag-triggered (with the - # two registry publishes gated behind GitHub Environments), so its release- + # registry publishes gated behind GitHub Environments), so its release- # specific flow runs only on a real release. Two structural lints guard it # on every push/PR so regressions can't sneak in between releases: # * release_publish_invariants.sh — the *.cdx.json SBOM never reaches PyPI # (a generated SBOM once broke both publish paths). # * release_signed_release_invariants.sh — the signed-release / provenance # graph stays intact: release-assets-draft stays draft, the SLSA - # generator emits a .intoto.jsonl, both publishes need the draft assets, - # publish-crate proves byte-identity vs the attested .crate, and - # publish-github-release un-drafts ONLY after both publishes succeed. + # generator emits a .intoto.jsonl, registry publishes need the draft + # assets, both Rust crates prove byte-identity vs their attested .crate + # files, and publish-github-release un-drafts ONLY after all registry + # publishes succeed. # ---------------------------------------------------------------------- release-guard: name: release-publish invariants diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 53a9e0c..df30075 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,14 +1,14 @@ -# Unified, tag-triggered release pipeline for ordvec (crate + Python wheel). +# Unified, tag-triggered release pipeline for ordvec (Rust crates + Python wheel). # -# Cutting a stable `vMAJOR.MINOR.PATCH` tag fully automates: build (crate + +# Cutting a stable `vMAJOR.MINOR.PATCH` tag fully automates: build (crates + # wheels + sdist) -> canonicalize the Python dist (current build for new # versions, verified PyPI-served bytes if PyPI already owns the immutable # version) -> attest / SLSA-provenance the files this run actually built -> # stage EVERYTHING on the DRAFT GitHub Release (`release-assets-draft`) -> -# gated registry publishes / verification -> un-draft ONLY after BOTH gates -# succeed (`publish-github-release`). The two registry gates (crates.io, PyPI) -# are bound to GitHub Environments with Required Reviewers, so they pause for a -# human. +# gated registry publishes / verification -> un-draft ONLY after all registry +# jobs succeed (`publish-github-release`). The registry gates (two crates.io +# jobs plus PyPI) are bound to GitHub Environments with Required Reviewers, so +# they pause for a human. # # The un-draft-after-publish ordering is deliberate: it prevents a public # GitHub Release from existing for a version that crates.io / PyPI later @@ -16,19 +16,19 @@ # investigates / re-runs the failed job, after which `publish-github-release` # un-drafts. # -# `publish-crate` proves BYTE-IDENTITY between the .crate served by crates.io -# and the SLSA-attested artifact on BOTH sides of `cargo publish`: -# * pre-publish: downloads the `dist-crate` artifact, re-packages with -# `--locked`, sha256-compares — fail-closed BEFORE the OIDC token is -# minted (defends against toolchain drift / deterministic-packaging -# regression). +# `publish-crate` and `publish-manifest-crate` prove BYTE-IDENTITY between the +# .crate served by crates.io and the SLSA-attested artifact on BOTH sides of +# `cargo publish`: +# * pre-publish: downloads the package's dist artifact, re-packages with +# `--locked`, sha256-compares — fail-closed BEFORE the OIDC token is minted +# (defends against toolchain drift / deterministic-packaging regression). # * post-publish: downloads the just-published .crate from crates.io and -# sha256-compares to the attested artifact — empirical proof that the -# bytes crates.io actually serves equal the SLSA-attested bytes -# (`cargo publish` runs its own internal packaging step the pre-publish -# gate cannot inspect). A mismatch fails closed: `publish-github-release` -# never un-drafts the GitHub Release, and the failure is loudly audit- -# logged even though the version is then yank-only. +# sha256-compares to the attested artifact — empirical proof that the bytes +# crates.io actually serves equal the SLSA-attested bytes (`cargo publish` +# runs its own internal packaging step the pre-publish gate cannot inspect). +# A mismatch fails closed: `publish-github-release` never un-drafts the +# GitHub Release, and the failure is loudly audit-logged even though the +# version is then yank-only. # # Why one workflow (not the old three): the GitHub Release assets + the single # un-draft must be coordinated by explicit `needs:` edges. The previous @@ -41,8 +41,8 @@ # * SLSA generator -> `*.intoto.jsonl` on the Release (OpenSSF Scorecard # Signed-Releases provenance probe; older unsigned releases may keep that # score below 10 temporarily; SLSA Build L3). Recovery mode limits SLSA -# subjects to the crate built by this run, because PyPI files are immutable -# bytes from an earlier Trusted Publishing upload. +# subjects to the Rust crates built by this run, because PyPI files are +# immutable bytes from an earlier Trusted Publishing upload. # * actions/attest-build-provenance -> GitHub attestation store + a # `*.sigstore.json` bundle on the Release (`gh attestation verify`; also the # Scorecard signing probe sees this asset as a backup if the .intoto.jsonl @@ -57,18 +57,19 @@ # * crates.io / PyPI publish/verification via Trusted Publishing gates (OIDC # only when uploading) — NO stored tokens. # -# Fail-closed: `release-assets-draft` and both publishes `needs:` attest + +# Fail-closed: `release-assets-draft` and registry publishes `needs:` attest + # provenance and canonical Python dist selection, so nothing is attached or # published unless the artifact source is verified; and `publish-github-release` -# `needs:` both registry gates, so the Release stays DRAFT unless both pass. +# `needs:` every registry gate, so the Release stays DRAFT unless all pass. # The signed-release graph is pinned in # `tests/release_signed_release_invariants.sh` (run by ci.yml's release-guard # on every push/PR) so a future commit can't silently dismantle it. # # ONE-TIME SETUP before the first tag on this workflow (each fails CLOSED until # done, so landing this first is safe): -# * crates.io: ordvec > Settings > Trusted Publishing > edit the GitHub -# publisher -> workflow = `release.yml` (env stays `crates-io`). +# * crates.io: ordvec and ordvec-manifest > Settings > Trusted Publishing > +# edit each GitHub publisher -> workflow = `release.yml` (env stays +# `crates-io`). # * PyPI: ordvec > publishing > edit the GitHub publisher -> workflow = # `release.yml` (env stays `pypi`). # * GitHub Environments `crates-io` AND `pypi`: @@ -201,7 +202,7 @@ jobs: - name: Create the draft GitHub Release # Draft so notes + the auto-attached artifacts/provenance get assembled # by `release-assets-draft`, the publishes then run gated, and - # `publish-github-release` un-drafts ONLY after both publishes succeed. + # `publish-github-release` un-drafts ONLY after all publishes succeed. # `--verify-tag` ties it to this tag. env: GH_TOKEN: ${{ github.token }} @@ -251,6 +252,48 @@ jobs: path: ordvec.cdx.json if-no-files-found: error + build-manifest-crate: + name: build ordvec-manifest .crate + SBOM + needs: guard + if: needs.guard.outputs.ok == 'true' + runs-on: ubuntu-latest + steps: + - name: Harden the runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable (2026-03-27) + with: + toolchain: stable + - name: Package the manifest crate + # Emits the SLSA-attested sidecar-verifier .crate artifact. The + # lockstep manifest crate depends on the just-tagged `ordvec` version, + # which is not on crates.io until `publish-crate` succeeds, so this + # pre-publish packaging step must skip Cargo's registry-backed verify. + # `publish-manifest-crate` later runs a normal verified repackage after + # `ordvec` is published and compares those bytes to this artifact + # before minting the manifest crate's OIDC token. + run: cargo package -p ordvec-manifest --locked --no-verify + - name: Generate CycloneDX SBOM for the manifest crate + run: | + cargo install cargo-cyclonedx --version 0.5.9 --locked + cargo cyclonedx --manifest-path ordvec-manifest/Cargo.toml --format json + - name: Upload the manifest .crate + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dist-manifest-crate + path: target/package/ordvec-manifest-*.crate + if-no-files-found: error + - name: Upload the manifest crate SBOM + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sbom-manifest-crate + path: ordvec-manifest/ordvec-manifest.cdx.json + if-no-files-found: error + build-wheels: name: wheel ${{ matrix.platform.target }} (${{ matrix.platform.runner }}) needs: guard @@ -515,7 +558,7 @@ jobs: attest: name: GitHub artifact attestation (+ .sigstore.json bundle) - needs: [guard, build-crate, pypi-canonical-dist] + needs: [guard, build-crate, build-manifest-crate, pypi-canonical-dist] if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest permissions: @@ -533,13 +576,18 @@ jobs: with: name: dist-crate path: dist + - name: Collect the manifest crate distributable + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dist-manifest-crate + path: dist - name: Collect the canonical Python dist uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: pypi-canonical-dist path: dist # Fresh release: one consolidated attestation references every subject. - - name: Attest build provenance for crate + canonical wheels + sdist + - name: Attest build provenance for crates + canonical wheels + sdist id: attest_all if: needs.pypi-canonical-dist.outputs.source == 'build' uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0 @@ -550,7 +598,7 @@ jobs: dist/*.tar.gz # Recovery release: PyPI already owns immutable wheel/sdist bytes from a # previous Trusted Publishing upload. Do not claim this run rebuilt them. - - name: Attest build provenance for crate only + - name: Attest build provenance for crates only id: attest_crate if: needs.pypi-canonical-dist.outputs.source == 'pypi' uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0 @@ -569,7 +617,7 @@ jobs: combine-hashes: name: combine artifact digests for SLSA provenance - needs: [guard, build-crate, pypi-canonical-dist] + needs: [guard, build-crate, build-manifest-crate, pypi-canonical-dist] if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest outputs: @@ -584,6 +632,11 @@ jobs: with: name: dist-crate path: dist + - name: Collect the manifest crate distributable + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dist-manifest-crate + path: dist - name: Collect the canonical Python dist if: needs.pypi-canonical-dist.outputs.source == 'build' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -603,7 +656,7 @@ jobs: if [ "$PYPI_SOURCE" = "build" ]; then echo "hashes=$(sha256sum ./*.crate ./*.whl ./*.tar.gz | base64 -w0)" >> "$GITHUB_OUTPUT" elif [ "$PYPI_SOURCE" = "pypi" ]; then - echo "::notice::PyPI dist already exists; SLSA subjects are limited to the crate built by this run." + echo "::notice::PyPI dist already exists; SLSA subjects are limited to the Rust crates built by this run." echo "hashes=$(sha256sum ./*.crate | base64 -w0)" >> "$GITHUB_OUTPUT" else echo "::error::unexpected pypi-canonical-dist source: $PYPI_SOURCE" @@ -648,6 +701,11 @@ jobs: with: name: dist-crate path: dist + - name: Collect the manifest crate distributable + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dist-manifest-crate + path: dist - name: Collect the canonical Python dist uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -676,10 +734,10 @@ jobs: # SOLE Release-asset writer. SBOMs stay build artifacts (registries don't # host them); the GitHub-native bundle (.sigstore.json) and the SLSA # provenance (.intoto.jsonl) ship with the Release. In recovery mode, - # they attest only the crate built by this run; canonical Python files - # are verified against PyPI's immutable hashes instead. + # they attest only the Rust crates built by this run; canonical Python + # files are verified against PyPI's immutable hashes instead. # The Release is left DRAFT — un-drafting happens in - # `publish-github-release` only after BOTH registry publishes succeed, + # `publish-github-release` only after all registry publishes succeed, # so a partial publish never leaves a "public Release with no # registry artifact" half-state. env: @@ -806,6 +864,92 @@ jobs: fi echo "OK: crates.io-served .crate is byte-identical to the SLSA-attested artifact ($A_SHA)." + publish-manifest-crate: + name: publish ordvec-manifest to crates.io + needs: [guard, release-assets-draft, publish-crate] + if: needs.guard.outputs.ok == 'true' + runs-on: ubuntu-latest + environment: crates-io # MANUAL GATE — Required reviewer + permissions: + contents: read + id-token: write # Trusted Publishing (OIDC) + steps: + - name: Harden the runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable (2026-03-27) + with: + toolchain: stable + # Publish the sidecar verifier only after the core `ordvec` crate has + # published and passed its crates.io-served byte-identity readback. The + # manifest crate depends on the same lockstep core version. + - name: Download the attested .crate (from build-manifest-crate) + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dist-manifest-crate + path: ${{ runner.temp }}/attested + - name: Re-package the manifest crate (must be byte-identical to the attested .crate) + run: cargo package -p ordvec-manifest --locked + - name: Verify byte-identity vs the attested .crate + env: + VERSION: ${{ needs.guard.outputs.version }} + run: | + set -euo pipefail + ATTESTED="${RUNNER_TEMP}/attested/ordvec-manifest-${VERSION}.crate" + PACKAGED="target/package/ordvec-manifest-${VERSION}.crate" + [ -f "$ATTESTED" ] || { echo "::error::attested .crate not found at $ATTESTED"; exit 1; } + [ -f "$PACKAGED" ] || { echo "::error::packaged .crate not found at $PACKAGED"; exit 1; } + A_SHA=$(sha256sum "$ATTESTED" | cut -d' ' -f1) + P_SHA=$(sha256sum "$PACKAGED" | cut -d' ' -f1) + echo "attested sha256: $A_SHA" + echo "packaged sha256: $P_SHA" + if [ "$A_SHA" != "$P_SHA" ]; then + echo "::error::byte-identity check failed — the manifest .crate this publish job would upload differs from the SLSA-attested artifact. Refusing to publish." + exit 1 + fi + echo "OK: byte-identity verified ($A_SHA)" + - name: Mint a short-lived crates.io credential (OIDC) + id: auth + uses: rust-lang/crates-io-auth-action@bbd81622f20ce9e2dd9622e3218b975523e45bbe # v1.0.4 + - name: cargo publish + env: + CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} + run: cargo publish -p ordvec-manifest --locked + - name: Post-publish byte-identity (download from crates.io == attested) + env: + VERSION: ${{ needs.guard.outputs.version }} + run: | + set -euo pipefail + ATTESTED="${RUNNER_TEMP}/attested/ordvec-manifest-${VERSION}.crate" + [ -f "$ATTESTED" ] || { echo "::error::attested .crate missing at $ATTESTED"; exit 1; } + A_SHA=$(sha256sum "$ATTESTED" | cut -d' ' -f1) + API_URL="https://crates.io/api/v1/crates/ordvec-manifest/${VERSION}/download" + STATIC_URL="https://static.crates.io/crates/ordvec-manifest/ordvec-manifest-${VERSION}.crate" + CRATES_IO_USER_AGENT="ordvec-release-verify/${VERSION} (https://github.com/Fieldnote-Echo/ordvec)" + PUBLISHED="${RUNNER_TEMP}/published.crate" + for i in 1 2 3 4 5 6 7 8 9 10 11 12; do + rm -f "$PUBLISHED" + if curl -fsSL --user-agent "$CRATES_IO_USER_AGENT" "$API_URL" -o "$PUBLISHED"; then break; fi + if curl -fsSL --user-agent "$CRATES_IO_USER_AGENT" "$STATIC_URL" -o "$PUBLISHED"; then break; fi + rm -f "$PUBLISHED" + echo " waiting for crates.io to serve ordvec-manifest ${VERSION} (${i}/12)..." + if [ "$i" != 12 ]; then sleep 5; fi + done + [ -s "$PUBLISHED" ] \ + || { echo "::error::could not download published .crate from $API_URL or $STATIC_URL after retries"; exit 1; } + P_SHA=$(sha256sum "$PUBLISHED" | cut -d' ' -f1) + echo "attested: $A_SHA" + echo "crates.io-served: $P_SHA" + if [ "$A_SHA" != "$P_SHA" ]; then + echo "::error::PUBLISHED manifest .crate on crates.io is NOT byte-identical to the SLSA-attested artifact ($P_SHA != $A_SHA). The version is now on crates.io (yank-only) but the GitHub Release will NOT un-draft. Investigate immediately." + exit 1 + fi + echo "OK: crates.io-served manifest .crate is byte-identical to the SLSA-attested artifact ($A_SHA)." + publish-pypi: name: publish to PyPI needs: [guard, pypi-canonical-dist, release-assets-draft] @@ -849,8 +993,8 @@ jobs: --dist-dir dist publish-github-release: - name: un-draft the GitHub Release (only after BOTH registry publishes succeed) - needs: [guard, publish-crate, publish-pypi] + name: un-draft the GitHub Release (only after all registry publishes succeed) + needs: [guard, publish-crate, publish-manifest-crate, publish-pypi] if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest permissions: @@ -860,9 +1004,9 @@ jobs: uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 with: egress-policy: audit - # Un-drafting ONLY here, after both registry publishes have succeeded, + # Un-drafting ONLY here, after all registry publishes have succeeded, # is what prevents a public GitHub Release from existing for a version - # that crates.io / PyPI later refused. If either publish job fails or is + # that crates.io / PyPI later refused. If any publish job fails or is # skipped, this job is also skipped and the Release stays DRAFT — the # maintainer investigates / re-runs the failed publish, after which a # re-run of this job un-drafts. diff --git a/README.md b/README.md index 0a58c45..9a8067f 100644 --- a/README.md +++ b/README.md @@ -174,8 +174,8 @@ candidate slices passed to `Search` until the call returns. [`docs/determinism.md`](https://github.com/Fieldnote-Echo/ordvec/blob/main/docs/determinism.md), [`THREAT_MODEL.md`](https://github.com/Fieldnote-Echo/ordvec/blob/main/THREAT_MODEL.md) - **Manifest verifier, C ABI, and Go wrapper:** - `ordvec-manifest` is versioned in lockstep with the core crate and is - package-gated separately; use the GitHub checkout for `ordvec-ffi/`, + `ordvec-manifest` is versioned and published in lockstep with the core crate + through its own package gate; use the GitHub checkout for `ordvec-ffi/`, `ordvec-go/`, and [`docs/c-api.md`](https://github.com/Fieldnote-Echo/ordvec/blob/main/docs/c-api.md). - **Pre-1.0 compatibility policy:** diff --git a/RELEASING.md b/RELEASING.md index d614be2..0d9832b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,17 +1,18 @@ # Releasing `ordvec` -> **Publish is held.** A real `cargo publish` / PyPI publish happens only on +> **Publish is held.** A real crates.io / PyPI publish happens only on > the maintainer's explicit approval. CI never publishes — the unified release > pipeline builds, attests, and attaches everything to the GitHub Release > automatically on a tag push, then **waits at the `crates-io` and `pypi` -> environment gates** for a required-reviewer approval before either registry +> environment gates** for a required-reviewer approval before any registry > push. -`ordvec` (the Rust crate) and `ordvec` on PyPI (the PyO3 wheel built from -`ordvec-python/`) are released by **pushing a `vMAJOR.MINOR.PATCH` tag** to a -commit on `main`. The release workflow handles build, canonical Python artifact -selection, attestation, SLSA provenance, Release-asset attach, and un-draft -automatically; only the two registry gates are manual. +`ordvec` (the Rust crate), `ordvec-manifest` (the lockstep manifest verifier +crate), and `ordvec` on PyPI (the PyO3 wheel built from `ordvec-python/`) are +released by **pushing a `vMAJOR.MINOR.PATCH` tag** to a commit on `main`. The +release workflow handles build, canonical Python artifact selection, +attestation, SLSA provenance, Release-asset attach, and un-draft automatically; +only the registry environment approvals are manual. ## Release pipeline controls @@ -24,7 +25,7 @@ The unified `release.yml`: `main` for the tagged SHA — `ci.yml`, `python.yml`, `fuzz.yml`, `codeql.yml` (a *successful* run for that exact SHA on `main`); - publishes via **OIDC trusted publishing** (no long-lived crates.io / PyPI - tokens in the repo); + tokens in the repo) for both Rust crates and the Python distribution; - canonicalizes the Python dist before attestation and release upload: for a new PyPI version it uses the current run's wheels/sdist; if PyPI already owns that immutable version during recovery, it downloads the exact PyPI-served @@ -35,31 +36,41 @@ The unified `release.yml`: **before** the gated publishes — a failed attestation fails the release closed, so nothing ships without provenance recorded. In recovery mode where PyPI files already exist, the GitHub/SLSA subjects are deliberately limited - to the crate built by the current run; the Python files are verified immutable - PyPI bytes from the earlier Trusted Publishing upload, not falsely claimed as - rebuilt by the recovery run; -- stages the **`.crate`, canonical wheels, canonical sdist, `*.sigstore.json` bundle, and - `*.intoto.jsonl` provenance** on the GitHub Release while it is still **a + to the Rust crates built by the current run; the Python files are verified + immutable PyPI bytes from the earlier Trusted Publishing upload, not falsely + claimed as rebuilt by the recovery run; +- stages the **`.crate` files, canonical wheels, canonical sdist, + `*.sigstore.json` bundle, and `*.intoto.jsonl` provenance** on the GitHub Release while it is still **a DRAFT** (`release-assets-draft` is the sole Release-asset writer — no manual attach, which is what v0.2.0's manual step missed); -- proves **byte-identity** in `publish-crate` on both sides of `cargo publish`: +- proves **byte-identity** in `publish-crate` and `publish-manifest-crate` on + both sides of `cargo publish`: 1. **pre-publish gate** — downloads the SLSA-attested `.crate` artifact, re-packages with `--locked`, and `sha256`-compares before minting the crates.io OIDC token. Defends against toolchain drift / deterministic- packaging regression; if they differ, fails closed **before** the token is minted (nothing reaches crates.io); 2. **post-publish empirical proof** — downloads the just-published `.crate` - from `crates.io/api/v1/crates/ordvec//download` and `sha256`-compares - to the attested artifact. `cargo publish` runs its own internal - packaging step the pre-publish gate cannot inspect; this is the only - check that proves the bytes crates.io actually serves equal the SLSA- - attested bytes. A mismatch fails closed, so `publish-github-release` - never un-drafts the Release (the version is then yank-only on - crates.io, but the failure is loudly observable); -- **un-drafts the GitHub Release ONLY after BOTH `publish-crate` AND - `publish-pypi` succeed** (`publish-github-release` is the sole un-draft - point). If either publish fails or is skipped, the Release stays DRAFT — no - public Release ever exists for a version the registries refused; + from `crates.io/api/v1/crates///download` and + `sha256`-compares to the attested artifact. `cargo publish` runs its own + internal packaging step the pre-publish gate cannot inspect; this is the + only check that proves the bytes crates.io actually serves equal the + SLSA-attested bytes. A mismatch fails closed, so + `publish-github-release` never un-drafts the Release (the version is + then yank-only on crates.io, but the failure is loudly observable); +- publishes `ordvec-manifest` only after the lockstep `ordvec` crate has + published and passed its crates.io-served byte-identity readback. The + pre-publish manifest `.crate` artifact is created with `cargo package + --no-verify` because Cargo cannot verify a lockstep dependency version before + `ordvec` exists on crates.io; `publish-manifest-crate` then runs the normal + verified `cargo package -p ordvec-manifest --locked` after `ordvec` is + published and byte-compares that output to the attested artifact before + minting its own OIDC token; +- **un-drafts the GitHub Release ONLY after `publish-crate`, + `publish-manifest-crate`, AND `publish-pypi` succeed** + (`publish-github-release` is the sole un-draft point). If any publish fails + or is skipped, the Release stays DRAFT — no public Release ever exists for a + version the registries refused; - pins every third-party action by **commit SHA** (the one mandated exception is the SLSA reusable workflow, tag-pinned per SLSA's trust model), sets `persist-credentials: false`, and defaults to `permissions: contents: read`. @@ -93,11 +104,15 @@ the GitHub Release. ### Trusted-publisher configuration (one-time, in the registries) The crates.io and PyPI Trusted Publisher records must point at this workflow -filename. Until either is updated, the corresponding gated publish fails +filename. Until a record is updated, the corresponding gated publish fails **closed** at the OIDC exchange (no risk of a bad publish; just a failed run). - **crates.io** → `ordvec` → Settings → Trusted Publishing → GitHub publisher: `workflow = release.yml`, `environment = crates-io`. +- **crates.io** → `ordvec-manifest` → Settings → Trusted Publishing → GitHub + publisher: `workflow = release.yml`, `environment = crates-io`. If crates.io + requires an initial owner bootstrap before a new crate's Trusted Publisher can + be configured, do that explicit maintainer-approved bootstrap before tagging. - **PyPI** → `ordvec` → Publishing → GitHub publisher: `workflow = release.yml`, `environment = pypi`. @@ -167,8 +182,9 @@ filename. Until either is updated, the corresponding gated publish fails This verifies the GitHub Environments still require the expected reviewer and accept only the stable release tag pattern. Separately verify the - registry Trusted Publisher records by hand: crates.io must point to - `release.yml` / `crates-io`, and PyPI must point to `release.yml` / `pypi`. + registry Trusted Publisher records by hand: crates.io must point both + `ordvec` and `ordvec-manifest` to `release.yml` / `crates-io`, and PyPI must + point `ordvec` to `release.yml` / `pypi`. 6. Get the maintainer's explicit go to publish. 7. Push the version tag from `main` (signed): @@ -177,30 +193,32 @@ filename. Until either is updated, the corresponding gated publish fails git push origin vX.Y.Z ``` - `release.yml` triggers automatically. It builds the `.crate`, wheels, and - sdist; selects the canonical Python dist (current build for a new PyPI - version, verified PyPI bytes for an existing immutable version); attests the - files this run can honestly attest (GitHub attestation store + + `release.yml` triggers automatically. It builds the two `.crate` files, + wheels, and sdist; selects the canonical Python dist (current build for a + new PyPI version, verified PyPI bytes for an existing immutable version); + attests the files this run can honestly attest (GitHub attestation store + `*.sigstore.json`); generates the SLSA `*.intoto.jsonl`; and stages every artifact, the attestation bundle, and the provenance on the GitHub Release - — **as a DRAFT**. It then pauses at the two registry environment gates. -8. **Approve the two publish environments** when they pause in the Actions UI - (one for `crates-io`, one for `pypi`). The required-reviewer approval is - what authorises the registry push. - - `publish-crate` first sha256-compares its repackaged `.crate` to the - SLSA-attested artifact — if they diverge (toolchain drift, etc.) the job - fails closed BEFORE the OIDC token is minted, so nothing reaches - crates.io. Re-run / investigate. - - Once **both** registry gates succeed, `publish-github-release` un-drafts - the GitHub Release automatically. If one gate fails, the Release stays - DRAFT — investigate and re-run from a fixed workflow rather than approving - the other registry into another partial state. + — **as a DRAFT**. It then pauses at the registry environment gates. +8. **Approve each publish environment pause** in the Actions UI. There are + three registry publish jobs: `publish-crate`, `publish-manifest-crate`, and + `publish-pypi`. The two crates.io jobs use the same `crates-io` environment + and may require separate approvals; PyPI uses the `pypi` environment. + Required-reviewer approval is what authorises each registry push. + - `publish-crate` and `publish-manifest-crate` first sha256-compare their + repackaged `.crate` to the SLSA-attested artifact — if either diverges + (toolchain drift, etc.) the job fails closed BEFORE the OIDC token is + minted, so nothing reaches crates.io. Re-run / investigate. + - Once **all** registry publish jobs succeed, `publish-github-release` + un-drafts the GitHub Release automatically. If one gate fails, the Release + stays DRAFT — investigate and re-run from a fixed workflow rather than + approving another registry into a partial state. - `publish-pypi` either uploads the fresh canonical dist or, if PyPI already serves that version, skips upload and verifies the existing files. In both modes it compares every PyPI-served wheel/sdist SHA-256 digest against the canonical `dist/` files before the GitHub Release can un-draft. 9. Verify each published artifact and its provenance: - - crates.io / docs.rs; + - crates.io / docs.rs for `ordvec` and `ordvec-manifest`; - PyPI (confirm the post-publish hash-verification log, optionally `pip download ordvec==X.Y.Z` and inspect, plus check the PEP 740 attestation at `GET https://pypi.org/integrity/ordvec/X.Y.Z//provenance`); @@ -212,6 +230,6 @@ filename. Until either is updated, the corresponding gated publish fails ## Coordinated release note -The crate publish, the PyPI wheel, and the paper's Zenodo deposit are +The Rust crate publishes, the PyPI wheel, and the paper's Zenodo deposit are coordinated (the paper consumes the bindings for a final cold-repro run). Do not ship one leg in isolation without the maintainer's go. diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index aac32c0..aefe527 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -307,14 +307,16 @@ trust model requires be pinned by version *tag*); `persist-credentials: false` on every checkout; `permissions: contents: read` default. The **release workflow** (`release.yml`) is tag-triggered with a strict-SemVer guard; build, GitHub attestation, SLSA provenance, Release-asset attach, and un-draft all -run automatically, while the **`crates.io`** and **`pypi`** publishes are -gated behind GitHub Environments with **Required reviewers** (the only manual -step). It runs a `require-ci-green` gate against `main`, publishes via **OIDC -trusted publishing** (no long-lived registry tokens), and emits **SLSA build +run automatically, while the two **`crates.io`** publish jobs (`ordvec` first, +then lockstep `ordvec-manifest`) and the **`pypi`** publish job are gated +behind GitHub Environments with **Required reviewers** (the only manual step). +It runs a `require-ci-green` gate against `main`, publishes via **OIDC trusted +publishing** (no long-lived registry tokens), and emits **SLSA build provenance** (`actions/attest-build-provenance` + a `slsa-github-generator` `*.intoto.jsonl` attached to the GitHub Release) **before** publish — a failed -attestation fails the release closed. PyPI additionally gets **PEP 740** -attestations via Trusted Publishing. +attestation fails the release closed. Each Rust publish job proves pre- and +post-publish crates.io byte identity against the attested `.crate`; PyPI +additionally gets **PEP 740** attestations via Trusted Publishing. **Static / supply-chain analysis:** **CodeQL** scans Rust, Python, and Actions (no-build databases); **OpenSSF Scorecard** publishes SARIF to code scanning @@ -336,7 +338,7 @@ branch-only allowlist would deadlock publishing — see RELEASING.md). The push-event CI run on `main`, and `main` itself is branch-protected (PR review, no force-push) — so a release cannot be cut from an unmerged or attacker branch, and no publish runs without an explicit human approval. The remaining residual is *maintainer-account compromise*: a -single owner both cuts the release tag and approves both publishes, so account takeover (or social +single owner both cuts the release tag and approves all publishes, so account takeover (or social engineering) is not caught by a second human. *Mitigations:* strong 2FA / passkeys on the maintainer account; recruiting a **second owner/maintainer** (also an open OpenSSF Best-Practices item) — which would additionally make a @@ -517,10 +519,11 @@ x86_64/aarch64 unsafe paths (SIMD-004). in the Rust and Python API docs (THREAT-QUERY-001). **Later (not release blockers):** a second maintainer/owner (then a release -wait timer becomes meaningful); an optional sidecar index verifier -(`ordvec verify` / external HMAC/BLAKE3 manifest) if a deployment requires -tamper-evidence (DESER-002); a `safe_copy=True` FFI isolation option -(FFI-001); layering ASAN onto the Intel SDE AVX-512 leg. +wait timer becomes meaningful); stronger deployment-specific manifest +trust-policy UX such as external signatures/HMACs if a deployment requires +tamper-evidence beyond `ordvec-manifest`'s hash-bound sidecar verification +(DESER-002); a `safe_copy=True` FFI isolation option (FFI-001); layering ASAN +onto the Intel SDE AVX-512 leg. --- diff --git a/tests/release_publish_invariants.py b/tests/release_publish_invariants.py index fb5c987..9ae25e1 100644 --- a/tests/release_publish_invariants.py +++ b/tests/release_publish_invariants.py @@ -143,15 +143,15 @@ def shell_logical_lines(script: str) -> list[str]: line = raw_line.strip() if not line or line.startswith("#"): continue - if line.endswith("\\"): - current += line[:-1].strip() + " " - continue - current += line - if current: - lines.append(current.strip()) - current = "" + continued = line.endswith("\\") + if continued: + line = line[:-1].strip() + current = f"{current} {line}".strip() if current else line + if not continued: + lines.append(current) + current = "" if current: - lines.append(current.strip()) + lines.append(current) return lines @@ -174,6 +174,31 @@ def shell_curl_commands(script: str) -> list[list[str]]: return commands +def has_cargo_package_arg(words: list[str], package: str) -> bool: + for index, word in enumerate(words): + if word in {"-p", "--package"}: + if index + 1 < len(words) and words[index + 1] == package: + return True + elif word.startswith("--package=") and word.split("=", 1)[1] == package: + return True + elif word.startswith("-p") and word != "-p" and word[2:] == package: + return True + return False + + +def has_cargo_command(run: str, subcommand: str, package: str) -> bool: + for line in shell_logical_lines(run): + try: + words = shlex.split(line) + except ValueError: + continue + if len(words) < 3 or words[0] != "cargo" or words[1] != subcommand: + continue + if "--locked" in words and has_cargo_package_arg(words[2:], package): + return True + return False + + def has_shell_arg(words: list[str], values: set[str]) -> bool: return any(word in values for word in words) @@ -361,33 +386,46 @@ def check_publish_pypi(workflow: dict[str, Any], path: str) -> None: fail(f"{path}: {label} must not place non-canonical artifacts in dist") -def check_publish_crate(workflow: dict[str, Any], path: str) -> None: +def check_publish_crate_job( + workflow: dict[str, Any], path: str, job_name: str, package: str, artifact_name: str +) -> None: jobs = mapping(workflow.get("jobs"), f"{path}: jobs") - job = mapping(jobs.get("publish-crate"), f"{path}: jobs.publish-crate") - steps = sequence(job.get("steps"), f"{path}: jobs.publish-crate.steps") + job = mapping(jobs.get(job_name), f"{path}: jobs.{job_name}") + steps = sequence(job.get("steps"), f"{path}: jobs.{job_name}.steps") crate_downloads: list[tuple[int, dict[str, Any], dict[str, Any]]] = [] + package_runs: list[str] = [] + publish_runs: list[str] = [] for index, raw_step in enumerate(steps): - step = mapping(raw_step, f"{path}: jobs.publish-crate.steps[{index}]") - if action_name(step) != "actions/download-artifact": - continue - with_block = step.get("with", {}) - with_map = mapping(with_block, f"{path}: {step_label(index, step)} with") - if with_map.get("name") == "dist-crate": - crate_downloads.append((index, step, with_map)) + step = mapping(raw_step, f"{path}: jobs.{job_name}.steps[{index}]") + run = step.get("run") + if isinstance(run, str): + if has_cargo_command(run, "package", package): + package_runs.append(run) + if has_cargo_command(run, "publish", package): + publish_runs.append(run) + if action_name(step) == "actions/download-artifact": + with_block = step.get("with", {}) + with_map = mapping(with_block, f"{path}: {step_label(index, step)} with") + if with_map.get("name") == artifact_name: + crate_downloads.append((index, step, with_map)) if len(crate_downloads) != 1: - fail(f"{path}: publish-crate must download exactly one dist-crate artifact") + fail(f"{path}: {job_name} must download exactly one {artifact_name} artifact") index, step, with_map = crate_downloads[0] label = step_label(index, step) artifact_path = norm_path(with_map.get("path")) if artifact_path != "${{ runner.temp }}/attested": fail( - f"{path}: {label} downloads dist-crate to {artifact_path or 'the default path'!r}; " + f"{path}: {label} downloads {artifact_name} to {artifact_path or 'the default path'!r}; " "it must use ${{ runner.temp }}/attested so cargo package sees a clean checkout" ) + if len(package_runs) != 1: + fail(f"{path}: {job_name} must run exactly one `cargo package -p {package} --locked`") + if len(publish_runs) != 1: + fail(f"{path}: {job_name} must run exactly one `cargo publish -p {package} --locked`") verify_step_names = { "Verify byte-identity vs the attested .crate", @@ -396,54 +434,72 @@ def check_publish_crate(workflow: dict[str, Any], path: str) -> None: verify_steps: list[dict[str, Any]] = [] found_names: set[str] = set() for index, raw_step in enumerate(steps): - step = mapping(raw_step, f"{path}: jobs.publish-crate.steps[{index}]") + step = mapping(raw_step, f"{path}: jobs.{job_name}.steps[{index}]") name = step.get("name") if name in verify_step_names: verify_steps.append(step) found_names.add(name) if found_names != verify_step_names: - fail(f"{path}: publish-crate must have both attested .crate verification steps") + fail(f"{path}: {job_name} must have both attested .crate verification steps") + attested_path = f"${{RUNNER_TEMP}}/attested/{package}-${{VERSION}}.crate" for step in verify_steps: name = step.get("name") run = step.get("run") if not isinstance(run, str): - fail(f"{path}: publish-crate step {name!r} must be a run step") - if "${RUNNER_TEMP}/attested/ordvec-${VERSION}.crate" not in run: + fail(f"{path}: {job_name} step {name!r} must be a run step") + if attested_path not in run: fail( - f"{path}: publish-crate step {name!r} must read the attested .crate " - "from ${RUNNER_TEMP}/attested" + f"{path}: {job_name} step {name!r} must read the attested .crate " + f"from {attested_path}" ) if name == "Post-publish byte-identity (download from crates.io == attested)": if "/tmp/published.crate" in run: - fail(f"{path}: publish-crate post-publish readback must not write to /tmp") + fail(f"{path}: {job_name} post-publish readback must not write to /tmp") if not ( "${RUNNER_TEMP}/published.crate" in run or "$RUNNER_TEMP/published.crate" in run ): - fail(f"{path}: publish-crate post-publish readback must write under ${{RUNNER_TEMP}}") + fail(f"{path}: {job_name} post-publish readback must write under ${{RUNNER_TEMP}}") version = r"\$(?:\{VERSION\}|VERSION)" if not has_assignment( - run, "API_URL", rf"https://crates\.io/api/v1/crates/ordvec/{version}/download" + run, "API_URL", rf"https://crates\.io/api/v1/crates/{package}/{version}/download" ): - fail(f"{path}: publish-crate post-publish readback must define the crates.io API URL") + fail(f"{path}: {job_name} post-publish readback must define the crates.io API URL") if not has_assignment( - run, "STATIC_URL", rf"https://static\.crates\.io/crates/ordvec/ordvec-{version}\.crate" + run, + "STATIC_URL", + rf"https://static\.crates\.io/crates/{package}/{package}-{version}\.crate", ): - fail(f"{path}: publish-crate post-publish readback must define the static.crates.io fallback") + fail(f"{path}: {job_name} post-publish readback must define the static.crates.io fallback") curl_commands = shell_curl_commands(run) if not any(readback_curl_uses(words, "API_URL") for words in curl_commands): fail( - f"{path}: publish-crate post-publish readback must curl $API_URL " + f"{path}: {job_name} post-publish readback must curl $API_URL " "with CRATES_IO_USER_AGENT into $PUBLISHED" ) if not any(readback_curl_uses(words, "STATIC_URL") for words in curl_commands): fail( - f"{path}: publish-crate post-publish readback must curl $STATIC_URL " + f"{path}: {job_name} post-publish readback must curl $STATIC_URL " "with CRATES_IO_USER_AGENT into $PUBLISHED" ) +def check_publish_crates(workflow: dict[str, Any], path: str) -> None: + jobs = mapping(workflow.get("jobs"), f"{path}: jobs") + manifest_job = mapping(jobs.get("publish-manifest-crate"), f"{path}: jobs.publish-manifest-crate") + if not has_need(manifest_job, "publish-crate"): + fail(f"{path}: publish-manifest-crate must need publish-crate so ordvec publishes first") + check_publish_crate_job(workflow, path, "publish-crate", "ordvec", "dist-crate") + check_publish_crate_job( + workflow, + path, + "publish-manifest-crate", + "ordvec-manifest", + "dist-manifest-crate", + ) + + def check_sde_setup_action(path: str) -> None: action = load_workflow(path) runs = mapping(action.get("runs"), f"{path}: runs") @@ -585,7 +641,7 @@ def main() -> None: ) check_aarch64_smoke_selector(workflow, WORKFLOW_PATH) check_pypi_canonical_dist(workflow, WORKFLOW_PATH) - check_publish_crate(workflow, WORKFLOW_PATH) + check_publish_crates(workflow, WORKFLOW_PATH) check_publish_pypi(workflow, WORKFLOW_PATH) check_sde_cache_invariants() diff --git a/tests/release_publish_invariants.sh b/tests/release_publish_invariants.sh index 7a99865..2f8712c 100755 --- a/tests/release_publish_invariants.sh +++ b/tests/release_publish_invariants.sh @@ -10,10 +10,10 @@ set -euo pipefail fail() { echo "::error::release-publish invariant violated: $*"; exit 1; } -# Both generated SBOMs must be gitignored. A tracked/untracked *.cdx.json +# Generated SBOMs must be gitignored. A tracked/untracked *.cdx.json # makes `cargo publish` refuse the dirty tree and would otherwise bundle # the SBOM into the .crate. -for f in ordvec.cdx.json ordvec-python/ordvec-python.cdx.json; do +for f in ordvec.cdx.json ordvec-manifest/ordvec-manifest.cdx.json ordvec-python/ordvec-python.cdx.json; do git check-ignore -q -- "$f" || fail "$f is not gitignored (it is a generated SBOM artifact)" done diff --git a/tests/release_signed_release_invariants.sh b/tests/release_signed_release_invariants.sh index 7a42909..b4facd2 100755 --- a/tests/release_signed_release_invariants.sh +++ b/tests/release_signed_release_invariants.sh @@ -7,23 +7,24 @@ # unsigned releases may keep the score below 10 temporarily. The same graph # keeps the build-attest-publish chain honest: # -# build-{crate,wheels,sdist} (raw artifacts) +# build-{crate,manifest-crate,wheels,sdist} (raw artifacts) # | # +-> pypi-canonical-dist (current build, or verified immutable PyPI files) # | # +-> attest (id-token + attestations + .sigstore.json; -# | crate-only when PyPI files already exist) +# | Rust-crates-only when PyPI files already exist) # +-> provenance (slsa-github-generator @vX.Y.Z, .intoto.jsonl; -# | crate-only when PyPI files already exist) +# | Rust-crates-only when PyPI files already exist) # | # v # release-assets-draft (uploads .crate/canonical .whl/.tar.gz/.sigstore.json/.intoto.jsonl to DRAFT release) # | -# +--> publish-crate (byte-identity check vs attested .crate, then cargo publish) -# +--> publish-pypi (Trusted Publishing, or existing-file verification) +# +--> publish-crate (byte-identity check vs attested .crate, then cargo publish) +# +--> publish-manifest-crate (after publish-crate; same byte-identity proof) +# +--> publish-pypi (Trusted Publishing, or existing-file verification) # | # v -# publish-github-release (un-draft, ONLY after both publishes succeed) +# publish-github-release (un-draft, ONLY after all registry publishes succeed) # # A regression in any of these edges (a future commit drops a needs:, renames # the provenance file, lets release-assets-draft un-draft itself, forgets the @@ -70,6 +71,24 @@ require_job_line() { printf '%s\n' "$line" } +job_downloads_artifact_to_path() { + local jobname="$1" artifact="$2" expected_path="$3" + job_body "$jobname" | awk -v artifact="$artifact" -v expected_path="$expected_path" ' + function flush_step() { + if (has_download && has_name && has_path) { + found = 1 + } + has_download = has_name = has_path = 0 + } + + /^[[:space:]]+-[[:space:]]/ { flush_step() } + $0 ~ "uses:[[:space:]]*actions/download-artifact" { has_download = 1 } + $0 ~ "^[[:space:]]+name:[[:space:]]*" artifact "[[:space:]]*$" { has_name = 1 } + $0 ~ "^[[:space:]]+path:[[:space:]]*" expected_path "[[:space:]]*$" { has_path = 1 } + END { flush_step(); exit found ? 0 : 1 } + ' +} + # ---------------------------------------------------------------------- # (1) release-assets-draft needs attest + provenance + require-ci-green + notes # + exact linux/aarch64 wheel smoke @@ -90,6 +109,10 @@ for ext in '\.crate' '\.whl' '\.tar\.gz' '\.sigstore\.json' '\.intoto\.jsonl'; d done printf '%s\n' "$body_draft" | grep -qE 'name:[[:space:]]*pypi-canonical-dist' \ || fail "release-assets-draft must upload canonical Python dist, not raw rebuilt wheel/sdist artifacts" +job_downloads_artifact_to_path release-assets-draft dist-crate dist \ + || fail "release-assets-draft must download the core dist-crate artifact into dist" +job_downloads_artifact_to_path release-assets-draft dist-manifest-crate dist \ + || fail "release-assets-draft must download the manifest dist-manifest-crate artifact into dist" printf '%s\n' "$body_draft" | grep -qE "$github_repo_env_re" \ || fail "release-assets-draft must set \`GH_REPO: \${{ github.repository }}\` (no checkout, so gh release upload needs explicit repo context)" @@ -99,7 +122,7 @@ printf '%s\n' "$body_draft" | grep -qE "$github_repo_env_re" \ # publish failure mode). # ---------------------------------------------------------------------- if printf '%s\n' "$body_draft" | grep -qE 'gh release edit.*--draft=false'; then - fail "release-assets-draft must NOT un-draft the Release (un-drafting belongs in publish-github-release, after both publishes succeed)" + fail "release-assets-draft must NOT un-draft the Release (un-drafting belongs in publish-github-release, after all registry publishes succeed)" fi # ---------------------------------------------------------------------- @@ -132,11 +155,25 @@ printf '%s\n' "$att" | grep -qE '^[[:space:]]+id-token:[[:space:]]*write' \ || fail "attest job must grant \`id-token: write\` (Sigstore OIDC signing cert)" printf '%s\n' "$att" | grep -qE '^[[:space:]]+attestations:[[:space:]]*write' \ || fail "attest job must grant \`attestations: write\` (persist to the GitHub attestation store)" +job_needs attest build-manifest-crate \ + || fail "attest must \`needs: build-manifest-crate\` so the manifest .crate is an attestation subject" +job_downloads_artifact_to_path attest dist-manifest-crate dist \ + || fail "attest must download the dist-manifest-crate artifact into dist" + +comb="$(job_body combine-hashes)" +job_needs combine-hashes build-manifest-crate \ + || fail "combine-hashes must \`needs: build-manifest-crate\` so the manifest .crate is a SLSA subject" +job_downloads_artifact_to_path combine-hashes dist-manifest-crate dist \ + || fail "combine-hashes must download the dist-manifest-crate artifact into dist" + +build_manifest="$(job_body build-manifest-crate)" +printf '%s\n' "$build_manifest" | grep -qE 'cargo[[:space:]]+package[[:space:]]+-p[[:space:]]+ordvec-manifest[[:space:]]+--locked[[:space:]]+--no-verify' \ + || fail "build-manifest-crate must package with --no-verify before the lockstep core crate exists on crates.io" # ---------------------------------------------------------------------- -# (8) Both publish jobs grant id-token: write AND need release-assets-draft. +# (8) Registry publish jobs grant id-token: write AND need release-assets-draft. # ---------------------------------------------------------------------- -for pub in publish-crate publish-pypi; do +for pub in publish-crate publish-manifest-crate publish-pypi; do body="$(job_body "$pub")" printf '%s\n' "$body" | grep -qE '^[[:space:]]+id-token:[[:space:]]*write' \ || fail "$pub must grant \`id-token: write\` (Trusted Publishing OIDC)" @@ -145,7 +182,7 @@ for pub in publish-crate publish-pypi; do done # ---------------------------------------------------------------------- -# (9) publish-crate proves byte-identity vs the attested .crate on BOTH +# (9) Rust crate publish jobs prove byte-identity vs the attested .crate on BOTH # sides of `cargo publish`: # (9a) pre-publish: download the attested .crate, re-run `cargo package`, # sha256-compare. Fail-closed BEFORE the OIDC token is minted. @@ -155,28 +192,38 @@ done # cannot inspect — this is the empirical proof that the bytes # actually served by crates.io match the SLSA-attested artifact. # ---------------------------------------------------------------------- -pcb="$(job_body publish-crate)" -printf '%s\n' "$pcb" | grep -qE 'uses:[[:space:]]*actions/download-artifact' \ - || fail "publish-crate must download the attested dist-crate artifact (byte-identity gate)" -printf '%s\n' "$pcb" | grep -qE 'name:[[:space:]]*dist-crate' \ - || fail "publish-crate must download the artifact named \`dist-crate\` (the attested .crate)" -printf '%s\n' "$pcb" | grep -qE 'cargo[[:space:]]+package[[:space:]]+-p[[:space:]]+ordvec[[:space:]]+--locked' \ - || fail "publish-crate must re-run \`cargo package -p ordvec --locked\` so it can sha256-compare to the attested .crate (pre-publish gate)" -printf '%s\n' "$pcb" | grep -qE 'sha256sum' \ - || fail "publish-crate must sha256sum-compare the repackaged .crate vs the attested .crate before publishing" -printf '%s\n' "$pcb" | grep -qE 'crates\.io/api/v1/crates/ordvec|static\.crates\.io/crates/ordvec' \ - || fail "publish-crate must download the just-published .crate from crates.io after \`cargo publish\` (post-publish byte-identity proof; pre-publish alone cannot inspect cargo publish's internal packaging)" - -pre_line="$(require_job_line publish-crate '^[[:space:]]+- name:[[:space:]]*Verify byte-identity vs the attested \.crate' 'a pre-publish byte-identity verification step')" -oidc_line="$(require_job_line publish-crate '^[[:space:]]+- name:[[:space:]]*Mint a short-lived crates\.io credential' 'an OIDC credential mint step')" -publish_line="$(require_job_line publish-crate '^[[:space:]]+- name:[[:space:]]*cargo publish' 'a cargo publish step')" -post_line="$(require_job_line publish-crate '^[[:space:]]+- name:[[:space:]]*Post-publish byte-identity' 'a post-publish crates.io byte-identity step')" -[ "$pre_line" -lt "$oidc_line" ] \ - || fail "publish-crate must verify byte-identity BEFORE minting the crates.io OIDC credential" -[ "$oidc_line" -lt "$publish_line" ] \ - || fail "publish-crate must mint the crates.io OIDC credential BEFORE \`cargo publish\`" -[ "$publish_line" -lt "$post_line" ] \ - || fail "publish-crate must run the crates.io post-publish download/compare AFTER \`cargo publish\`" +check_crate_publish_job() { + local jobname="$1" package="$2" artifact="$3" body pre_line oidc_line publish_line post_line + body="$(job_body "$jobname")" + printf '%s\n' "$body" | grep -qE 'uses:[[:space:]]*actions/download-artifact' \ + || fail "$jobname must download the attested $artifact artifact (byte-identity gate)" + printf '%s\n' "$body" | grep -qE "name:[[:space:]]*${artifact}" \ + || fail "$jobname must download the artifact named \`$artifact\` (the attested .crate)" + printf '%s\n' "$body" | grep -qE "cargo[[:space:]]+package[[:space:]]+-p[[:space:]]+${package}[[:space:]]+--locked" \ + || fail "$jobname must re-run \`cargo package -p $package --locked\` so it can sha256-compare to the attested .crate (pre-publish gate)" + printf '%s\n' "$body" | grep -qE "cargo[[:space:]]+publish[[:space:]]+-p[[:space:]]+${package}[[:space:]]+--locked" \ + || fail "$jobname must run \`cargo publish -p $package --locked\`" + printf '%s\n' "$body" | grep -qE 'sha256sum' \ + || fail "$jobname must sha256sum-compare the repackaged .crate vs the attested .crate before publishing" + printf '%s\n' "$body" | grep -qE "crates\.io/api/v1/crates/${package}|static\.crates\.io/crates/${package}" \ + || fail "$jobname must download the just-published .crate from crates.io after \`cargo publish\` (post-publish byte-identity proof; pre-publish alone cannot inspect cargo publish's internal packaging)" + + pre_line="$(require_job_line "$jobname" '^[[:space:]]+- name:[[:space:]]*Verify byte-identity vs the attested \.crate' 'a pre-publish byte-identity verification step')" + oidc_line="$(require_job_line "$jobname" '^[[:space:]]+- name:[[:space:]]*Mint a short-lived crates\.io credential' 'an OIDC credential mint step')" + publish_line="$(require_job_line "$jobname" '^[[:space:]]+- name:[[:space:]]*cargo publish' 'a cargo publish step')" + post_line="$(require_job_line "$jobname" '^[[:space:]]+- name:[[:space:]]*Post-publish byte-identity' 'a post-publish crates.io byte-identity step')" + [ "$pre_line" -lt "$oidc_line" ] \ + || fail "$jobname must verify byte-identity BEFORE minting the crates.io OIDC credential" + [ "$oidc_line" -lt "$publish_line" ] \ + || fail "$jobname must mint the crates.io OIDC credential BEFORE \`cargo publish\`" + [ "$publish_line" -lt "$post_line" ] \ + || fail "$jobname must run the crates.io post-publish download/compare AFTER \`cargo publish\`" +} + +check_crate_publish_job publish-crate ordvec dist-crate +check_crate_publish_job publish-manifest-crate ordvec-manifest dist-manifest-crate +job_needs publish-manifest-crate publish-crate \ + || fail "publish-manifest-crate must \`needs: publish-crate\` so the lockstep core crate publishes first" pcd="$(job_body pypi-canonical-dist)" printf '%s\n' "$pcd" | grep -qE 'release_pypi_canonical_dist\.py canonicalize' \ @@ -195,11 +242,11 @@ grep -q 'pypi.org/pypi' tests/release_pypi_canonical_dist.py \ || fail "release_pypi_canonical_dist.py must query PyPI for served file hashes" # ---------------------------------------------------------------------- -# (10) publish-github-release un-drafts ONLY AFTER both registry publishes succeed. +# (10) publish-github-release un-drafts ONLY AFTER all registry publishes succeed. # ---------------------------------------------------------------------- -for dep in publish-crate publish-pypi; do +for dep in publish-crate publish-manifest-crate publish-pypi; do job_needs publish-github-release "$dep" \ - || fail "publish-github-release must \`needs: $dep\` (un-draft only after BOTH registry publishes succeed)" + || fail "publish-github-release must \`needs: $dep\` (un-draft only after all registry publishes succeed)" done unp="$(job_body publish-github-release)" printf '%s\n' "$unp" | grep -qE 'gh release edit.*--draft=false' \