diff --git a/.github/workflows/mac-build.yml b/.github/workflows/mac-build.yml index 5840df273..c6e879357 100644 --- a/.github/workflows/mac-build.yml +++ b/.github/workflows/mac-build.yml @@ -86,6 +86,23 @@ jobs: node-version: 26 - run: npm install --no-audit --no-fund --loglevel=error + # The typosquat corpus is a SEPARATE artifact from the primer, and build.rs + # treats it as mandatory under AUBE_REQUIRE_PRIMER=1 (set on the build below). + # Having node is not enough: build.rs regenerates the primer by shelling out + # to generate-primer.mjs WITHOUT --popular-names-out, so nothing else on this + # runner ever writes the corpus and the build dies in write_popular_names_blob + # with "popular package names are required". `--top 1` is the cheapest way to + # reach the script's popular-names branch; the primer output is discarded + # outside data/ so build.rs still regenerates the real one itself. + - name: Generate typosquat popularity corpus + run: | + set -euo pipefail + cd vendor/aube + node scripts/generate-primer.mjs \ + --top 1 \ + --out "$RUNNER_TEMP/primer-discard.json" \ + --popular-names-out crates/aube-resolver/data/popular-top100000-v1.json + # Belt and braces against the silent-primer path above. - name: Build the CLI env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3934958e2..00c58debb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -345,18 +345,54 @@ jobs: fi echo "primer $out is $bytes bytes" echo "path=vendor/aube/$out" >> "$GITHUB_OUTPUT" - # Upload whichever source won. Exactly ONE primer-* file exists in data/ at - # this point (smart .rkyv.zst XOR public .rkyv.json), both named for the - # build-expected TOP/CAP/schema; the glob captures it. The downstream build - # job downloads this artifact into the same data/ dir, so build.rs picks it up - # directly. `if-no-files-found: error` keeps the empty-primer guard: a missing - # file here (both sources failed) fails the release loud rather than shipping - # an empty primer. + # The typosquat corpus, which is a SEPARATE artifact from the primer and is + # required by build.rs whenever AUBE_REQUIRE_PRIMER=1 (which the build job + # sets) — without it every build job dies in write_popular_names_blob with + # "popular package names are required". It has to be its own step because + # the smart-blob path above never runs generate-primer.mjs at all, so + # hanging --popular-names-out off the public fallback (as upstream aube + # does, .github/workflows/release.yml there) would leave the smart path + # short. `--top 1` is the cheapest way to reach the script's popular-names + # branch — one packument — and its primer output is discarded outside data/ + # so it can never be mistaken for the real blob. + # + # This file is what makes the corpus DOWNLOAD-RANKED rather than the + # alphabetical metadata-primer fallback, which is the condition + # aube_resolver::popular_package_names_are_ranked() reports and the + # similar-name gate refuses to run without. + - name: Generate typosquat popularity corpus + shell: bash + run: | + cd vendor/aube + attempt=0 + until node scripts/generate-primer.mjs \ + --top 1 \ + --out "$RUNNER_TEMP/primer-discard.json" \ + --popular-names-out crates/aube-resolver/data/popular-top100000-v1.json; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 3 ]; then + echo "::error::generate-primer.mjs failed to produce the popularity corpus after $attempt attempts" + exit 1 + fi + echo "--- popularity-corpus attempt $attempt failed; retrying in 30s ---" + sleep 30 + done + bytes=$(wc -c < crates/aube-resolver/data/popular-top100000-v1.json) + echo "popularity corpus is $bytes bytes" + # Upload whichever source won, plus the popularity corpus. Exactly ONE + # primer-* file exists in data/ at this point (smart .rkyv.zst XOR public + # .rkyv.json), both named for the build-expected TOP/CAP/schema; the glob + # captures it. The downstream build job downloads this artifact into the same + # data/ dir, so build.rs picks both files up directly. `if-no-files-found: + # error` keeps the empty-primer guard: a missing file here (both sources + # failed) fails the release loud rather than shipping an empty primer. - name: Upload metadata primer artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: metadata-primer - path: vendor/aube/crates/aube-resolver/data/primer-*-s*.rkyv.* + path: | + vendor/aube/crates/aube-resolver/data/primer-*-s*.rkyv.* + vendor/aube/crates/aube-resolver/data/popular-top100000-v1.json if-no-files-found: error test: @@ -493,6 +529,10 @@ jobs: # the real primer instead of the empty fallback. With AUBE_REQUIRE_PRIMER=1 # (workflow env), a missing artifact here fails the build loud rather than # silently shipping an empty primer (the 0.0.37 bug). + # + # The same artifact carries `popular-top100000-v1.json`, the download-ranked + # typosquat corpus. build.rs treats it as mandatory under the require guard, + # so it is not optional decoration: without it every build job here fails. - name: Download metadata primer uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: diff --git a/vendor/aube/crates/aube-resolver/build.rs b/vendor/aube/crates/aube-resolver/build.rs index d888b5d19..80acd07b9 100644 --- a/vendor/aube/crates/aube-resolver/build.rs +++ b/vendor/aube/crates/aube-resolver/build.rs @@ -243,7 +243,21 @@ fn write_package_blob(out_dir: &Path, compressed: &[u8]) -> Vec { fallback_names } +/// Emit the popularity corpus, and tell the crate whether what it got is a real +/// DOWNLOAD-RANKED list or the degraded fallback. +/// +/// The distinction is load-bearing, not cosmetic. The real file is ordered +/// most-downloaded first, so a name's index IS its popularity rank. The fallback +/// is `fallback_names`, which `write_package_blob` derives by iterating a +/// `BTreeMap` — i.e. ALPHABETICALLY — and holds only `primer_top()` entries (100 +/// in dev, 2000 in release). Reporting an alphabetical position in a 100-name +/// list as a "top-100,000 popularity rank" is a lie, and every consumer that +/// reasons about relative popularity is wrong on it, so the similar-name gate +/// gates itself on this flag instead. fn write_popular_names_blob(out_dir: &Path, source: &Path, fallback_names: &[String]) { + if source.is_file() { + println!("cargo:rustc-env=AUBE_POPULAR_NAMES_RANKED=1"); + } let names = if source.is_file() { let input = std::fs::read(source).unwrap_or_else(|e| { panic!( diff --git a/vendor/aube/crates/aube-resolver/src/lib.rs b/vendor/aube/crates/aube-resolver/src/lib.rs index 674dea414..d7d821f8d 100644 --- a/vendor/aube/crates/aube-resolver/src/lib.rs +++ b/vendor/aube/crates/aube-resolver/src/lib.rs @@ -32,7 +32,8 @@ pub use peer_context::{ }; pub use platform::{SupportedArchitectures, is_supported}; pub use primer::{ - PruneStats as PrimerPruneStats, popular_package_names, prune_cache as prune_primer_cache, + PruneStats as PrimerPruneStats, popular_package_names, popular_package_names_are_ranked, + prune_cache as prune_primer_cache, }; pub use semver_util::{AgeGateCause, PickResult, pick_version_for_add}; pub use trust::{ diff --git a/vendor/aube/crates/aube-resolver/src/primer.rs b/vendor/aube/crates/aube-resolver/src/primer.rs index 70ff25e13..971c9f8f3 100644 --- a/vendor/aube/crates/aube-resolver/src/primer.rs +++ b/vendor/aube/crates/aube-resolver/src/primer.rs @@ -229,11 +229,23 @@ pub(crate) fn names() -> impl Iterator { PRIMER_INDEX.iter().map(|(name, _, _)| *name) } +/// Whether [`popular_package_names`] is ordered by download popularity. +/// +/// True only when the build embedded the real `popular-top100000-v1.json`, whose +/// order IS the popularity ranking. False for the metadata-primer fallback, +/// which is alphabetical and holds only the primer's top-N names — a corpus that +/// can support neither a "rank #N" claim nor a comparison of two names' relative +/// popularity. Callers that reason about rank must check this first. +pub fn popular_package_names_are_ranked() -> bool { + option_env!("AUBE_POPULAR_NAMES_RANKED").is_some() +} + /// Ranked public npm package names used as typo/squatting reference data. /// -/// Release builds embed the top 100,000 names. Development and downstream -/// builds without the generated artifact fall back to the metadata-primer -/// names, so callers always get a valid newline-delimited corpus. +/// Release builds embed the top 100,000 names, most-downloaded first. Development +/// and downstream builds without the generated artifact fall back to the +/// metadata-primer names, so callers always get a valid newline-delimited corpus +/// — but that fallback is NOT ranked; see [`popular_package_names_are_ranked`]. pub fn popular_package_names() -> &'static str { POPULAR_NAMES .get_or_init(|| { @@ -418,6 +430,21 @@ mod tests { assert!(names.lines().all(|name| !name.is_empty())); } + // The ranked flag is what consumers gate rank-based judgements on, so it must + // never claim a ranking the embedded corpus cannot back. The real file is + // exactly POPULAR_NAMES_TOP entries (build.rs rejects any other length under + // the require guard); the alphabetical primer fallback is far smaller. + #[test] + fn ranked_flag_matches_the_embedded_corpus() { + if popular_package_names_are_ranked() { + assert_eq!( + popular_package_names().lines().count(), + 100_000, + "a ranked corpus must be the full popular-top100000 list" + ); + } + } + #[test] fn bundled_primer_synthesizes_tarball_urls() { // The generator omits the tarball URL when it matches the diff --git a/vendor/aube/crates/aube/src/commands/add_supply_chain.rs b/vendor/aube/crates/aube/src/commands/add_supply_chain.rs index 72f77a1c5..5398d5bca 100644 --- a/vendor/aube/crates/aube/src/commands/add_supply_chain.rs +++ b/vendor/aube/crates/aube/src/commands/add_supply_chain.rs @@ -14,7 +14,10 @@ //! //! 2. **Popular-name similarity** — namespace-aware edit-distance //! comparison against the top 100,000 npm packages catches names -//! designed to look like an established dependency. +//! designed to look like an established dependency. A name the +//! corpus itself lists is cleared outright, and the gate is skipped +//! on a build whose corpus is not the real download-ranked one (see +//! `find_similar_package_name` and `similar_name_gate`). //! //! 3. **Weekly-downloads floor** — interactive confirm prompt below //! the threshold, hard refusal in non-interactive contexts unless @@ -817,6 +820,15 @@ struct PackageNameSuggestion { } async fn similar_name_gate(names: &[String], prompt: &LowDownloadPrompt) -> miette::Result<()> { + // The gate reads ABSENCE from the corpus as "nobody installs this", which + // only holds for the real 100,000-name list. Builds without it fall back to + // the metadata-primer names — alphabetical, and 100 entries in dev or 2000 in + // release — where absence means nothing and the reported "top-100,000 + // popularity rank #N" is an alphabetical position. Refusing an install on + // that is worse than not checking. + if !aube_resolver::popular_package_names_are_ranked() { + return Ok(()); + } let corpus = aube_resolver::popular_package_names(); for name in names { let Some(suggestion) = find_similar_package_name(name, corpus) else { @@ -840,11 +852,32 @@ async fn similar_name_gate(names: &[String], prompt: &LowDownloadPrompt) -> miet Ok(()) } +/// Find the most plausible package the requested name is a typo OF. +/// +/// A package among the 100,000 most-downloaded on npm is not a typosquat, so a +/// request that appears in the corpus at all is cleared outright rather than +/// compared against the rest of it. +/// +/// The blunt form of that rule is what the precision requires. Comparing every +/// name against the whole corpus refuses essentially everything worth installing +/// and points users AT the squatter as it goes — `react` rejected in favour of +/// `preact`, `lodash` of `loadash`, `debug` of `dbug`. Restricting candidates to +/// names that OUTRANK the request fixes those but still refuses **847 of the top +/// 10,000** (measured): `ws` for `ms`, `qs` for `ms`, `micromatch` for +/// `picomatch`, `safer-buffer` for `safe-buffer` — all legitimate, and this gate +/// hard-fails a non-interactive install. Clearing corpus members outright takes +/// that to **0 of the top 10,000** while `lodahs`, `raect`, `axois`, `expresss` +/// are all still caught, because a squat that nobody installs cannot rank. +/// +/// What it gives up is a squat popular enough to be in the corpus itself +/// (`expres`, #58196). That one has real download volume, which is the OSV +/// advisory check's and the downloads floor's territory rather than a +/// spelling heuristic's. fn find_similar_package_name(name: &str, corpus: &str) -> Option { let mut best: Option = None; for (index, candidate) in corpus.lines().enumerate() { if name == candidate { - continue; + return None; } let Some((requested_part, candidate_part)) = comparable_name_parts(name, candidate) else { continue; @@ -1418,6 +1451,41 @@ mod tests { ); } + // A request that is itself in the corpus is a package hundreds of thousands + // of people install on purpose, whichever lookalikes surround it. Clearing it + // outright is what takes the false-positive count across the top 10,000 from + // 847 to 0 — `ms` sits one edit from both `ws` and `qs`, and all three ship. + #[test] + fn similar_name_clears_any_request_the_corpus_lists() { + assert_eq!(find_similar_package_name("ms", "ws\nqs\nms\n"), None); + assert_eq!(find_similar_package_name("qs", "ms\nws\nqs\n"), None); + } + + // The exemption is decided by the whole corpus, not by the part scanned + // before the match — a lookalike appearing EARLIER must not win the race and + // refuse a request that the corpus goes on to list. + #[test] + fn similar_name_clears_a_listed_request_found_after_its_lookalike() { + assert_eq!( + find_similar_package_name("picomatch", "micromatch\npicomatch\n"), + None + ); + } + + // The case the gate exists for: a name the corpus does not list at all, one + // edit from a name it does. + #[test] + fn similar_name_flags_a_request_the_corpus_does_not_list() { + assert_eq!( + find_similar_package_name("expresss", "react\nlodash\nexpress\n"), + Some(PackageNameSuggestion { + name: "express".to_string(), + rank: 3, + distance: 1, + }) + ); + } + #[test] fn exact_allowed_names_skip_reputation_gates() { let names = vec!["locked[tiny]".to_string(), "new-tiny".to_string()];