diff --git a/Cargo.lock b/Cargo.lock index c110546e6..1ced10015 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -294,6 +294,20 @@ dependencies = [ "static-files 0.3.1", ] +[[package]] +name = "actix-ws" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12d4f2fbee3ef7a22fa6cb0e416b962237a167ed0419f22d4e451da2d7f082f8" +dependencies = [ + "actix-codec", + "actix-http", + "actix-web", + "bytestring", + "futures-core", + "tokio", +] + [[package]] name = "adler2" version = "2.0.1" @@ -8502,6 +8516,43 @@ dependencies = [ "zip 8.6.0", ] +[[package]] +name = "trustify-module-correlation" +version = "0.5.0-rc.1" +dependencies = [ + "actix-http", + "actix-web", + "anyhow", + "arc-swap", + "clap", + "futures", + "humantime", + "lenient_semver", + "log", + "regex", + "sea-orm", + "sea-query", + "semver", + "serde", + "serde_json", + "serde_qs", + "test-context", + "test-log", + "thiserror 2.0.18", + "time", + "tokio", + "tracing", + "trustify-auth", + "trustify-common", + "trustify-entity", + "trustify-module-fundamental", + "trustify-module-ingestor", + "trustify-test-context", + "utoipa", + "utoipa-actix-web", + "uuid", +] + [[package]] name = "trustify-module-fundamental" version = "0.5.0-rc.1" @@ -8688,6 +8739,31 @@ dependencies = [ "zip 8.6.0", ] +[[package]] +name = "trustify-module-notification" +version = "0.5.0-rc.1" +dependencies = [ + "actix-http", + "actix-web", + "actix-ws", + "anyhow", + "clap", + "futures", + "humantime", + "serde", + "serde_json", + "test-context", + "test-log", + "tokio", + "tracing", + "trustify-auth", + "trustify-common", + "trustify-infrastructure", + "trustify-test-context", + "utoipa-actix-web", + "uuid", +] + [[package]] name = "trustify-module-storage" version = "0.5.0-rc.1" @@ -8804,9 +8880,11 @@ dependencies = [ "trustify-db", "trustify-infrastructure", "trustify-module-analysis", + "trustify-module-correlation", "trustify-module-fundamental", "trustify-module-importer", "trustify-module-ingestor", + "trustify-module-notification", "trustify-module-storage", "trustify-module-ui", "trustify-module-user", diff --git a/Cargo.toml b/Cargo.toml index 47c750262..ea3c19d8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,9 +9,11 @@ members = [ "entity", "migration", "modules/analysis", + "modules/correlation", "modules/fundamental", "modules/importer", "modules/ingestor", + "modules/notification", "modules/storage", "modules/ui", "modules/user", @@ -36,10 +38,12 @@ actix-cors = "0.7" actix-http = "3.3.1" actix-tls = "3" actix-web = "4.3.1" +actix-ws = "0.3" actix-web-extras = "0.1" actix-web-httpauth = "0.8" actix-web-static-files = "4.0.1" anyhow = "1.0.72" +arc-swap = "1" async-compression = "0.4.13" async-recursion = "1" async-tar = { version = "0.6", default-features = false, features = ["runtime-tokio"] } @@ -129,7 +133,7 @@ sha2 = "0.11.0" spdx = "0.13.3" spdx-expression = "0.5.2" spdx-rs = "0.5.3" -sqlx = { version = "0.8", features = ["tls-native-tls"] } # keep aligned with sea-orm +sqlx = { version = "0.8", features = ["tls-native-tls", "postgres"] } # keep aligned with sea-orm strum = "0.28.0" tar = "0.4.45" temp-env = "0.3" @@ -167,9 +171,11 @@ trustify-entity = { path = "entity" } trustify-infrastructure = { path = "common/infrastructure" } trustify-migration = { path = "migration" } trustify-module-analysis = { path = "modules/analysis" } +trustify-module-correlation = { path = "modules/correlation" } trustify-module-fundamental = { path = "modules/fundamental" } trustify-module-importer = { path = "modules/importer" } trustify-module-ingestor = { path = "modules/ingestor" } +trustify-module-notification = { path = "modules/notification" } trustify-module-storage = { path = "modules/storage" } trustify-module-ui = { path = "modules/ui", default-features = false } trustify-module-user = { path = "modules/user" } diff --git a/common/Cargo.toml b/common/Cargo.toml index 5e44e1577..d1fca1ce3 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -52,7 +52,7 @@ tokio = { workspace = true } tracing = { workspace = true } urlencoding = { workspace = true } utoipa = { workspace = true, features = ["url"] } -uuid = { workspace = true, features = ["v5", "serde"] } +uuid = { workspace = true, features = ["v5", "v7", "serde"] } walker-common = { workspace = true, features = ["bzip2", "lzma", "flate2"] } [dev-dependencies] diff --git a/common/src/config.rs b/common/src/config.rs index 48dd3c81d..124841dfe 100644 --- a/common/src/config.rs +++ b/common/src/config.rs @@ -157,12 +157,12 @@ impl Database { format!( "postgres://{username}:{password}@{host}:{port}/{db_name}?sslmode={sslmode}", - username = &self.username, - password = &self.password.0, - host = &self.host, + username = self.username, + password = self.password.0, + host = self.host, port = self.port, - db_name = &self.name, - sslmode = &self.sslmode, + db_name = self.name, + sslmode = self.sslmode, ) } diff --git a/common/src/db/change.rs b/common/src/db/change.rs new file mode 100644 index 000000000..499c695e0 --- /dev/null +++ b/common/src/db/change.rs @@ -0,0 +1,367 @@ +use sea_orm::{ConnectionTrait, DbBackend, DbErr, Statement}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::broadcast; +use uuid::Uuid; + +const CHANNEL: &str = "trustify_changes"; +const DEFAULT_POLL_INTERVAL: Duration = Duration::from_secs(30); +const CLEANUP_INTERVAL: Duration = Duration::from_secs(300); + +/// The kind of entity that changed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ChangeEntity { + Advisory, + Sbom, +} + +impl ChangeEntity { + fn as_str(self) -> &'static str { + match self { + Self::Advisory => "advisory", + Self::Sbom => "sbom", + } + } + + fn from_str(s: &str) -> Option { + match s { + "advisory" => Some(Self::Advisory), + "sbom" => Some(Self::Sbom), + _ => None, + } + } +} + +/// The operation that occurred. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ChangeOperation { + Added, + Deleted, +} + +impl ChangeOperation { + fn as_str(self) -> &'static str { + match self { + Self::Added => "added", + Self::Deleted => "deleted", + } + } + + fn from_str(s: &str) -> Option { + match s { + "added" => Some(Self::Added), + "deleted" => Some(Self::Deleted), + _ => None, + } + } +} + +/// A single change log entry read from the database. +#[derive(Debug, Clone, serde::Serialize)] +pub struct ChangeEntry { + pub cursor: Uuid, + pub r#type: ChangeEntity, + pub id: Option, + pub operation: ChangeOperation, +} + +/// Records a change event in the change_log table. +/// +/// Called within the caller's transaction so the event is committed +/// atomically with the data change. The database trigger fires +/// `pg_notify` on commit. +pub async fn record_change( + conn: &impl ConnectionTrait, + entity_type: ChangeEntity, + entity_id: Option, + operation: ChangeOperation, +) -> Result<(), DbErr> { + let id = Uuid::now_v7(); + conn.execute(Statement::from_sql_and_values( + DbBackend::Postgres, + "INSERT INTO change_log (id, entity_type, entity_id, operation) VALUES ($1, $2, $3, $4)", + vec![ + id.into(), + entity_type.as_str().into(), + entity_id.into(), + operation.as_str().into(), + ], + )) + .await?; + Ok(()) +} + +/// Watches the change_log table via PostgreSQL LISTEN/NOTIFY with +/// a periodic polling fallback. All sqlx types are encapsulated — +/// callers only interact through the public API. +pub struct ChangeListener { + pool: sqlx::PgPool, + poll_interval: Duration, + retention: Duration, +} + +impl ChangeListener { + /// Creates a listener from a ReadWrite connection. + /// + /// Panics if the database backend is not PostgreSQL (checked at startup). + pub fn new(db: &super::ReadWrite, retention: Duration) -> Result { + let pool = db.get_postgres_connection_pool().clone(); + + Ok(Self { + pool, + poll_interval: DEFAULT_POLL_INTERVAL, + retention, + }) + } + + /// Sets the polling interval for the fallback sweep. + pub fn with_poll_interval(mut self, interval: Duration) -> Self { + self.poll_interval = interval; + self + } + + /// Sets the retention period for cleaning old entries. + pub fn with_retention(mut self, retention: Duration) -> Self { + self.retention = retention; + self + } + + /// Runs forever, calling `on_change` with batches of new entries. + /// + /// On startup, sets the cursor to the current maximum ID so only + /// new events are delivered. Automatically reconnects the LISTEN + /// connection on failure. + pub async fn run(self, on_change: F) + where + F: Fn(Vec) + Send + 'static, + { + let mut cursor = self.fetch_max_id().await; + tracing::info!(?cursor, "change listener starting"); + + let mut last_cleanup = tokio::time::Instant::now(); + + loop { + match self + .listen_loop(&on_change, &mut cursor, &mut last_cleanup) + .await + { + Ok(()) => break, + Err(err) => { + tracing::warn!(%err, "change listener connection lost, reconnecting in 5s"); + tokio::time::sleep(Duration::from_secs(5)).await; + } + } + } + } + + /// Inner loop that creates a PgListener and processes events until an error occurs. + async fn listen_loop( + &self, + on_change: &F, + cursor: &mut Uuid, + last_cleanup: &mut tokio::time::Instant, + ) -> Result<(), anyhow::Error> + where + F: Fn(Vec) + Send + 'static, + { + let mut listener = sqlx::postgres::PgListener::connect_with(&self.pool).await?; + listener.listen(CHANNEL).await?; + tracing::info!("change listener connected and listening on '{CHANNEL}'"); + + let mut poll_interval = tokio::time::interval(self.poll_interval); + poll_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + loop { + tokio::select! { + notification = listener.recv() => { + match notification { + Ok(_) => { + self.sweep(on_change, cursor).await; + } + Err(err) => { + return Err(err.into()); + } + } + } + _ = poll_interval.tick() => { + self.sweep(on_change, cursor).await; + } + } + + if last_cleanup.elapsed() >= CLEANUP_INTERVAL { + self.cleanup().await; + *last_cleanup = tokio::time::Instant::now(); + } + } + } + + /// Queries new change_log entries after the cursor and delivers them. + async fn sweep(&self, on_change: &F, cursor: &mut Uuid) + where + F: Fn(Vec), + { + match self.fetch_after(cursor).await { + Ok(entries) if entries.is_empty() => {} + Ok(entries) => { + if let Some(last) = entries.last() { + *cursor = last.cursor; + } + tracing::debug!(count = entries.len(), "delivering change events"); + on_change(entries); + } + Err(err) => { + tracing::warn!(%err, "failed to sweep change_log"); + } + } + } + + /// Fetches the latest change_log ID for cursor initialization. + async fn fetch_max_id(&self) -> Uuid { + let result: Result, _> = + sqlx::query_as("SELECT id FROM change_log ORDER BY id DESC LIMIT 1") + .fetch_optional(&self.pool) + .await; + + match result { + Ok(Some((id,))) => id, + Ok(None) => Uuid::nil(), + Err(err) => { + tracing::warn!(%err, "failed to fetch latest change_log id, starting from zero"); + Uuid::nil() + } + } + } + + /// Fetches all change_log entries with id > cursor. + async fn fetch_after(&self, cursor: &Uuid) -> Result, anyhow::Error> { + let rows: Vec<(Uuid, String, Option, String)> = sqlx::query_as( + "SELECT id, entity_type, entity_id, operation FROM change_log WHERE id > $1 ORDER BY id", + ) + .bind(cursor) + .fetch_all(&self.pool) + .await?; + + let entries = rows + .into_iter() + .filter_map(|(cursor, r#type, id, operation)| { + let r#type = ChangeEntity::from_str(&r#type)?; + let operation = ChangeOperation::from_str(&operation)?; + Some(ChangeEntry { + cursor, + r#type, + id, + operation, + }) + }) + .collect(); + + Ok(entries) + } + + /// Deletes change_log entries older than the retention period. + async fn cleanup(&self) { + let retention_secs = self.retention.as_secs() as i64; + let result = sqlx::query(&format!( + "DELETE FROM change_log WHERE created_at < NOW() - INTERVAL '{retention_secs} seconds'" + )) + .execute(&self.pool) + .await; + + match result { + Ok(r) => { + if r.rows_affected() > 0 { + tracing::debug!( + deleted = r.rows_affected(), + "cleaned up old change_log entries" + ); + } + } + Err(err) => { + tracing::warn!(%err, "failed to clean up change_log"); + } + } + } +} + +/// Fan-out broadcaster for change events. +/// +/// Wraps a single [`ChangeListener`] and distributes events to multiple +/// subscribers via [`tokio::sync::broadcast`]. Created once at startup. +#[derive(Clone)] +pub struct ChangeBroadcaster { + tx: broadcast::Sender, + pool: sqlx::PgPool, + _task: Arc>, +} + +impl ChangeBroadcaster { + pub fn new(db_rw: &super::ReadWrite, retention: Duration) -> Result { + let pool = db_rw.get_postgres_connection_pool().clone(); + let (tx, _) = broadcast::channel(1024); + let listener = ChangeListener::new(db_rw, retention)?; + let sender = tx.clone(); + + let task = tokio::spawn(async move { + listener + .run(move |entries| { + for entry in entries { + let _ = sender.send(entry); + } + }) + .await; + }); + + Ok(Self { + tx, + pool, + _task: Arc::new(task), + }) + } + + pub fn subscribe(&self) -> broadcast::Receiver { + self.tx.subscribe() + } + + /// Returns the latest event cursor, or `Uuid::nil()` if the change_log is empty. + pub async fn fetch_latest_cursor(&self) -> Uuid { + let result: Result, _> = + sqlx::query_as("SELECT id FROM change_log ORDER BY id DESC LIMIT 1") + .fetch_optional(&self.pool) + .await; + match result { + Ok(Some((id,))) => id, + Ok(None) => Uuid::nil(), + Err(err) => { + tracing::warn!(%err, "failed to fetch latest change_log cursor"); + Uuid::nil() + } + } + } + + pub async fn fetch_after(&self, cursor: &Uuid) -> Result, anyhow::Error> { + let rows: Vec<(Uuid, String, Option, String)> = sqlx::query_as( + "SELECT id, entity_type, entity_id, operation FROM change_log WHERE id > $1 ORDER BY id", + ) + .bind(cursor) + .fetch_all(&self.pool) + .await?; + + let entries = rows + .into_iter() + .filter_map(|(cursor, r#type, id, operation)| { + let r#type = ChangeEntity::from_str(&r#type)?; + let operation = ChangeOperation::from_str(&operation)?; + Some(ChangeEntry { + cursor, + r#type, + id, + operation, + }) + }) + .collect(); + + Ok(entries) + } +} diff --git a/common/src/db/mod.rs b/common/src/db/mod.rs index 80c36aa4d..5c49fd969 100644 --- a/common/src/db/mod.rs +++ b/common/src/db/mod.rs @@ -1,3 +1,4 @@ +pub mod change; pub mod chunk; pub mod limiter; pub mod multi_model; diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 20db34bc0..138569680 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -66,6 +66,7 @@ mod m0002210_sbom_node_name_index; mod m0002220_drop_qualified_purl_gist_indexes; mod m0002230_sle_license_id_index; mod m0002240_product_version_sbom_index; +mod m0002250_create_change_log; pub trait MigratorExt: Send { fn build_migrations() -> Migrations; @@ -147,6 +148,7 @@ impl MigratorExt for Migrator { .normal(m0002220_drop_qualified_purl_gist_indexes::Migration) .normal(m0002230_sle_license_id_index::Migration) .normal(m0002240_product_version_sbom_index::Migration) + .normal(m0002250_create_change_log::Migration) } } diff --git a/migration/src/m0002250_create_change_log.rs b/migration/src/m0002250_create_change_log.rs new file mode 100644 index 000000000..056e7573e --- /dev/null +++ b/migration/src/m0002250_create_change_log.rs @@ -0,0 +1,116 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(ChangeLog::Table) + .if_not_exists() + .col( + ColumnDef::new(ChangeLog::Id) + .uuid() + .not_null() + .primary_key(), + ) + .col(ColumnDef::new(ChangeLog::EntityType).text().not_null()) + .col(ColumnDef::new(ChangeLog::EntityId).uuid()) + .col(ColumnDef::new(ChangeLog::Operation).text().not_null()) + .col( + ColumnDef::new(ChangeLog::CreatedAt) + .timestamp_with_time_zone() + .not_null() + .default(Expr::current_timestamp()), + ) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .if_not_exists() + .table(ChangeLog::Table) + .name(Indexes::IdxChangeLogCreatedAt.to_string()) + .col(ChangeLog::CreatedAt) + .to_owned(), + ) + .await?; + + // Trigger function that notifies listeners on INSERT + manager + .get_connection() + .execute_unprepared( + r#" + CREATE OR REPLACE FUNCTION notify_change_log() RETURNS trigger AS $$ + BEGIN + PERFORM pg_notify('trustify_changes', NEW.id::text); + RETURN NEW; + END; + $$ LANGUAGE plpgsql + "#, + ) + .await?; + + manager + .get_connection() + .execute_unprepared( + r#" + CREATE TRIGGER change_log_notify + AFTER INSERT ON change_log + FOR EACH ROW + EXECUTE FUNCTION notify_change_log() + "#, + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .get_connection() + .execute_unprepared("DROP TRIGGER IF EXISTS change_log_notify ON change_log") + .await?; + + manager + .get_connection() + .execute_unprepared("DROP FUNCTION IF EXISTS notify_change_log()") + .await?; + + manager + .drop_index( + Index::drop() + .if_exists() + .table(ChangeLog::Table) + .name(Indexes::IdxChangeLogCreatedAt.to_string()) + .to_owned(), + ) + .await?; + + manager + .drop_table(Table::drop().if_exists().table(ChangeLog::Table).to_owned()) + .await?; + + Ok(()) + } +} + +#[derive(DeriveIden)] +enum ChangeLog { + Table, + Id, + EntityType, + EntityId, + Operation, + CreatedAt, +} + +#[derive(DeriveIden)] +enum Indexes { + IdxChangeLogCreatedAt, +} diff --git a/modules/correlation/Cargo.toml b/modules/correlation/Cargo.toml new file mode 100644 index 000000000..be0db231b --- /dev/null +++ b/modules/correlation/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "trustify-module-correlation" +version.workspace = true +edition.workspace = true +publish.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +trustify-auth = { workspace = true } +trustify-common = { workspace = true } +trustify-entity = { workspace = true } +trustify-module-fundamental = { workspace = true } + +actix-http = { workspace = true } +actix-web = { workspace = true } +anyhow = { workspace = true } +futures = { workspace = true } +arc-swap = { workspace = true } +clap = { workspace = true } +lenient_semver = { workspace = true } +regex = { workspace = true } +sea-orm = { workspace = true } +sea-query = { workspace = true } +semver = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +serde_qs = { workspace = true } +thiserror = { workspace = true } +time = { workspace = true } +tokio = { workspace = true, features = ["sync"] } +tracing = { workspace = true } +utoipa = { workspace = true, features = ["actix_extras", "uuid", "time", "rc_schema"] } +utoipa-actix-web = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +actix-http = { workspace = true } +humantime = { workspace = true } +log = { workspace = true } +serde_json = { workspace = true } +test-context = { workspace = true } +test-log = { workspace = true, features = ["log", "trace"] } +tokio = { workspace = true, features = ["full"] } +trustify-module-fundamental = { workspace = true } +trustify-module-ingestor = { workspace = true } +trustify-test-context = { workspace = true } diff --git a/modules/correlation/FINDINGS.md b/modules/correlation/FINDINGS.md new file mode 100644 index 000000000..1c2a18354 --- /dev/null +++ b/modules/correlation/FINDINGS.md @@ -0,0 +1,88 @@ +# correlate_purls: product_by_name dedup bug + +## Status + +The `correlate_purls` product_by_name lookup is implemented and working, but the +`analyze_vulnerability_ids_match_sql` correctness test still fails because of a +dedup bug. + +## What was done + +* `PurlCorrelationMatch` fields made optional (`purl_status_id`, `product_status_id`, + `version_range`) to support both purl_status and product_status match sources +* `correlate_purls` now queries `AdvisoryIndex.product_by_name` after the existing + `by_purl` loop, using bare name and `namespace/name` as lookup keys +* All three hydration functions (`hydrate_analysis`, `hydrate_purl_advisories`, + `hydrate_recommend_matches`) updated for the optional fields +* Correctness tests updated to hard equality assertions (no more subset checks) + +## Passing tests + +* `sbom_advisory_count_matches_sql` (22 advisories for quarkus-bom) +* `vulnerability_advisory_count_matches_sql` (CVE-2023-4853) +* `purl_advisory_count_matches_sql` (CVE-2023-0044 now found) +* `recommend_returns_results` +* `sbom_ubi8_advisory_count_matches_sql` (1 advisory for ubi8) + +## Failing test + +* `analyze_vulnerability_ids_match_sql` -- missing CVE-2023-0044 + +## Root cause + +In `modules/correlation/src/service/mod.rs`, the `correlate_purls` method deduplicates +product_by_name matches against existing by_purl matches using a `HashSet<(Uuid, Arc)>` +keyed on `(advisory_id, vulnerability_id)`. + +The CSAF document for CVE-2023-0044 (`etc/datasets/ds3/csaf/2023/cve-2023-0044.json`) +contains BOTH: +* `known_not_affected` entries for `quarkus-vertx-http` (matched via by_purl as purl_status) +* `known_affected` entries for `quarkus-vertx-http` (matched via product_by_name as product_status) + +The by_purl loop runs first and inserts `(advisory_id, CVE-2023-0044)` into `seen` with +`not_affected` status. When the product_by_name loop encounters the `affected` entry for +the same `(advisory_id, CVE-2023-0044)`, it's already in `seen` and gets skipped. + +Later, `hydrate_analysis` (line 384-386) filters to only `affected` and +`under_investigation` statuses, so the `not_affected` entry is dropped -- and the +`affected` entry was never added. + +Debug tracing confirmed this: CVE-2023-0044 shows `is_new=true` in the purl test (which +doesn't go through hydrate_analysis filtering), but in the analyze test the affected entry +is blocked by the dedup. + +## Fix + +Change the dedup key from `(advisory_id, vulnerability_id)` to +`(advisory_id, vulnerability_id, status_id)` so that different statuses for the same +advisory+CVE pair are not collapsed. This is on line 263 of `service/mod.rs`: + +```rust +// Current (broken): +let mut seen: HashSet<(Uuid, Arc)> = matches + .iter() + .map(|m| (m.advisory_id, Arc::clone(&m.vulnerability_id))) + .collect(); + +// Fix: +let mut seen: HashSet<(Uuid, Arc, Uuid)> = matches + .iter() + .map(|m| (m.advisory_id, Arc::clone(&m.vulnerability_id), m.status_id)) + .collect(); +``` + +And update the `seen.insert()` call on line 271 to include `entry.status_id`: + +```rust +let is_new = seen.insert(( + entry.advisory_id, + Arc::clone(&entry.vulnerability_id), + entry.status_id, +)); +``` + +## Cleanup needed after fix + +* Remove the temporary `tracing::debug!` block in `correlate_purls` (lines 275-281) +* Run `cargo xtask precommit` +* All 6 correctness tests should pass diff --git a/modules/correlation/src/config.rs b/modules/correlation/src/config.rs new file mode 100644 index 000000000..e159735ba --- /dev/null +++ b/modules/correlation/src/config.rs @@ -0,0 +1,11 @@ +/// Configuration for the correlation service. +#[derive(clap::Args, Debug, Clone, Default)] +pub struct CorrelationConfig { + /// Polling interval in seconds for the change_log fallback sweep. + #[arg(long, env = "TRUSTD_CORRELATION_POLL_INTERVAL", default_value = "30")] + pub correlation_poll_interval_secs: u64, + + /// Debounce window in seconds before reloading after a change event. + #[arg(long, env = "TRUSTD_CORRELATION_DEBOUNCE_SECS", default_value = "2")] + pub correlation_debounce_secs: u64, +} diff --git a/modules/correlation/src/endpoints/mod.rs b/modules/correlation/src/endpoints/mod.rs new file mode 100644 index 000000000..8b3949ae0 --- /dev/null +++ b/modules/correlation/src/endpoints/mod.rs @@ -0,0 +1,725 @@ +#[cfg(test)] +mod test; + +use crate::service::{CorrelationService, hydrate}; +use actix_web::{HttpResponse, Responder, get, post, web}; +use regex::Regex; +use sea_orm::{ + ColumnTrait, Condition, ConnectionTrait, EntityTrait, ModelTrait, QueryFilter, QuerySelect, + RelationTrait, SelectColumns, +}; +use sea_query::JoinType; +use serde_qs::actix::QsQuery; +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; +use std::sync::LazyLock; +use tracing::instrument; +use trustify_auth::{ + Permission, ReadAdvisory, ReadSbom, + authenticator::user::UserInformation, + authorizer::{Authorizer, Require}, + utoipa::AuthResponse, +}; +use trustify_common::db; +use trustify_common::db::chunk::chunked_with; +use trustify_common::db::pagination_cache::PaginationCache; +use trustify_common::db::query::Query; +use trustify_common::id::IdError; +use trustify_common::memo::Memo; +use trustify_common::model::{Paginated, PaginatedResults}; +use trustify_common::purl::Purl; +use trustify_common::requested_field::BoolRequestedField; +use trustify_common::requested_field::RequestedField; +use trustify_entity::{ + advisory_vulnerability, advisory_vulnerability_score, base_purl, qualified_purl, + sbom_license_expanded, sbom_node, sbom_node_purl_ref, sbom_package_license, versioned_purl, + vulnerability, +}; +use trustify_module_fundamental::common::LicenseInfo; +use trustify_module_fundamental::common::license_filtering::license_text_coalesce; +use trustify_module_fundamental::common::model::ScoredVector; +use trustify_module_fundamental::purl::model::details::purl::{PurlDetails, PurlLicenseResult}; +use trustify_module_fundamental::purl::model::{ + BasePurlHead, PurlHead, RecommendRequest, RecommendResponse, VersionedPurlHead, +}; +use trustify_module_fundamental::sbom::model::{SbomPackageSummary, SbomSummary}; +use trustify_module_fundamental::sbom::service::SbomService; +use trustify_module_fundamental::sbom::service::sbom::{FetchOptions, LicenseBasicInfo}; +use trustify_module_fundamental::vulnerability::model::{ + VulnerabilityDetails, VulnerabilityHead, + analyze::{AnalysisRequest, AnalysisResponseV3}, +}; +use utoipa_actix_web::service_config::ServiceConfig; +use uuid::Uuid; + +/// Registers in-memory correlation endpoints (replaces the SQL-based v3a path). +pub fn configure( + config: &mut ServiceConfig, + db: db::ReadOnly, + correlation: CorrelationService, + cache: PaginationCache, +) { + let sbom_service = SbomService::new(cache); + + config + .app_data(web::Data::new(correlation)) + .app_data(web::Data::new(db)) + .app_data(web::Data::new(sbom_service)) + .service(get_sbom_advisories) + .service(list_sboms) + .service(analyze_v3) + .service(get_purl) + .service(get_vulnerability) + .service(recommend) + .service(correlation_status); +} + +#[utoipa::path( + tag = "correlation", + operation_id = "getCorrelationSbomAdvisories", + params( + ("id" = Uuid, Path, description = "SBOM ID"), + ), + responses( + AuthResponse, + (status = 200, description = "Advisories affecting this SBOM"), + (status = 404, description = "SBOM not found"), + (status = 503, description = "Correlation service not ready"), + ), +)] +#[get("/v3/sbom/{id}/advisory")] +/// Find advisories affecting an SBOM using in-memory correlation. +async fn get_sbom_advisories( + service: web::Data, + db: web::Data, + id: web::Path, + _user: Require, +) -> actix_web::Result { + let matches = service.correlate_sbom(*id)?; + let statuses = service.status_slugs(); + let txn = db.begin().await?; + let advisories = hydrate::hydrate_matches(matches, &statuses, &txn).await?; + + Ok(HttpResponse::Ok().json(advisories)) +} + +/// List SBOMs with in-memory severity counts replacing the SQL-based advisory summary. +#[utoipa::path( + tag = "correlation", + operation_id = "listSboms", + params( + Query, + Paginated, + GroupFilterQuery, + SbomListParams, + ), + responses( + AuthResponse, + (status = 200, description = "Matching SBOMs", body = PaginatedResults>), + ), +)] +#[get("/v3/sbom")] +#[allow(clippy::too_many_arguments)] +async fn list_sboms( + sbom_service: web::Data, + correlation: web::Data, + db: web::Data, + web::Query(search): web::Query, + web::Query(paginated): web::Query, + web::Query(params): web::Query, + QsQuery(group_filter): QsQuery, + authorizer: web::Data, + user: UserInformation, +) -> actix_web::Result { + authorizer.require(&user, Permission::ReadSbom)?; + + let tx = db.begin().await?; + + // Always fetch without advisories — we'll patch in-memory counts if requested. + let mut options = FetchOptions::default(); + if !group_filter.group.is_empty() { + options = options.groups(group_filter.group); + } + + let mut result = sbom_service + .fetch_sboms::<_, SbomPackageSummary>(search, paginated, options, &tx) + .await?; + + if params.advisories { + let sbom_ids: Vec<_> = result.items.iter().map(|s| s.head.id).collect(); + let counts = correlation.batch_severity_counts(&sbom_ids); + + for item in &mut result.items { + let summary = counts.get(&item.head.id).cloned().unwrap_or_default(); + item.advisories = RequestedField::Requested(Some(summary)); + } + } + + Ok(HttpResponse::Ok().json(result)) +} + +/// Analyze PURLs for known vulnerabilities using in-memory correlation. +#[utoipa::path( + operation_id = "analyze_v3", + tag = "correlation", + request_body = AnalysisRequest, + responses( + AuthResponse, + (status = 200, description = "Vulnerability analysis results", body = AnalysisResponseV3), + ), +)] +#[post("/v3/vulnerability/analyze")] +async fn analyze_v3( + correlation: web::Data, + db: web::Data, + web::Json(AnalysisRequest { purls }): web::Json, + _: Require, +) -> actix_web::Result { + let parsed: Vec<_> = purls + .iter() + .filter_map(|p| Purl::try_from(p.as_str()).ok()) + .collect(); + + let matches = correlation.correlate_purls(&parsed)?; + let statuses = correlation.status_slugs(); + let tx = db.begin().await?; + let response = hydrate::hydrate_analysis(matches, &statuses, &tx).await?; + + Ok(HttpResponse::Ok().json(response)) +} + +#[derive(Clone, Debug, Default, serde::Deserialize, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +struct GroupFilterQuery { + #[serde(default)] + group: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Default, serde::Deserialize, utoipa::IntoParams)] +struct SbomListParams { + /// Include advisory severity summary per SBOM. + #[serde(default)] + pub advisories: bool, +} + +/// Retrieve PURL details with in-memory advisory correlation. +/// +/// Loads PURL head/version/base/license data from the database, then replaces +/// the advisory matching with in-memory correlation results. +#[utoipa::path( + operation_id = "getPurl", + tag = "correlation", + params( + ("key" = String, Path, description = "opaque identifier for a fully-qualified PURL, or URL-encoded pURL itself") + ), + responses( + AuthResponse, + (status = 200, description = "Details for the qualified PURL", body = PurlDetails), + (status = 404, description = "PURL not found"), + (status = 503, description = "Correlation service not ready"), + ), +)] +#[get("/v3/purl/{key}")] +#[allow(clippy::too_many_arguments)] +async fn get_purl( + correlation: web::Data, + db: web::Data, + key: web::Path, + _: Require, +) -> Result { + let tx = db.begin().await?; + + // Resolve qualified_purl by PURL string or UUID + let qualified = if key.starts_with("pkg") { + let purl = Purl::from_str(&key).map_err(|e| crate::Error::Any(e.into()))?; + let canonical = qualified_purl::CanonicalPurl::from(purl); + qualified_purl::Entity::find() + .filter(qualified_purl::Column::Purl.eq(canonical)) + .one(&tx) + .await? + } else { + let id = + Uuid::from_str(&key).map_err(|e| crate::Error::Any(IdError::InvalidUuid(e).into()))?; + qualified_purl::Entity::find_by_id(id).one(&tx).await? + }; + + let qualified = match qualified { + Some(q) => q, + None => return Ok(HttpResponse::NotFound().finish()), + }; + + // Resolve versioned_purl and base_purl + let versioned = qualified + .find_related(versioned_purl::Entity) + .one(&tx) + .await? + .ok_or_else(|| crate::Error::Any(anyhow::anyhow!("underlying versioned purl missing")))?; + + let base = versioned + .find_related(base_purl::Entity) + .one(&tx) + .await? + .ok_or_else(|| crate::Error::Any(anyhow::anyhow!("underlying base purl missing")))?; + + // Build head types + let head = PurlHead::from_entity(&base, &versioned, &qualified); + let version = VersionedPurlHead::from_entity(&base, &versioned); + let base_head = BasePurlHead::from_entity(&base); + + // In-memory advisory correlation + let purl = Purl { + ty: base.r#type.clone(), + namespace: base.namespace.clone(), + name: base.name.clone(), + version: Some(versioned.version.clone()), + qualifiers: Default::default(), + }; + + let matches = correlation.correlate_purls(&[purl])?; + let statuses = correlation.status_slugs(); + + // Get matches for the PURL we looked up (correlate_purls keys by purl string) + let purl_matches = matches.into_values().next().unwrap_or_default(); + + let advisories = hydrate::hydrate_purl_advisories(purl_matches, &statuses, &tx).await?; + + // Load licenses (same query as PurlDetails::from_entity) + let licenses = load_purl_licenses(qualified.id, &tx).await?; + + #[allow(deprecated)] + let details = PurlDetails { + head, + version, + base: base_head, + advisories, + licenses, + licenses_ref_mapping: vec![], + }; + + Ok(HttpResponse::Ok().json(details)) +} + +/// Loads license information for a qualified PURL. +async fn load_purl_licenses( + qualified_purl_id: Uuid, + connection: &impl ConnectionTrait, +) -> Result, crate::Error> { + let licenses = sbom_node_purl_ref::Entity::find() + .distinct() + .select_only() + .column_as(license_text_coalesce(), "license_name") + .select_column(sbom_package_license::Column::LicenseType) + .filter(sbom_node_purl_ref::Column::QualifiedPurlId.eq(qualified_purl_id)) + .join(JoinType::Join, sbom_node_purl_ref::Relation::Node.def()) + .join(JoinType::Join, sbom_node::Relation::PackageLicense.def()) + .join( + JoinType::LeftJoin, + sbom_package_license::Relation::SbomLicenseExpanded.def(), + ) + .join( + JoinType::LeftJoin, + sbom_license_expanded::Relation::ExpandedLicense.def(), + ) + .join( + JoinType::LeftJoin, + sbom_package_license::Relation::License.def(), + ) + .into_model::() + .all(connection) + .await? + .iter() + .map(|r| { + LicenseInfo::from(LicenseBasicInfo { + license_name: r.license_name.clone(), + license_type: r.license_type, + }) + }) + .collect(); + + Ok(licenses) +} + +#[derive(Clone, Debug, PartialEq, Eq, Default, serde::Deserialize, utoipa::IntoParams)] +struct VulnerabilityGetParams { + /// Include the full scores array from the advisory that contributed the base_score. + #[serde(default)] + pub scores: bool, +} + +/// Retrieve vulnerability details using in-memory correlation and DB hydration. +/// +/// Loads the vulnerability entity from the database, uses in-memory correlation +/// to identify affected SBOMs, then hydrates the response from the database. +#[utoipa::path( + operation_id = "getVulnerability", + tag = "correlation", + params( + ("id", Path, description = "ID of the vulnerability"), + VulnerabilityGetParams, + ), + responses( + AuthResponse, + (status = 200, description = "Specified vulnerability", body = VulnerabilityDetails), + (status = 404, description = "The vulnerability could not be found"), + (status = 503, description = "Correlation service not ready"), + ), +)] +#[get("/v3/vulnerability/{id}")] +async fn get_vulnerability( + correlation: web::Data, + db: web::Data, + id: web::Path, + web::Query(VulnerabilityGetParams { + scores: include_scores, + }): web::Query, + _: Require, +) -> Result { + let tx = db.begin().await?; + + // Load vulnerability from DB + let vuln = vulnerability::Entity::find_by_id(&*id).one(&tx).await?; + + let Some(vuln) = vuln else { + return Ok(HttpResponse::NotFound().finish()); + }; + + // Load advisory_vulnerabilities and scores from DB + let (advisory_vulns, vuln_scores) = tokio::try_join!( + advisory_vulnerability::Entity::find() + .filter(advisory_vulnerability::Column::VulnerabilityId.eq(&*id)) + .all(&tx), + advisory_vulnerability_score::Entity::find() + .filter(advisory_vulnerability_score::Column::VulnerabilityId.eq(&*id)) + .all(&tx), + )?; + + // In-memory correlation for SBOM matches + let matches = correlation.correlate_vulnerability(&id)?; + let vuln_entries = correlation.vulnerability_entries(&id); + let statuses = correlation.status_slugs(); + + // Hydrate from DB + let advisories = hydrate::hydrate_vulnerability_advisories( + &vuln, + &advisory_vulns, + &vuln_scores, + matches, + &vuln_entries, + &statuses, + &tx, + ) + .await?; + + let head = VulnerabilityHead::from_vulnerability_entity(&vuln, Memo::NotProvided, &tx).await?; + + // Build authoritative scores from DB when requested + let authoritative_scores = include_scores.then_requested(|| { + vuln.authoritative_advisory_id.map(|advisory_id| { + vuln_scores + .iter() + .filter(|s| s.advisory_id == advisory_id) + .map(|s| ScoredVector::from(s.clone())) + .collect() + }) + }); + + let details = VulnerabilityDetails { + head, + advisories, + scores: authoritative_scores, + }; + + Ok(HttpResponse::Ok().json(details)) +} + +/// Recommend Red Hat patched versions using in-memory correlation. +/// +/// Finds the highest Red Hat patch version for each input PURL (same major.minor.patch +/// with a `redhat-NNNNN` suffix), then uses in-memory correlation to determine +/// which vulnerabilities affect those patched versions. +#[utoipa::path( + operation_id = "recommend", + tag = "correlation", + request_body = RecommendRequest, + responses( + AuthResponse, + (status = 200, description = "Recommendations and remediations for provided PURLs", body = RecommendResponse), + (status = 503, description = "Correlation service not ready"), + ), +)] +#[post("/v3/purl/recommend")] +async fn recommend( + correlation: web::Data, + db: web::Data, + web::Json(RecommendRequest { purls }): web::Json, + _: Require, +) -> Result { + let tx = db.begin().await?; + + let input_purls: Vec<_> = purls.iter().filter_map(parse_input_purl).collect(); + if input_purls.is_empty() { + return Ok(HttpResponse::Ok().json(RecommendResponse::default())); + } + + let base_purls = fetch_base_purls(&input_purls, &tx).await?; + if base_purls.is_empty() { + let mut recommendations = HashMap::with_capacity(input_purls.len()); + for ip in &input_purls { + recommendations.insert(ip.purl.to_string(), Vec::new()); + } + return Ok(HttpResponse::Ok().json(RecommendResponse { recommendations })); + } + + let versioned_by_base = fetch_versioned_purls_by_base(&base_purls, &tx).await?; + + let base_purl_map: HashMap<_, _> = base_purls + .iter() + .map(|bp| { + ( + ( + bp.r#type.as_str(), + bp.namespace.as_deref(), + bp.name.as_str(), + ), + bp, + ) + }) + .collect(); + + static REDHAT_PATTERN: LazyLock = + LazyLock::new(|| Regex::new("redhat-[0-9]+$").unwrap_or_else(|_| unreachable!())); + let pattern = &*REDHAT_PATTERN; + + let mut recommendations = HashMap::with_capacity(input_purls.len()); + let mut winner_purls = Vec::new(); + let mut winner_purl_strings = Vec::new(); + + for ip in &input_purls { + let key = ( + ip.purl.ty.as_str(), + ip.purl.namespace.as_deref(), + ip.purl.name.as_str(), + ); + let Some(&base) = base_purl_map.get(&key) else { + recommendations.insert(ip.purl.to_string(), Vec::new()); + continue; + }; + + let highest = + find_highest_redhat_patch(pattern, &ip.input_version, versioned_by_base.get(&base.id)); + + if let Some(winner_vp) = highest { + let winner_purl = Purl { + ty: base.r#type.clone(), + namespace: base.namespace.clone(), + name: base.name.clone(), + version: Some(winner_vp.version.clone()), + qualifiers: Default::default(), + }; + let winner_purl_string = winner_purl.to_string(); + winner_purls.push((ip.purl.to_string(), winner_purl)); + winner_purl_strings.push(winner_purl_string); + } else { + recommendations.insert(ip.purl.to_string(), Vec::new()); + } + } + + if winner_purls.is_empty() { + return Ok(HttpResponse::Ok().json(RecommendResponse { recommendations })); + } + + // Correlate winner PURLs in-memory + let purl_refs: Vec<_> = winner_purls.iter().map(|(_, p)| p.clone()).collect(); + let matches = correlation.correlate_purls(&purl_refs)?; + + // Remap from winner PURL strings to input PURL strings + let winner_to_input: HashMap = winner_purls + .iter() + .map(|(input, winner)| (winner.to_string(), input.clone())) + .collect(); + + let mut matches_by_input: HashMap> = HashMap::new(); + for (winner_str, match_vec) in matches { + let input_str = winner_to_input + .get(&winner_str) + .cloned() + .unwrap_or(winner_str.clone()); + // Keep the winner purl string as the package name in the entry + matches_by_input + .entry(input_str) + .or_default() + .extend(match_vec); + } + + let statuses = correlation.status_slugs(); + + // Build a separate map for hydration keyed by winner PURL string + let mut hydration_matches = HashMap::new(); + for (input_str, match_vec) in &matches_by_input { + let winner_str = winner_purls + .iter() + .find(|(inp, _)| inp == input_str) + .map(|(_, w)| w.to_string()) + .unwrap_or_default(); + hydration_matches.insert(winner_str, match_vec.clone()); + } + + let mut hydrated = + hydrate::hydrate_recommend_matches(hydration_matches, &statuses, &tx).await?; + + // Map hydrated results back to input PURL strings + for (input_str, _) in &winner_purls { + let winner_str = winner_purls + .iter() + .find(|(inp, _)| inp == input_str) + .map(|(_, w)| w.to_string()) + .unwrap_or_default(); + let entries = hydrated.remove(&winner_str).unwrap_or_default(); + recommendations.insert(input_str.clone(), entries); + } + + Ok(HttpResponse::Ok().json(RecommendResponse { recommendations })) +} + +/// A user-supplied PURL paired with its parsed semver version for version comparison. +struct InputPurl { + purl: Purl, + input_version: semver::Version, +} + +/// Parses a PURL into an InputPurl if it has a valid semver version. +fn parse_input_purl(purl: &Purl) -> Option { + let version_str = purl.version.as_ref()?; + let input_version = lenient_semver::parse(version_str) + .inspect_err(|_| { + tracing::debug!( + "input purl {} version {:?} failed to parse", + purl, + version_str + ); + }) + .ok()?; + Some(InputPurl { + purl: purl.clone(), + input_version, + }) +} + +/// Batch-fetches base PURL entities matching the deduplicated set of input PURLs. +#[instrument(skip_all, err(level = tracing::Level::INFO))] +async fn fetch_base_purls( + input_purls: &[InputPurl], + connection: &impl ConnectionTrait, +) -> Result, crate::Error> { + let mut seen_keys = HashSet::new(); + let mut unique_conditions = Vec::new(); + + for ip in input_purls { + let key = ( + ip.purl.ty.clone(), + ip.purl.namespace.clone(), + ip.purl.name.clone(), + ); + if seen_keys.insert(key) { + let mut cond = Condition::all() + .add(base_purl::Column::Type.eq(&ip.purl.ty)) + .add(base_purl::Column::Name.eq(&ip.purl.name)); + if let Some(ns) = &ip.purl.namespace { + cond = cond.add(base_purl::Column::Namespace.eq(ns)); + } else { + cond = cond.add(base_purl::Column::Namespace.is_null()); + } + unique_conditions.push(cond); + } + } + + let mut results = Vec::new(); + let chunks = chunked_with(3, unique_conditions.into_iter()); + for chunk in &chunks { + let chunk: Vec<_> = chunk.collect(); + let condition = chunk + .into_iter() + .fold(Condition::any(), |c, cond| c.add(cond)); + let batch = base_purl::Entity::find() + .filter(condition) + .all(connection) + .await?; + results.extend(batch); + } + Ok(results) +} + +/// Loads all versioned PURLs for the given base PURLs, grouped by base PURL ID. +#[instrument(skip_all, err(level = tracing::Level::INFO))] +async fn fetch_versioned_purls_by_base( + base_purls: &[base_purl::Model], + connection: &impl ConnectionTrait, +) -> Result>, crate::Error> { + let base_purl_ids: Vec<_> = base_purls.iter().map(|bp| bp.id).collect(); + + let mut by_base: HashMap<_, Vec<_>> = HashMap::new(); + let id_chunks = chunked_with(1, base_purl_ids.into_iter()); + for chunk in &id_chunks { + let chunk: Vec<_> = chunk.collect(); + let batch = versioned_purl::Entity::find() + .filter(versioned_purl::Column::BasePurlId.is_in(chunk)) + .all(connection) + .await?; + for vp in batch { + by_base.entry(vp.base_purl_id).or_default().push(vp); + } + } + Ok(by_base) +} + +/// Selects the versioned PURL with the highest Red Hat pre-release suffix matching the input version. +fn find_highest_redhat_patch<'a>( + pattern: &Regex, + input_version: &semver::Version, + versioned_purls: Option<&'a Vec>, +) -> Option<&'a versioned_purl::Model> { + versioned_purls? + .iter() + .filter(|vp| pattern.is_match(&vp.version)) + .filter_map(|vp| { + lenient_semver::parse(&vp.version) + .inspect_err(|_| { + tracing::debug!("purl version {:?} failed to parse", vp.version); + }) + .ok() + .map(|v| (vp, v)) + }) + .filter(|(_, version)| { + version.major == input_version.major + && version.minor == input_version.minor + && version.patch == input_version.patch + }) + .max_by(|(_, a), (_, b)| a.pre.cmp(&b.pre)) + .map(|(vp, _)| vp) +} + +#[utoipa::path( + tag = "correlation", + operation_id = "getCorrelationStatus", + responses( + AuthResponse, + (status = 200, description = "Correlation service status"), + ), +)] +#[get("/v3/correlation/status")] +/// Get the status of the correlation service. +async fn correlation_status( + service: web::Data, + _user: Require, +) -> actix_web::Result { + let state = service.state(); + let advisory_count = state.advisory_index.by_purl.len(); + let sbom_count = state.sbom_index.by_sbom.len(); + let package_count: usize = state.sbom_index.by_sbom.values().map(|p| p.len()).sum(); + + Ok(HttpResponse::Ok().json(serde_json::json!({ + "advisory_purl_keys": advisory_count, + "sboms": sbom_count, + "package_entries": package_count, + }))) +} diff --git a/modules/correlation/src/endpoints/test.rs b/modules/correlation/src/endpoints/test.rs new file mode 100644 index 000000000..254e0669e --- /dev/null +++ b/modules/correlation/src/endpoints/test.rs @@ -0,0 +1 @@ +// Endpoint integration tests will be added once the service is wired into the server. diff --git a/modules/correlation/src/error.rs b/modules/correlation/src/error.rs new file mode 100644 index 000000000..26ebbbc98 --- /dev/null +++ b/modules/correlation/src/error.rs @@ -0,0 +1,70 @@ +use actix_web::body::BoxBody; +use actix_web::{HttpResponse, ResponseError}; +use sea_orm::DbErr; +use trustify_auth::authenticator::error::AuthorizationError; +use trustify_common::db::DbError; +use trustify_common::error::ErrorInformation; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error(transparent)] + Database(DbErr), + #[error(transparent)] + Authorization(#[from] AuthorizationError), + #[error(transparent)] + Any(#[from] anyhow::Error), + #[error("Correlation service not ready")] + NotReady, + #[error("SBOM not found: {0}")] + SbomNotFound(String), + #[error("Bad request: {0}")] + BadRequest(String), + #[error(transparent)] + Fundamental(trustify_module_fundamental::Error), +} + +unsafe impl Send for Error {} + +unsafe impl Sync for Error {} + +impl From for Error { + fn from(value: DbErr) -> Self { + Self::Database(value) + } +} + +impl From for Error { + fn from(value: DbError) -> Self { + match value { + DbError::Database(err) => Self::Database(err), + DbError::Unavailable | DbError::ReadOnly => Self::Any(anyhow::anyhow!("{value}")), + } + } +} + +impl From for Error { + fn from(value: trustify_module_fundamental::Error) -> Self { + Self::Fundamental(value) + } +} + +impl ResponseError for Error { + fn error_response(&self) -> HttpResponse { + match self { + Self::Authorization(inner) => inner.error_response(), + Self::NotReady => { + HttpResponse::ServiceUnavailable().json(ErrorInformation::new("NotReady", self)) + } + Self::SbomNotFound(id) => { + HttpResponse::NotFound().json(ErrorInformation::new("SbomNotFound", id)) + } + Self::BadRequest(msg) => { + HttpResponse::BadRequest().json(ErrorInformation::new("BadRequest", msg)) + } + err => { + tracing::warn!("{err}"); + HttpResponse::InternalServerError().json(ErrorInformation::new("Internal", "")) + } + } + } +} diff --git a/modules/correlation/src/lib.rs b/modules/correlation/src/lib.rs new file mode 100644 index 000000000..f8f506ce9 --- /dev/null +++ b/modules/correlation/src/lib.rs @@ -0,0 +1,7 @@ +pub mod config; +pub mod endpoints; +pub mod error; +pub mod model; +pub mod service; + +pub use error::Error; diff --git a/modules/correlation/src/model/mod.rs b/modules/correlation/src/model/mod.rs new file mode 100644 index 000000000..12d6537e1 --- /dev/null +++ b/modules/correlation/src/model/mod.rs @@ -0,0 +1,322 @@ +pub mod version; + +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use trustify_entity::{advisory_vulnerability_score::Severity, version_scheme::VersionScheme}; +use trustify_module_fundamental::sbom::model::AffectedSeverity; +use uuid::Uuid; + +/// Converts an entity-level CVSS severity into the affected-severity enum. +pub fn severity_to_affected(severity: Severity) -> AffectedSeverity { + match severity { + Severity::None => AffectedSeverity::None, + Severity::Low => AffectedSeverity::Low, + Severity::Medium => AffectedSeverity::Medium, + Severity::High => AffectedSeverity::High, + Severity::Critical => AffectedSeverity::Critical, + } +} + +/// Composite key for matching purls between advisories and SBOMs. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PurlKey { + pub ty: Arc, + pub namespace: Option>, + pub name: Arc, +} + +/// Version range data needed for in-memory version matching. +/// +/// For semver-family schemes, `low_parsed` and `high_parsed` hold pre-parsed +/// `semver::Version` values to avoid re-parsing on every comparison. +#[derive(Debug, Clone)] +pub struct VersionRangeData { + pub version_scheme: VersionScheme, + pub low_version: Option>, + pub low_inclusive: bool, + pub high_version: Option>, + pub high_inclusive: bool, + pub low_parsed: Option, + pub high_parsed: Option, +} + +/// A single purl_status entry stored in the advisory index. +#[derive(Debug, Clone)] +pub struct PurlStatusEntry { + pub purl_status_id: Uuid, + pub advisory_id: Uuid, + pub vulnerability_id: Arc, + pub status_id: Uuid, + pub version_range: VersionRangeData, + pub context_cpe_id: Option, +} + +/// A single product_status entry for name-based matching. +#[derive(Debug, Clone)] +pub struct ProductStatusEntry { + pub product_status_id: Uuid, + pub advisory_id: Uuid, + pub vulnerability_id: Arc, + pub status_id: Uuid, + pub context_cpe_id: Option, +} + +/// Loaded data for a single advisory, ready to apply to the index. +/// +/// Deprecated advisories are filtered out at the SQL level and never appear here. +#[derive(Debug, Clone, Default)] +pub struct AdvisoryPatch { + /// Purl status entries grouped by purl key. + pub purl_statuses: HashMap>, + /// Product status entries grouped by package name. + pub product_statuses: HashMap, Vec>, + /// Max severity per (advisory_id, vulnerability_id) pair. + pub severity: SeverityIndex, +} + +/// Max severity per (advisory_id, vulnerability_id) pair. +pub type SeverityIndex = HashMap<(Uuid, Arc), AffectedSeverity>; + +/// Source reference for a vulnerability reverse index entry. +#[derive(Debug, Clone)] +pub enum VulnEntrySource { + Purl { + purl_key: PurlKey, + version_range: VersionRangeData, + }, + Product { + package_name: Arc, + }, +} + +/// An entry in the reverse vulnerability index (vulnerability_id → entries). +#[derive(Debug, Clone)] +pub struct VulnIndexEntry { + pub advisory_id: Uuid, + pub status_id: Uuid, + pub context_cpe_id: Option, + pub source: VulnEntrySource, +} + +/// Advisory-side index: maps purl key to vulnerability status entries. +/// +/// Deprecated advisories are filtered out at the SQL level and never loaded. +#[derive(Debug, Clone)] +pub struct AdvisoryIndex { + /// Primary lookup: (type, namespace, name) → purl_status entries. + pub by_purl: HashMap>, + /// Product status lookup by package name (simple name match). + pub product_by_name: HashMap, Vec>, + /// Status slugs by ID (affected, fixed, not_affected, etc.). + pub statuses: HashMap>, + /// Max severity per (advisory_id, vulnerability_id) pair. + pub severity: SeverityIndex, + /// Reverse index: vulnerability_id → all purl/product entries referencing it. + pub by_vulnerability: HashMap, Vec>, +} + +impl AdvisoryIndex { + /// Removes all entries belonging to a specific advisory. + fn remove_advisory(&mut self, advisory_id: Uuid) { + for entries in self.by_purl.values_mut() { + entries.retain(|e| e.advisory_id != advisory_id); + } + self.by_purl.retain(|_, v| !v.is_empty()); + + for entries in self.product_by_name.values_mut() { + entries.retain(|e| e.advisory_id != advisory_id); + } + self.product_by_name.retain(|_, v| !v.is_empty()); + + self.severity + .retain(|(adv_id, _), _| *adv_id != advisory_id); + + for entries in self.by_vulnerability.values_mut() { + entries.retain(|e| e.advisory_id != advisory_id); + } + self.by_vulnerability.retain(|_, v| !v.is_empty()); + } + + /// Applies a patch: removes old data for this advisory, then inserts new data. + pub fn apply_patch(&mut self, advisory_id: Uuid, patch: AdvisoryPatch) { + self.remove_advisory(advisory_id); + + for (purl_key, entries) in &patch.purl_statuses { + for entry in entries { + self.by_vulnerability + .entry(Arc::clone(&entry.vulnerability_id)) + .or_default() + .push(VulnIndexEntry { + advisory_id: entry.advisory_id, + status_id: entry.status_id, + context_cpe_id: entry.context_cpe_id, + source: VulnEntrySource::Purl { + purl_key: purl_key.clone(), + version_range: entry.version_range.clone(), + }, + }); + } + } + + for (package, entries) in &patch.product_statuses { + for entry in entries { + self.by_vulnerability + .entry(Arc::clone(&entry.vulnerability_id)) + .or_default() + .push(VulnIndexEntry { + advisory_id: entry.advisory_id, + status_id: entry.status_id, + context_cpe_id: entry.context_cpe_id, + source: VulnEntrySource::Product { + package_name: Arc::clone(package), + }, + }); + } + } + + for (purl_key, entries) in patch.purl_statuses { + self.by_purl.entry(purl_key).or_default().extend(entries); + } + + for (package, entries) in patch.product_statuses { + self.product_by_name + .entry(package) + .or_default() + .extend(entries); + } + + for ((adv_id, vuln_id), sev) in patch.severity { + self.severity.insert((adv_id, vuln_id), sev); + } + } +} + +/// A package entry within an SBOM, storing only what's needed for matching. +#[derive(Debug, Clone)] +pub struct SbomPackageEntry { + pub ty: Arc, + pub name: Arc, + pub namespace: Option>, + pub version: Arc, +} + +/// Loaded data for a single SBOM, ready to apply to the index. +#[derive(Debug, Clone, Default)] +pub struct SbomPatch { + /// Packages belonging to this SBOM. + pub packages: Vec, + /// Describing CPE IDs for this SBOM. + pub describing_cpes: HashSet, +} + +/// SBOM-side index: maps sbom_id to its package entries. +#[derive(Debug, Clone)] +pub struct SbomIndex { + /// sbom_id → package entries for that SBOM. + pub by_sbom: HashMap>, + /// Per-SBOM describing CPE IDs for context filtering. + pub describing_cpes: HashMap>, + /// Reverse index: PurlKey → SBOMs containing packages with that key. + pub by_purl_key: HashMap>, +} + +impl SbomIndex { + /// Applies a patch: replaces packages and CPEs for this SBOM. + pub fn apply_patch(&mut self, sbom_id: Uuid, patch: SbomPatch) { + for entries in self.by_purl_key.values_mut() { + entries.retain(|id| *id != sbom_id); + } + self.by_purl_key.retain(|_, v| !v.is_empty()); + + if patch.packages.is_empty() { + self.by_sbom.remove(&sbom_id); + } else { + for entry in &patch.packages { + let key = PurlKey { + ty: Arc::clone(&entry.ty), + namespace: entry.namespace.as_ref().map(Arc::clone), + name: Arc::clone(&entry.name), + }; + self.by_purl_key.entry(key).or_default().push(sbom_id); + } + self.by_sbom + .insert(sbom_id, Arc::from(patch.packages.into_boxed_slice())); + } + + if patch.describing_cpes.is_empty() { + self.describing_cpes.remove(&sbom_id); + } else { + self.describing_cpes.insert(sbom_id, patch.describing_cpes); + } + } +} + +/// All in-memory state needed for correlation. +#[derive(Debug, Clone)] +pub struct CorrelationState { + pub advisory_index: AdvisoryIndex, + pub sbom_index: SbomIndex, +} + +impl CorrelationState { + /// Creates an empty state for use before the initial load completes. + pub fn empty() -> Self { + Self { + advisory_index: AdvisoryIndex { + by_purl: HashMap::new(), + product_by_name: HashMap::new(), + statuses: HashMap::new(), + severity: HashMap::new(), + by_vulnerability: HashMap::new(), + }, + sbom_index: SbomIndex { + by_sbom: HashMap::new(), + describing_cpes: HashMap::new(), + by_purl_key: HashMap::new(), + }, + } + } +} + +/// Result of the in-memory correlation phase (before DB hydration). +#[derive(Debug, Clone)] +pub struct CorrelationMatch { + pub advisory_id: Uuid, + pub vulnerability_id: Arc, + pub status_id: Uuid, + pub context_cpe_id: Option, + pub purl_key: PurlKey, + pub version: Arc, +} + +/// Result of correlating a standalone PURL (no SBOM context). +/// +/// Matches can originate from either the `purl_status` table (version-range +/// matching) or the `product_status` table (name-based matching from CSAF). +/// Exactly one of `purl_status_id` / `product_status_id` is set. +#[derive(Debug, Clone)] +pub struct PurlCorrelationMatch { + pub purl_status_id: Option, + pub product_status_id: Option, + pub advisory_id: Uuid, + pub vulnerability_id: Arc, + pub status_id: Uuid, + pub context_cpe_id: Option, + pub version_range: Option, +} + +/// Result of correlating a vulnerability against the SBOM index. +/// +/// Each match represents a specific PURL version in a specific SBOM that is +/// affected by the vulnerability according to an advisory. +#[derive(Debug, Clone)] +pub struct VulnCorrelationMatch { + pub advisory_id: Uuid, + pub status_id: Uuid, + pub context_cpe_id: Option, + pub sbom_id: Uuid, + pub purl_key: PurlKey, + pub version: Arc, +} diff --git a/modules/correlation/src/model/version.rs b/modules/correlation/src/model/version.rs new file mode 100644 index 000000000..f8a1fda8e --- /dev/null +++ b/modules/correlation/src/model/version.rs @@ -0,0 +1,744 @@ +use crate::model::VersionRangeData; +use std::cmp::Ordering; +use trustify_entity::version_scheme::VersionScheme; + +/// Checks whether a version string falls within the given version range, +/// using the range's version scheme for comparison. This is the Rust +/// equivalent of the PostgreSQL `version_matches()` PL/pgSQL function. +pub fn version_matches(candidate: &str, range: &VersionRangeData) -> bool { + match range.version_scheme { + VersionScheme::Semver + | VersionScheme::Npm + | VersionScheme::Gem + | VersionScheme::NuGet + | VersionScheme::Packagist + | VersionScheme::Hex + | VersionScheme::Swift + | VersionScheme::Pub + | VersionScheme::Cargo => semver_range_check(candidate, range), + + VersionScheme::Golang => { + let normalized = candidate.strip_prefix('v').unwrap_or(candidate); + semver_range_check(normalized, range) + } + + VersionScheme::Rpm => range_check(rpm_cmp, candidate, range), + VersionScheme::Maven => range_check(maven_cmp, candidate, range), + VersionScheme::Python => range_check(python_cmp, candidate, range), + + VersionScheme::Generic | VersionScheme::Git => generic_version_matches(candidate, range), + } +} + +/// Semver-specific range check that uses pre-parsed boundary versions when available. +fn semver_range_check(candidate: &str, range: &VersionRangeData) -> bool { + let candidate_v = match lenient_semver::parse(candidate) { + Ok(v) => v, + Err(_) => return false, + }; + + let low_cmp = match (&range.low_parsed, &range.low_version) { + (Some(parsed), _) => Some(candidate_v.cmp(parsed)), + (None, Some(raw)) => lenient_semver::parse(raw).ok().map(|v| candidate_v.cmp(&v)), + (None, None) => None, + }; + + if let Some(ord) = low_cmp { + if range.low_inclusive { + if ord == Ordering::Less { + return false; + } + } else if ord != Ordering::Greater { + return false; + } + } + + let high_cmp = match (&range.high_parsed, &range.high_version) { + (Some(parsed), _) => Some(candidate_v.cmp(parsed)), + (None, Some(raw)) => lenient_semver::parse(raw).ok().map(|v| candidate_v.cmp(&v)), + (None, None) => None, + }; + + if let Some(ord) = high_cmp { + if range.high_inclusive { + if ord == Ordering::Greater { + return false; + } + } else if ord != Ordering::Less { + return false; + } + } + + low_cmp.is_some() || high_cmp.is_some() +} + +/// Applies low/high bound checks using the provided comparison function. +/// Returns false if no bounds are defined. +fn range_check( + cmp_fn: fn(&str, &str) -> Option, + candidate: &str, + range: &VersionRangeData, +) -> bool { + let low_cmp = range + .low_version + .as_deref() + .and_then(|lv| cmp_fn(candidate, lv)); + + if let Some(ord) = low_cmp { + if range.low_inclusive { + if ord == Ordering::Less { + return false; + } + } else if ord != Ordering::Greater { + return false; + } + } + + let high_cmp = range + .high_version + .as_deref() + .and_then(|hv| cmp_fn(candidate, hv)); + + if let Some(ord) = high_cmp { + if range.high_inclusive { + if ord == Ordering::Greater { + return false; + } + } else if ord != Ordering::Less { + return false; + } + } + + low_cmp.is_some() || high_cmp.is_some() +} + +/// Generic/git: exact string equality only. +fn generic_version_matches(candidate: &str, range: &VersionRangeData) -> bool { + if let Some(low) = &range.low_version + && let Some(high) = &range.high_version + { + return candidate == &**low && candidate == &**high; + } + false +} + +// --- RPM comparison --- +// Ported from PL/pgSQL rpmver_cmp(). Segment-by-segment comparison with +// special handling for tilde (~) and caret (^) markers. + +/// Compares two version strings using RPM versioning rules. +fn rpm_cmp(a: &str, b: &str) -> Option { + if a == b { + return Some(Ordering::Equal); + } + + let a_segments = rpm_split_segments(a); + let b_segments = rpm_split_segments(b); + + let min_len = a_segments.len().min(b_segments.len()); + + for i in 0..min_len { + let a_seg = &a_segments[i]; + let b_seg = &b_segments[i]; + + let a_is_digit = a_seg.starts_with(|c: char| c.is_ascii_digit()); + let b_is_digit = b_seg.starts_with(|c: char| c.is_ascii_digit()); + + if a_is_digit && b_is_digit { + let a_trimmed = a_seg.trim_start_matches('0'); + let b_trimmed = b_seg.trim_start_matches('0'); + match a_trimmed.len().cmp(&b_trimmed.len()) { + Ordering::Equal => {} + ord => return Some(ord), + } + } else if a_is_digit { + return Some(Ordering::Greater); + } else if b_is_digit { + return Some(Ordering::Less); + } else if *a_seg == "~" { + if *b_seg != "~" { + return Some(Ordering::Less); + } + } else if *b_seg == "~" { + return Some(Ordering::Greater); + } else if *a_seg == "^" { + if *b_seg != "^" { + return Some(Ordering::Greater); + } + } else if *b_seg == "^" { + return Some(Ordering::Less); + } + + if a_seg != b_seg { + return Some(a_seg.cmp(b_seg)); + } + } + + // Check trailing segments + if let Some(seg) = b_segments.get(a_segments.len()) { + if *seg == "~" { + return Some(Ordering::Greater); + } + if *seg == "^" { + return Some(Ordering::Less); + } + } + if let Some(seg) = a_segments.get(b_segments.len()) { + if *seg == "~" { + return Some(Ordering::Less); + } + if *seg == "^" { + return Some(Ordering::Greater); + } + } + + Some(a_segments.len().cmp(&b_segments.len())) +} + +/// Splits an RPM version string into segments (digit runs, alpha runs, +/// or special characters ~ and ^). +fn rpm_split_segments(s: &str) -> Vec<&str> { + let mut segments = Vec::new(); + let mut chars = s.char_indices().peekable(); + + while let Some(&(start, c)) = chars.peek() { + if c == '~' || c == '^' { + segments.push(&s[start..start + 1]); + chars.next(); + } else if c.is_ascii_digit() { + while let Some(&(_, nc)) = chars.peek() { + if nc.is_ascii_digit() { + chars.next(); + } else { + break; + } + } + let end = chars.peek().map_or(s.len(), |&(i, _)| i); + segments.push(&s[start..end]); + } else if c.is_ascii_alphabetic() { + while let Some(&(_, nc)) = chars.peek() { + if nc.is_ascii_alphabetic() { + chars.next(); + } else { + break; + } + } + let end = chars.peek().map_or(s.len(), |&(i, _)| i); + segments.push(&s[start..end]); + } else { + // Skip separators (dots, dashes, etc.) + chars.next(); + } + } + + segments +} + +// --- Maven comparison --- +// Ported from PL/pgSQL mavenver_cmp(). Parses major.minor.revision with +// an optional qualifier-or-build suffix after a hyphen. + +/// Compares two version strings using Maven versioning rules. +fn maven_cmp(left: &str, right: &str) -> Option { + let (left_base, left_suffix) = maven_split(left); + let (right_base, right_suffix) = maven_split(right); + + let left_parts = maven_parse_base(left_base); + let right_parts = maven_parse_base(right_base); + + // Compare major.minor.revision + for (l, r) in left_parts.iter().zip(right_parts.iter()) { + match l.cmp(r) { + Ordering::Equal => continue, + ord => return Some(ord), + } + } + + // Compare cardinality (more parts = greater, matching SQL behavior) + match left_parts.len().cmp(&right_parts.len()) { + Ordering::Equal => {} + ord => return Some(ord), + } + + // Compare qualifier/build suffix + match (left_suffix, right_suffix) { + (None, None) => Some(Ordering::Equal), + (None, Some(_)) => Some(Ordering::Greater), + (Some(_), None) => Some(Ordering::Less), + (Some(l), Some(r)) => { + // Both are numeric: compare as numbers + if let (Ok(ln), Ok(rn)) = (l.parse::(), r.parse::()) { + Some(ln.cmp(&rn)) + } else { + // Compare as lowercase strings + Some(l.to_lowercase().cmp(&r.to_lowercase())) + } + } + } +} + +/// Splits a Maven version into base part and optional suffix after '-'. +fn maven_split(s: &str) -> (&str, Option<&str>) { + if let Some(pos) = s.find('-') { + (&s[..pos], Some(&s[pos + 1..])) + } else { + (s, None) + } +} + +/// Parses the base part of a Maven version (e.g., "1.2.3") into numeric parts. +fn maven_parse_base(base: &str) -> Vec { + base.split('.') + .map(|p| p.parse::().unwrap_or(0)) + .collect() +} + +// --- Python comparison --- +// Ported from PL/pgSQL pythonver_cmp(). Handles PEP 440 with pre-release +// (a/b/rc), post-release, dev-release, and local version segments. + +/// Compares two version strings using Python PEP 440 versioning rules. +fn python_cmp(left: &str, right: &str) -> Option { + let left_v = PythonVersion::parse(left)?; + let right_v = PythonVersion::parse(right)?; + Some(left_v.cmp(&right_v)) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct PythonVersion { + major: i64, + minor: i64, + patch: i64, + pre: Option<(String, i64)>, + post: Option, + dev: Option, + local: Option, +} + +impl PythonVersion { + /// Parses a PEP 440 version string. + fn parse(s: &str) -> Option { + let base_end = find_base_end(s); + + let base = &s[..base_end]; + let rest = &s[base_end..]; + + let parts: Vec<&str> = base.split('.').collect(); + let major = parts.first().and_then(|p| p.parse().ok()).unwrap_or(0); + let minor = parts.get(1).and_then(|p| p.parse().ok()).unwrap_or(0); + let patch = parts.get(2).and_then(|p| p.parse().ok()).unwrap_or(0); + + // Strip optional separator before pre-release + let rest = rest + .strip_prefix('-') + .or_else(|| rest.strip_prefix('_')) + .or_else(|| rest.strip_prefix('.')) + .unwrap_or(rest); + + let pre = extract_pre(rest); + let post = extract_post(s); + let dev = extract_dev(s); + let local = extract_local(s); + + Some(PythonVersion { + major, + minor, + patch, + pre, + post, + dev, + local, + }) + } +} + +impl Ord for PythonVersion { + fn cmp(&self, other: &Self) -> Ordering { + // Compare major.minor.patch + match self.major.cmp(&other.major) { + Ordering::Equal => {} + ord => return ord, + } + match self.minor.cmp(&other.minor) { + Ordering::Equal => {} + ord => return ord, + } + match self.patch.cmp(&other.patch) { + Ordering::Equal => {} + ord => return ord, + } + + // Pre-release: present < absent + match (&self.pre, &other.pre) { + (Some(_), None) => return Ordering::Less, + (None, Some(_)) => return Ordering::Greater, + (Some((lp, ln)), Some((rp, rn))) => { + match lp.cmp(rp) { + Ordering::Equal => {} + ord => return ord, + } + match ln.cmp(rn) { + Ordering::Equal => {} + ord => return ord, + } + } + (None, None) => {} + } + + // Post-release: present > absent + match (self.post, other.post) { + (Some(_), None) => return Ordering::Greater, + (None, Some(_)) => return Ordering::Less, + (Some(l), Some(r)) => match l.cmp(&r) { + Ordering::Equal => {} + ord => return ord, + }, + (None, None) => {} + } + + // Dev-release: present < absent + match (self.dev, other.dev) { + (Some(_), None) => return Ordering::Less, + (None, Some(_)) => return Ordering::Greater, + (Some(l), Some(r)) => match l.cmp(&r) { + Ordering::Equal => {} + ord => return ord, + }, + (None, None) => {} + } + + // Local: present > absent + match (&self.local, &other.local) { + (Some(_), None) => Ordering::Greater, + (None, Some(_)) => Ordering::Less, + (Some(l), Some(r)) => l.cmp(r), + (None, None) => Ordering::Equal, + } + } +} + +impl PartialOrd for PythonVersion { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// Finds where the numeric base portion of a PEP 440 version ends. +fn find_base_end(s: &str) -> usize { + let bytes = s.as_bytes(); + let mut i = 0; + while i < bytes.len() { + let c = bytes[i] as char; + if c.is_ascii_digit() || c == '.' { + i += 1; + } else { + break; + } + } + i +} + +/// Extracts pre-release tag (a, b, rc) and optional number. +fn extract_pre(rest: &str) -> Option<(String, i64)> { + for tag in &["rc", "b", "a"] { + if let Some(pos) = rest.find(tag) + && (pos == 0 || !rest.as_bytes()[pos - 1].is_ascii_alphabetic()) + { + let after = &rest[pos + tag.len()..]; + let num_str: String = after.chars().take_while(|c| c.is_ascii_digit()).collect(); + let num = num_str.parse::().unwrap_or(0); + return Some((tag.to_string(), num)); + } + } + None +} + +/// Extracts post-release number from a version string. +fn extract_post(s: &str) -> Option { + if let Some(pos) = s.find("post") { + let after = &s[pos + 4..]; + let num_str: String = after.chars().take_while(|c| c.is_ascii_digit()).collect(); + if num_str.is_empty() { + return None; + } + return num_str.parse().ok(); + } + None +} + +/// Extracts dev-release number from a version string. +fn extract_dev(s: &str) -> Option { + if let Some(pos) = s.find("dev") { + let after = &s[pos + 3..]; + let num_str: String = after.chars().take_while(|c| c.is_ascii_digit()).collect(); + if num_str.is_empty() { + return None; + } + return num_str.parse().ok(); + } + None +} + +/// Extracts local version segment (after +). +fn extract_local(s: &str) -> Option { + if let Some(pos) = s.find('+') { + let local = &s[pos + 1..]; + if local.is_empty() { + return None; + } + return Some(local.to_string()); + } + None +} + +#[cfg(test)] +mod test { + use super::*; + use crate::model::VersionRangeData; + use std::sync::Arc; + use trustify_entity::version_scheme::VersionScheme; + + /// Builds a test VersionRangeData with pre-parsed semver bounds when applicable. + fn range( + scheme: VersionScheme, + low: Option<&str>, + low_incl: bool, + high: Option<&str>, + high_incl: bool, + ) -> VersionRangeData { + let is_semver = matches!( + scheme, + VersionScheme::Semver + | VersionScheme::Npm + | VersionScheme::Gem + | VersionScheme::NuGet + | VersionScheme::Packagist + | VersionScheme::Hex + | VersionScheme::Swift + | VersionScheme::Pub + | VersionScheme::Cargo + | VersionScheme::Golang + ); + VersionRangeData { + version_scheme: scheme, + low_parsed: if is_semver { + low.and_then(|v| lenient_semver::parse(v).ok()) + } else { + None + }, + high_parsed: if is_semver { + high.and_then(|v| lenient_semver::parse(v).ok()) + } else { + None + }, + low_version: low.map(Arc::from), + low_inclusive: low_incl, + high_version: high.map(Arc::from), + high_inclusive: high_incl, + } + } + + /// Compares two semver strings (test-only helper replacing the old semver_cmp function). + fn semver_cmp(left: &str, right: &str) -> Option { + let left_v = lenient_semver::parse(left).ok()?; + let right_v = lenient_semver::parse(right).ok()?; + Some(left_v.cmp(&right_v)) + } + + // --- Semver tests --- + + #[test] + fn semver_basic_cmp() { + assert_eq!(semver_cmp("1.0.0", "1.0.0"), Some(Ordering::Equal)); + assert_eq!(semver_cmp("1.0.1", "1.0.0"), Some(Ordering::Greater)); + assert_eq!(semver_cmp("1.0.0", "1.0.1"), Some(Ordering::Less)); + assert_eq!(semver_cmp("2.0.0", "1.9.9"), Some(Ordering::Greater)); + } + + #[test] + fn semver_prerelease() { + assert_eq!(semver_cmp("1.0.0-alpha", "1.0.0"), Some(Ordering::Less)); + assert_eq!( + semver_cmp("1.0.0-alpha", "1.0.0-beta"), + Some(Ordering::Less) + ); + } + + #[test] + fn semver_range_inclusive() { + let r = range( + VersionScheme::Semver, + Some("1.0.0"), + true, + Some("2.0.0"), + true, + ); + assert!(version_matches("1.0.0", &r)); + assert!(version_matches("1.5.0", &r)); + assert!(version_matches("2.0.0", &r)); + assert!(!version_matches("0.9.0", &r)); + assert!(!version_matches("2.0.1", &r)); + } + + #[test] + fn semver_range_exclusive() { + let r = range( + VersionScheme::Semver, + Some("1.0.0"), + false, + Some("2.0.0"), + false, + ); + assert!(!version_matches("1.0.0", &r)); + assert!(version_matches("1.5.0", &r)); + assert!(!version_matches("2.0.0", &r)); + } + + #[test] + fn semver_open_upper() { + let r = range(VersionScheme::Semver, Some("1.0.0"), true, None, false); + assert!(version_matches("1.0.0", &r)); + assert!(version_matches("99.0.0", &r)); + assert!(!version_matches("0.9.0", &r)); + } + + #[test] + fn semver_open_lower() { + let r = range(VersionScheme::Semver, None, false, Some("2.0.0"), false); + assert!(version_matches("1.0.0", &r)); + assert!(!version_matches("2.0.0", &r)); + } + + // --- Golang tests --- + + #[test] + fn golang_strips_v_prefix() { + let r = range( + VersionScheme::Golang, + Some("1.0.0"), + true, + Some("2.0.0"), + false, + ); + assert!(version_matches("v1.5.0", &r)); + assert!(version_matches("1.5.0", &r)); + assert!(!version_matches("v2.0.0", &r)); + } + + // --- RPM tests --- + + #[test] + fn rpm_basic_cmp() { + assert_eq!(rpm_cmp("1.0", "1.0"), Some(Ordering::Equal)); + assert_eq!(rpm_cmp("1.1", "1.0"), Some(Ordering::Greater)); + assert_eq!(rpm_cmp("1.0", "1.1"), Some(Ordering::Less)); + } + + #[test] + fn rpm_tilde() { + // Tilde sorts before anything, even empty + assert_eq!(rpm_cmp("1.0~rc1", "1.0"), Some(Ordering::Less)); + } + + #[test] + fn rpm_caret() { + // Caret sorts after release + assert_eq!(rpm_cmp("1.0^post1", "1.0"), Some(Ordering::Greater)); + } + + #[test] + fn rpm_numeric_vs_alpha() { + // Numeric segments sort after alphabetic + assert_eq!(rpm_cmp("1.0.1", "1.0.a"), Some(Ordering::Greater)); + } + + #[test] + fn rpm_range() { + let r = range(VersionScheme::Rpm, Some("1.0"), true, Some("2.0"), false); + assert!(version_matches("1.0", &r)); + assert!(version_matches("1.5", &r)); + assert!(!version_matches("2.0", &r)); + assert!(!version_matches("0.9", &r)); + } + + // --- Maven tests --- + + #[test] + fn maven_basic_cmp() { + assert_eq!(maven_cmp("1.0.0", "1.0.0"), Some(Ordering::Equal)); + assert_eq!(maven_cmp("2.0.0", "1.0.0"), Some(Ordering::Greater)); + } + + #[test] + fn maven_qualifier() { + // No qualifier > with qualifier (release > snapshot) + assert_eq!( + maven_cmp("1.0.0", "1.0.0-SNAPSHOT"), + Some(Ordering::Greater) + ); + assert_eq!(maven_cmp("1.0.0-alpha", "1.0.0-beta"), Some(Ordering::Less)); + } + + #[test] + fn maven_range() { + let r = range( + VersionScheme::Maven, + Some("1.0.0"), + true, + Some("2.0.0"), + false, + ); + assert!(version_matches("1.0.0", &r)); + assert!(version_matches("1.5.0", &r)); + assert!(!version_matches("2.0.0", &r)); + } + + // --- Python tests --- + + #[test] + fn python_basic_cmp() { + assert_eq!(python_cmp("1.0.0", "1.0.0"), Some(Ordering::Equal)); + assert_eq!(python_cmp("1.1.0", "1.0.0"), Some(Ordering::Greater)); + } + + #[test] + fn python_prerelease() { + assert_eq!(python_cmp("1.0.0a1", "1.0.0"), Some(Ordering::Less)); + assert_eq!(python_cmp("1.0.0b1", "1.0.0a1"), Some(Ordering::Greater)); + assert_eq!(python_cmp("1.0.0rc1", "1.0.0b1"), Some(Ordering::Greater)); + } + + #[test] + fn python_post_release() { + assert_eq!(python_cmp("1.0.0.post1", "1.0.0"), Some(Ordering::Greater)); + } + + #[test] + fn python_dev_release() { + assert_eq!(python_cmp("1.0.0.dev1", "1.0.0"), Some(Ordering::Less)); + } + + #[test] + fn python_range() { + let r = range( + VersionScheme::Python, + Some("1.0.0"), + true, + Some("2.0.0"), + false, + ); + assert!(version_matches("1.0.0", &r)); + assert!(version_matches("1.5.0", &r)); + assert!(!version_matches("2.0.0", &r)); + assert!(!version_matches("1.0.0a1", &r)); + } + + // --- Generic tests --- + + #[test] + fn generic_exact_match() { + let r = range(VersionScheme::Generic, Some("1.0"), true, Some("1.0"), true); + assert!(version_matches("1.0", &r)); + assert!(!version_matches("1.1", &r)); + } +} diff --git a/modules/correlation/src/service/hydrate.rs b/modules/correlation/src/service/hydrate.rs new file mode 100644 index 000000000..940468606 --- /dev/null +++ b/modules/correlation/src/service/hydrate.rs @@ -0,0 +1,1195 @@ +use crate::Error; +use crate::model::{ + CorrelationMatch, PurlCorrelationMatch, PurlKey, VersionRangeData, VulnCorrelationMatch, + VulnEntrySource, VulnIndexEntry, +}; +use sea_orm::{ + ColumnTrait, Condition, ConnectionTrait, EntityTrait, JoinType, QueryFilter, QuerySelect, + RelationTrait, +}; +use sea_query::Expr; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; +use tracing::instrument; +use trustify_common::purl::Purl; +use trustify_entity::{ + advisory, advisory_vulnerability, advisory_vulnerability_score, cpe, + package_relates_to_package, relationship::Relationship, remediation, remediation_purl_status, + sbom, sbom_node, sbom_package, vulnerability, +}; +use trustify_module_fundamental::{ + advisory::model::AdvisoryHead, + common::model::ScoredVector, + purl::model::{ + BasePurlHead, RecommendEntry, VexStatus, VulnerabilityStatus, + details::{ + purl::{PurlAdvisory, PurlStatus, StatusContext}, + version_range::VersionRange, + }, + summary::{purl::PurlSummary, remediation::RemediationSummary}, + }, + sbom::model::{ + SbomHead, SbomPackage, + details::{SbomAdvisory, SbomStatus}, + }, + vulnerability::model::{ + VulnerabilityAdvisoryHead, VulnerabilityAdvisoryStatus, VulnerabilityAdvisorySummary, + VulnerabilityHead, VulnerabilitySbomStatus, + analyze::{AnalysisDetailsV3, AnalysisPurlStatus, AnalysisResponseV3, AnalysisResultV3}, + }, +}; +use uuid::Uuid; + +/// Grouping key for a single SbomStatus entry within an advisory. +#[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] +struct StatusKey { + vulnerability_id: String, + status_slug: String, + context_cpe_id: Option, +} + +/// Hydrates in-memory correlation matches into the full SbomAdvisory API response. +/// +/// Extracts unique entity IDs from the matches, runs batch DB queries for +/// advisory/vulnerability/score/CPE metadata, then groups matches into the +/// nested SbomAdvisory → SbomStatus → SbomPackage structure. +#[allow(deprecated)] +#[instrument(skip_all, err(level = tracing::Level::INFO))] +pub async fn hydrate_matches( + matches: Vec, + statuses: &HashMap>, + connection: &impl ConnectionTrait, +) -> Result, Error> { + if matches.is_empty() { + return Ok(Vec::new()); + } + + // Collect unique IDs for batch queries + let mut advisory_ids = HashSet::new(); + let mut vuln_ids = HashSet::new(); + let mut av_pairs: HashSet<(Uuid, String)> = HashSet::new(); + let mut cpe_ids = HashSet::new(); + + for m in &matches { + advisory_ids.insert(m.advisory_id); + vuln_ids.insert(m.vulnerability_id.as_ref().to_string()); + av_pairs.insert((m.advisory_id, m.vulnerability_id.as_ref().to_string())); + if let Some(cpe_id) = m.context_cpe_id { + cpe_ids.insert(cpe_id); + } + } + + let advisory_id_vec: Vec = advisory_ids.into_iter().collect(); + let vuln_id_vec: Vec = vuln_ids.into_iter().collect(); + + // Batch load all needed entities in parallel + let (advisory_models, av_models, vuln_models, score_models, cpe_models) = tokio::try_join!( + load_advisories(&advisory_id_vec, connection), + load_advisory_vulnerabilities(&advisory_id_vec, connection), + load_vulnerabilities(&vuln_id_vec, connection), + load_scores(&advisory_id_vec, connection), + load_cpes(&cpe_ids, connection), + )?; + + // Build advisory heads (includes issuer org batch load) + let advisory_heads = AdvisoryHead::from_entities(&advisory_models, connection).await?; + let advisory_head_map: HashMap = advisory_models + .iter() + .zip(advisory_heads) + .map(|(model, head)| (model.id, head)) + .collect(); + + // Index advisory_vulnerability models by (advisory_id, vulnerability_id) + let av_map: HashMap<(Uuid, String), advisory_vulnerability::Model> = av_models + .into_iter() + .map(|av| ((av.advisory_id, av.vulnerability_id.clone()), av)) + .collect(); + + // Index vulnerability models by id + let vuln_map: HashMap = + vuln_models.into_iter().map(|v| (v.id.clone(), v)).collect(); + + // Group scores by (advisory_id, vulnerability_id) + let mut score_map: HashMap<(Uuid, String), Vec> = + HashMap::new(); + for score in score_models { + score_map + .entry((score.advisory_id, score.vulnerability_id.clone())) + .or_default() + .push(score); + } + + // Index CPE models by id + let cpe_map: HashMap = cpe_models.into_iter().map(|c| (c.id, c)).collect(); + + // Group matches: advisory_id → StatusKey → Vec + let mut advisory_groups: BTreeMap>> = + BTreeMap::new(); + + for m in &matches { + let status_slug = statuses + .get(&m.status_id) + .map(|s| s.as_ref().to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + let key = StatusKey { + vulnerability_id: m.vulnerability_id.as_ref().to_string(), + status_slug, + context_cpe_id: m.context_cpe_id, + }; + + let pkg = build_sbom_package(&m.purl_key, &m.version); + + advisory_groups + .entry(m.advisory_id) + .or_default() + .entry(key) + .or_default() + .push(pkg); + } + + // Assemble the final Vec + let mut result = Vec::with_capacity(advisory_groups.len()); + + for (advisory_id, status_groups) in advisory_groups { + let head = match advisory_head_map.get(&advisory_id) { + Some(head) => head.clone(), + None => continue, + }; + + let mut sbom_statuses = Vec::with_capacity(status_groups.len()); + + for (key, packages) in status_groups { + let av_key = (advisory_id, key.vulnerability_id.clone()); + let av = match av_map.get(&av_key) { + Some(av) => av, + None => continue, + }; + let vuln = match vuln_map.get(&key.vulnerability_id) { + Some(v) => v, + None => continue, + }; + + let scores: Vec = score_map + .get(&av_key) + .cloned() + .unwrap_or_default() + .into_iter() + .map(ScoredVector::from) + .collect(); + + let context = key.context_cpe_id.and_then(|cpe_id| { + cpe_map + .get(&cpe_id) + .map(|c| StatusContext::Cpe(c.to_string())) + }); + + sbom_statuses.push(SbomStatus { + vulnerability: VulnerabilityHead::from_advisory_vulnerability_entity(av, vuln), + status: key.status_slug, + context, + packages, + scores, + }); + } + + result.push(SbomAdvisory { + head, + status: sbom_statuses, + }); + } + + Ok(result) +} + +/// Builds an SbomPackage from in-memory purl key and version data. +#[allow(deprecated)] +fn build_sbom_package(purl_key: &PurlKey, version: &Arc) -> SbomPackage { + let purl = Purl { + ty: purl_key.ty.to_string(), + namespace: purl_key.namespace.as_ref().map(|ns| ns.to_string()), + name: purl_key.name.to_string(), + version: Some(version.to_string()), + qualifiers: Default::default(), + }; + + let purl_id = match (&purl_key.namespace, purl_key.name.as_ref()) { + (Some(ns), name) => format!("pkg:{}/{}/{}@{}", purl_key.ty, ns, name, version), + (None, name) => format!("pkg:{}/{}@{}", purl_key.ty, name, version), + }; + + SbomPackage { + id: purl_id, + name: purl_key.name.to_string(), + group: purl_key.namespace.as_ref().map(|ns| ns.to_string()), + version: Some(version.to_string()), + purl: vec![PurlSummary::from(purl)], + cpe: vec![], + licenses: vec![], + licenses_ref_mapping: vec![], + } +} + +/// Batch loads advisory models by ID. +async fn load_advisories( + ids: &[Uuid], + connection: &impl ConnectionTrait, +) -> Result, Error> { + Ok(advisory::Entity::find() + .filter(advisory::Column::Id.is_in(ids.iter().copied())) + .all(connection) + .await?) +} + +/// Batch loads advisory_vulnerability models for the given advisory IDs. +async fn load_advisory_vulnerabilities( + advisory_ids: &[Uuid], + connection: &impl ConnectionTrait, +) -> Result, Error> { + Ok(advisory_vulnerability::Entity::find() + .filter(advisory_vulnerability::Column::AdvisoryId.is_in(advisory_ids.iter().copied())) + .all(connection) + .await?) +} + +/// Batch loads vulnerability models by ID. +async fn load_vulnerabilities( + ids: &[String], + connection: &impl ConnectionTrait, +) -> Result, Error> { + Ok(vulnerability::Entity::find() + .filter(vulnerability::Column::Id.is_in(ids.iter().cloned())) + .all(connection) + .await?) +} + +/// Batch loads advisory_vulnerability_score models for the given advisory IDs. +async fn load_scores( + advisory_ids: &[Uuid], + connection: &impl ConnectionTrait, +) -> Result, Error> { + Ok(advisory_vulnerability_score::Entity::find() + .filter( + advisory_vulnerability_score::Column::AdvisoryId.is_in(advisory_ids.iter().copied()), + ) + .all(connection) + .await?) +} + +/// Batch loads CPE models by ID. +async fn load_cpes( + ids: &HashSet, + connection: &impl ConnectionTrait, +) -> Result, Error> { + if ids.is_empty() { + return Ok(Vec::new()); + } + Ok(cpe::Entity::find() + .filter(cpe::Column::Id.is_in(ids.iter().copied())) + .all(connection) + .await?) +} + +/// Batch loads remediations linked to a set of purl_status IDs. +async fn load_purl_remediations( + purl_status_ids: &HashSet, + connection: &impl ConnectionTrait, +) -> Result>, Error> { + if purl_status_ids.is_empty() { + return Ok(HashMap::new()); + } + + let links = remediation_purl_status::Entity::find() + .filter( + remediation_purl_status::Column::PurlStatusId.is_in(purl_status_ids.iter().copied()), + ) + .all(connection) + .await?; + + if links.is_empty() { + return Ok(HashMap::new()); + } + + let remediation_ids: Vec = links.iter().map(|l| l.remediation_id).collect(); + let remediations = remediation::Entity::find() + .filter(remediation::Column::Id.is_in(remediation_ids)) + .all(connection) + .await?; + + let rem_map: HashMap = + remediations.into_iter().map(|r| (r.id, r)).collect(); + + let mut result: HashMap> = HashMap::new(); + for link in links { + if let Some(rem) = rem_map.get(&link.remediation_id) { + result + .entry(link.purl_status_id) + .or_default() + .push(RemediationSummary { + id: rem.id, + category: rem.category.clone(), + details: rem.details.clone(), + url: rem.url.clone(), + data: rem.data.clone(), + }); + } + } + + Ok(result) +} + +/// Converts in-memory VersionRangeData to the API VersionRange model. +fn version_range_to_api(vr: &VersionRangeData) -> Option { + match (&vr.low_version, &vr.high_version) { + (Some(low), Some(high)) => Some(VersionRange::Full { + version_scheme_id: vr.version_scheme.to_string(), + low_version: low.to_string(), + low_inclusive: vr.low_inclusive, + high_version: high.to_string(), + high_inclusive: vr.high_inclusive, + }), + (Some(low), None) => Some(VersionRange::Left { + version_scheme_id: vr.version_scheme.to_string(), + low_version: low.to_string(), + low_inclusive: vr.low_inclusive, + }), + (None, Some(high)) => Some(VersionRange::Right { + version_scheme_id: vr.version_scheme.to_string(), + high_version: high.to_string(), + high_inclusive: vr.high_inclusive, + }), + (None, None) => Some(VersionRange::Unbounded), + } +} + +/// Hydrates in-memory correlation matches into the AnalysisResponseV3 API response. +/// +/// Filters to only "affected" and "under_investigation" statuses, then batch-loads +/// advisory/vulnerability/score/CPE/remediation metadata from the database. +#[instrument(skip_all, err(level = tracing::Level::INFO))] +pub async fn hydrate_analysis( + matches: HashMap>, + statuses: &HashMap>, + connection: &impl ConnectionTrait, +) -> Result { + // Collect unique IDs across all matches for batch queries + let mut advisory_ids = HashSet::new(); + let mut vuln_ids = HashSet::new(); + let mut cpe_ids = HashSet::new(); + let mut purl_status_ids = HashSet::new(); + + for purl_matches in matches.values() { + for m in purl_matches { + let status_slug = statuses.get(&m.status_id); + let is_relevant = status_slug + .is_some_and(|s| s.as_ref() == "affected" || s.as_ref() == "under_investigation"); + if !is_relevant { + continue; + } + advisory_ids.insert(m.advisory_id); + vuln_ids.insert(m.vulnerability_id.as_ref().to_string()); + if let Some(id) = m.purl_status_id { + purl_status_ids.insert(id); + } + if let Some(cpe_id) = m.context_cpe_id { + cpe_ids.insert(cpe_id); + } + } + } + + let advisory_id_vec: Vec = advisory_ids.into_iter().collect(); + let vuln_id_vec: Vec = vuln_ids.into_iter().collect(); + + // Batch load all needed entities + let (advisory_models, av_models, vuln_models, score_models, cpe_models, remediation_map) = tokio::try_join!( + load_advisories(&advisory_id_vec, connection), + load_advisory_vulnerabilities(&advisory_id_vec, connection), + load_vulnerabilities(&vuln_id_vec, connection), + load_scores(&advisory_id_vec, connection), + load_cpes(&cpe_ids, connection), + load_purl_remediations(&purl_status_ids, connection), + )?; + + // Build lookup maps + let advisory_heads = AdvisoryHead::from_entities(&advisory_models, connection).await?; + let advisory_head_map: HashMap = advisory_models + .iter() + .zip(advisory_heads) + .map(|(model, head)| (model.id, head)) + .collect(); + + let av_map: HashMap<(Uuid, String), advisory_vulnerability::Model> = av_models + .into_iter() + .map(|av| ((av.advisory_id, av.vulnerability_id.clone()), av)) + .collect(); + + let vuln_map: HashMap = + vuln_models.into_iter().map(|v| (v.id.clone(), v)).collect(); + + let mut score_map: HashMap<(Uuid, String), Vec> = + HashMap::new(); + for score in score_models { + score_map + .entry((score.advisory_id, score.vulnerability_id.clone())) + .or_default() + .push(score); + } + + let cpe_map: HashMap = cpe_models.into_iter().map(|c| (c.id, c)).collect(); + + // Build the response grouped by input PURL + let mut response = BTreeMap::new(); + + for (purl_str, purl_matches) in &matches { + // Group this PURL's matches by vulnerability_id + let mut vuln_groups: BTreeMap> = BTreeMap::new(); + + for m in purl_matches { + let status_slug = statuses.get(&m.status_id); + let is_relevant = status_slug + .is_some_and(|s| s.as_ref() == "affected" || s.as_ref() == "under_investigation"); + if !is_relevant { + continue; + } + vuln_groups + .entry(m.vulnerability_id.as_ref().to_string()) + .or_default() + .push(m); + } + + let mut details = Vec::with_capacity(vuln_groups.len()); + + for (vuln_id, vuln_matches) in vuln_groups { + let vuln = match vuln_map.get(&vuln_id) { + Some(v) => v, + None => continue, + }; + + // Build purl_statuses from all matches for this vulnerability + let mut purl_statuses = Vec::with_capacity(vuln_matches.len()); + + for m in &vuln_matches { + let av_key = (m.advisory_id, vuln_id.clone()); + let av = match av_map.get(&av_key) { + Some(av) => av, + None => continue, + }; + + let advisory_head = match advisory_head_map.get(&m.advisory_id) { + Some(h) => h.clone(), + None => continue, + }; + + let scores: Vec = + score_map.get(&av_key).cloned().unwrap_or_default(); + + let status_slug = statuses + .get(&m.status_id) + .map(|s| s.as_ref().to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + let context_cpe = m + .context_cpe_id + .and_then(|id| cpe_map.get(&id).map(|c| c.to_string())); + + let version_range = m.version_range.as_ref().and_then(version_range_to_api); + + let vuln_head = VulnerabilityHead::from_advisory_vulnerability_entity(av, vuln); + + let purl_status = PurlStatus::from_head( + vuln_head, + advisory_head, + status_slug, + version_range, + context_cpe, + &scores, + )?; + + let remediations = m + .purl_status_id + .and_then(|id| remediation_map.get(&id)) + .cloned() + .unwrap_or_default(); + + purl_statuses.push(AnalysisPurlStatus { + purl_status, + remediations, + }); + } + + if !purl_statuses.is_empty() { + let head = VulnerabilityHead::from_vulnerability_entity_and_description(vuln, None); + details.push(AnalysisDetailsV3 { + head, + purl_statuses, + }); + } + } + + response.insert( + purl_str.clone(), + AnalysisResultV3 { + details, + warnings: Vec::new(), + }, + ); + } + + Ok(AnalysisResponseV3(response)) +} + +/// Hydrates in-memory PURL correlation matches into `Vec`. +/// +/// Groups matches by advisory_id, builds PurlStatus entries with batch-loaded +/// advisory/vulnerability/score/CPE metadata, and returns the advisory list +/// for a single PURL's details response. +#[instrument(skip_all, err(level = tracing::Level::INFO))] +pub async fn hydrate_purl_advisories( + matches: Vec, + statuses: &HashMap>, + connection: &impl ConnectionTrait, +) -> Result, Error> { + if matches.is_empty() { + return Ok(Vec::new()); + } + + let mut advisory_ids = HashSet::new(); + let mut vuln_ids = HashSet::new(); + let mut cpe_ids = HashSet::new(); + + for m in &matches { + advisory_ids.insert(m.advisory_id); + vuln_ids.insert(m.vulnerability_id.as_ref().to_string()); + if let Some(cpe_id) = m.context_cpe_id { + cpe_ids.insert(cpe_id); + } + } + + let advisory_id_vec: Vec = advisory_ids.into_iter().collect(); + let vuln_id_vec: Vec = vuln_ids.into_iter().collect(); + + let (advisory_models, av_models, vuln_models, score_models, cpe_models) = tokio::try_join!( + load_advisories(&advisory_id_vec, connection), + load_advisory_vulnerabilities(&advisory_id_vec, connection), + load_vulnerabilities(&vuln_id_vec, connection), + load_scores(&advisory_id_vec, connection), + load_cpes(&cpe_ids, connection), + )?; + + let advisory_heads = AdvisoryHead::from_entities(&advisory_models, connection).await?; + let advisory_head_map: HashMap = advisory_models + .iter() + .zip(advisory_heads) + .map(|(model, head)| (model.id, head)) + .collect(); + + let av_map: HashMap<(Uuid, String), advisory_vulnerability::Model> = av_models + .into_iter() + .map(|av| ((av.advisory_id, av.vulnerability_id.clone()), av)) + .collect(); + + let vuln_map: HashMap = + vuln_models.into_iter().map(|v| (v.id.clone(), v)).collect(); + + let mut score_map: HashMap<(Uuid, String), Vec> = + HashMap::new(); + for score in score_models { + score_map + .entry((score.advisory_id, score.vulnerability_id.clone())) + .or_default() + .push(score); + } + + let cpe_map: HashMap = cpe_models.into_iter().map(|c| (c.id, c)).collect(); + + // Group matches by advisory_id + let mut advisory_groups: BTreeMap> = BTreeMap::new(); + for m in &matches { + advisory_groups.entry(m.advisory_id).or_default().push(m); + } + + let mut result = Vec::with_capacity(advisory_groups.len()); + + for (advisory_id, group) in advisory_groups { + let head = match advisory_head_map.get(&advisory_id) { + Some(head) => head.clone(), + None => continue, + }; + + let mut purl_statuses = Vec::with_capacity(group.len()); + + for m in group { + let av_key = (advisory_id, m.vulnerability_id.as_ref().to_string()); + let av = match av_map.get(&av_key) { + Some(av) => av, + None => continue, + }; + let vuln = match vuln_map.get(m.vulnerability_id.as_ref()) { + Some(v) => v, + None => continue, + }; + + let scores: Vec = + score_map.get(&av_key).cloned().unwrap_or_default(); + + let status_slug = statuses + .get(&m.status_id) + .map(|s| s.as_ref().to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + let context_cpe = m + .context_cpe_id + .and_then(|id| cpe_map.get(&id).map(|c| c.to_string())); + + let version_range = m.version_range.as_ref().and_then(version_range_to_api); + + let vuln_head = VulnerabilityHead::from_advisory_vulnerability_entity(av, vuln); + + let purl_status = PurlStatus::from_head( + vuln_head, + head.clone(), + status_slug, + version_range, + context_cpe, + &scores, + )?; + + purl_statuses.push(purl_status); + } + + result.push(PurlAdvisory { + head, + status: purl_statuses, + }); + } + + Ok(result) +} + +/// Formats a VersionRangeData as a display string for VulnerabilityAdvisoryStatus. +fn format_version_range(vr: &VersionRangeData) -> String { + fn open_delim(incl: bool) -> char { + if incl { '[' } else { '(' } + } + fn close_delim(incl: bool) -> char { + if incl { ']' } else { ')' } + } + + match (&vr.low_version, &vr.high_version) { + (Some(low), Some(high)) if low == high => low.to_string(), + (Some(low), Some(high)) => { + format!( + "{}{},{}{}", + open_delim(vr.low_inclusive), + low, + high, + close_delim(vr.high_inclusive) + ) + } + (Some(low), None) => { + format!( + "{}{},{}", + open_delim(vr.low_inclusive), + low, + close_delim(vr.high_inclusive) + ) + } + (None, Some(high)) => { + format!( + "{},{}{}", + open_delim(vr.low_inclusive), + high, + close_delim(vr.high_inclusive) + ) + } + (None, None) => "*".to_string(), + } +} + +/// Batch loads sbom models by SBOM ID. +async fn load_sboms( + sbom_ids: &[Uuid], + connection: &impl ConnectionTrait, +) -> Result, Error> { + if sbom_ids.is_empty() { + return Ok(Vec::new()); + } + Ok(sbom::Entity::find() + .filter(sbom::Column::SbomId.is_in(sbom_ids.iter().copied())) + .all(connection) + .await?) +} + +/// Batch loads the describing sbom_node for each sbom (sbom_node.node_id = sbom.node_id). +async fn load_describing_sbom_nodes( + sbom_models: &[sbom::Model], + connection: &impl ConnectionTrait, +) -> Result, Error> { + if sbom_models.is_empty() { + return Ok(HashMap::new()); + } + + let mut condition = Condition::any(); + for s in sbom_models { + condition = condition.add( + Condition::all() + .add(sbom_node::Column::SbomId.eq(s.sbom_id)) + .add(sbom_node::Column::NodeId.eq(&s.node_id)), + ); + } + + let nodes = sbom_node::Entity::find() + .filter(condition) + .all(connection) + .await?; + + let node_map: HashMap = + nodes.into_iter().map(|n| (n.sbom_id, n)).collect(); + + Ok(node_map) +} + +/// Batch counts packages per SBOM. +async fn load_package_counts( + sbom_ids: &[Uuid], + connection: &impl ConnectionTrait, +) -> Result, Error> { + if sbom_ids.is_empty() { + return Ok(HashMap::new()); + } + + let counts: Vec<(Uuid, i64)> = sbom_package::Entity::find() + .filter(sbom_package::Column::SbomId.is_in(sbom_ids.iter().copied())) + .select_only() + .column(sbom_package::Column::SbomId) + .column_as(Expr::col(sbom_package::Column::NodeId).count(), "count") + .group_by(sbom_package::Column::SbomId) + .into_tuple() + .all(connection) + .await?; + + Ok(counts + .into_iter() + .map(|(id, count)| (id, count as u64)) + .collect()) +} + +/// Batch loads the describing package version per SBOM. +async fn load_describing_versions( + sbom_ids: &[Uuid], + connection: &impl ConnectionTrait, +) -> Result>, Error> { + if sbom_ids.is_empty() { + return Ok(HashMap::new()); + } + + let results: Vec<(Uuid, Option)> = package_relates_to_package::Entity::find() + .join( + JoinType::Join, + package_relates_to_package::Relation::RightPackage.def(), + ) + .filter(package_relates_to_package::Column::SbomId.is_in(sbom_ids.iter().copied())) + .filter(package_relates_to_package::Column::Relationship.eq(Relationship::Describes)) + .select_only() + .column(package_relates_to_package::Column::SbomId) + .column(sbom_package::Column::Version) + .into_tuple() + .all(connection) + .await?; + + let mut map = HashMap::with_capacity(results.len()); + for (sbom_id, version) in results { + map.entry(sbom_id).or_insert(version); + } + Ok(map) +} + +/// Batch counts advisory_vulnerability entries per advisory. +async fn load_advisory_vuln_counts( + advisory_ids: &[Uuid], + connection: &impl ConnectionTrait, +) -> Result, Error> { + if advisory_ids.is_empty() { + return Ok(HashMap::new()); + } + + let counts: Vec<(Uuid, i64)> = advisory_vulnerability::Entity::find() + .filter(advisory_vulnerability::Column::AdvisoryId.is_in(advisory_ids.iter().copied())) + .select_only() + .column(advisory_vulnerability::Column::AdvisoryId) + .column_as( + Expr::col(advisory_vulnerability::Column::VulnerabilityId).count(), + "count", + ) + .group_by(advisory_vulnerability::Column::AdvisoryId) + .into_tuple() + .all(connection) + .await?; + + Ok(counts + .into_iter() + .map(|(id, count)| (id, count as u64)) + .collect()) +} + +/// Builds an SbomHead from batch-loaded components, avoiding N+1 COUNT queries. +fn build_sbom_head_from_parts( + sbom_model: &sbom::Model, + sbom_node_model: &sbom_node::Model, + package_count: u64, +) -> SbomHead { + SbomHead { + id: sbom_model.sbom_id, + document_id: sbom_model.document_id.clone(), + labels: sbom_model.labels.clone(), + published: sbom_model.published, + authors: sbom_model.authors.clone(), + suppliers: sbom_model.suppliers.clone(), + name: sbom_node_model.name.clone(), + data_licenses: sbom_model.data_licenses.clone(), + number_of_packages: package_count, + } +} + +/// Hydrates in-memory vulnerability correlation matches into VulnerabilityAdvisorySummary entries. +/// +/// Groups matches by advisory, builds per-SBOM status entries with batch-loaded +/// metadata (sbom heads, package counts, describing versions). Falls back to +/// purl-level data from the index when no SBOM matches exist. +#[allow(deprecated)] +#[instrument(skip_all, err(level = tracing::Level::INFO))] +pub async fn hydrate_vulnerability_advisories( + _vulnerability: &vulnerability::Model, + advisory_vulnerabilities: &[advisory_vulnerability::Model], + vuln_scores: &[advisory_vulnerability_score::Model], + matches: Vec, + vuln_entries: &[VulnIndexEntry], + statuses: &HashMap>, + connection: &impl ConnectionTrait, +) -> Result, Error> { + let advisory_ids: Vec = advisory_vulnerabilities + .iter() + .map(|av| av.advisory_id) + .collect::>() + .into_iter() + .collect(); + + let sbom_ids: Vec = matches + .iter() + .map(|m| m.sbom_id) + .collect::>() + .into_iter() + .collect(); + + let mut cpe_ids = HashSet::new(); + for m in &matches { + if let Some(cpe_id) = m.context_cpe_id { + cpe_ids.insert(cpe_id); + } + } + if matches.is_empty() { + for entry in vuln_entries { + if let Some(cpe_id) = entry.context_cpe_id { + cpe_ids.insert(cpe_id); + } + } + } + + // Batch load advisory metadata, SBOM data, and CPEs in parallel + let (advisory_models, vuln_counts, cpe_models, sbom_models, pkg_counts, describing_versions) = + tokio::try_join!( + load_advisories(&advisory_ids, connection), + load_advisory_vuln_counts(&advisory_ids, connection), + load_cpes(&cpe_ids, connection), + load_sboms(&sbom_ids, connection), + load_package_counts(&sbom_ids, connection), + load_describing_versions(&sbom_ids, connection), + )?; + + // Load sbom_nodes (composite key lookup depends on sbom models) + let sbom_node_map = load_describing_sbom_nodes(&sbom_models, connection).await?; + + // Build advisory heads + let advisory_heads = AdvisoryHead::from_entities(&advisory_models, connection).await?; + let advisory_head_map: HashMap = advisory_models + .iter() + .zip(advisory_heads) + .map(|(model, head)| (model.id, head)) + .collect(); + + let sbom_map: HashMap = + sbom_models.iter().map(|s| (s.sbom_id, s)).collect(); + + let cpe_map: HashMap = cpe_models.into_iter().map(|c| (c.id, c)).collect(); + + // Group scores by advisory_id + let mut score_map: HashMap> = HashMap::new(); + for score in vuln_scores { + score_map + .entry(score.advisory_id) + .or_default() + .push(score.clone()); + } + + // Group SBOM matches: advisory_id → sbom_id → (status_slug → PurlSummary set) + let mut advisory_sbom_groups: HashMap< + Uuid, + HashMap>>, + > = HashMap::new(); + + for m in &matches { + let status_slug = statuses + .get(&m.status_id) + .map(|s| s.as_ref().to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + let purl = Purl { + ty: m.purl_key.ty.to_string(), + namespace: m.purl_key.namespace.as_ref().map(|ns| ns.to_string()), + name: m.purl_key.name.to_string(), + version: Some(m.version.to_string()), + qualifiers: Default::default(), + }; + + advisory_sbom_groups + .entry(m.advisory_id) + .or_default() + .entry(m.sbom_id) + .or_default() + .entry(status_slug) + .or_default() + .insert(PurlSummary::from(purl)); + } + + // Build the purl fallback data (only when no SBOM matches exist) + let purl_fallback: HashMap>> = + if matches.is_empty() { + build_purl_fallback(vuln_entries, statuses, &cpe_map) + } else { + HashMap::new() + }; + + // Assemble VulnerabilityAdvisorySummary per advisory + let mut summaries = Vec::with_capacity(advisory_vulnerabilities.len()); + + for av in advisory_vulnerabilities { + let head = match advisory_head_map.get(&av.advisory_id) { + Some(h) => h.clone(), + None => continue, + }; + + let scores: Vec = score_map + .get(&av.advisory_id) + .cloned() + .unwrap_or_default() + .into_iter() + .map(ScoredVector::from) + .collect(); + + let number_of_vulnerabilities = vuln_counts.get(&av.advisory_id).copied().unwrap_or(0); + + // Build SBOM statuses from correlation matches + let sboms = if let Some(sbom_groups) = advisory_sbom_groups.get(&av.advisory_id) { + let mut sbom_statuses = Vec::with_capacity(sbom_groups.len()); + for (&sid, purl_groups) in sbom_groups { + let (Some(sm), Some(sn)) = (sbom_map.get(&sid), sbom_node_map.get(&sid)) else { + continue; + }; + let pkg_count = pkg_counts.get(&sid).copied().unwrap_or(0); + let version = describing_versions.get(&sid).cloned().flatten(); + + sbom_statuses.push(VulnerabilitySbomStatus { + head: build_sbom_head_from_parts(sm, sn, pkg_count), + version, + purl_statuses: purl_groups.clone(), + }); + } + sbom_statuses + } else { + Vec::new() + }; + + let purls = purl_fallback + .get(&av.advisory_id) + .cloned() + .unwrap_or_default(); + + summaries.push(VulnerabilityAdvisorySummary { + head: VulnerabilityAdvisoryHead { head, scores }, + purls, + sboms, + number_of_vulnerabilities, + }); + } + + Ok(summaries) +} + +/// Builds VulnerabilityAdvisoryStatus entries from raw index data for the purl fallback. +/// +/// Used when correlate_vulnerability() finds no SBOM matches, replicating +/// the legacy behavior of showing raw purl status claims. +fn build_purl_fallback( + vuln_entries: &[VulnIndexEntry], + statuses: &HashMap>, + cpe_map: &HashMap, +) -> HashMap>> { + let mut result: HashMap>> = + HashMap::new(); + + for entry in vuln_entries { + let VulnEntrySource::Purl { + purl_key, + version_range, + } = &entry.source + else { + continue; + }; + + let status_slug = statuses + .get(&entry.status_id) + .map(|s| s.as_ref().to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + let base_purl = Purl { + ty: purl_key.ty.to_string(), + namespace: purl_key.namespace.as_ref().map(|ns| ns.to_string()), + name: purl_key.name.to_string(), + version: None, + qualifiers: Default::default(), + }; + + let context = entry.context_cpe_id.and_then(|cpe_id| { + cpe_map + .get(&cpe_id) + .map(|c| StatusContext::Cpe(c.to_string())) + }); + + result + .entry(entry.advisory_id) + .or_default() + .entry(status_slug) + .or_default() + .push(VulnerabilityAdvisoryStatus { + base_purl: BasePurlHead { + uuid: base_purl.package_uuid(), + purl: base_purl, + }, + version: format_version_range(version_range), + context, + }); + } + + result +} + +/// Hydrates recommend matches into RecommendEntry values. +/// +/// For each winner PURL string, resolves vulnerability IDs, status slugs, and +/// remediations from the database. Deduplicates by vulnerability, keeping the +/// match from the most recently modified advisory. +#[instrument(skip_all, err(level = tracing::Level::INFO))] +pub async fn hydrate_recommend_matches( + matches_by_purl: HashMap>, + statuses: &HashMap>, + connection: &impl ConnectionTrait, +) -> Result>, Error> { + let mut result = HashMap::with_capacity(matches_by_purl.len()); + + let all_matches: Vec<&PurlCorrelationMatch> = + matches_by_purl.values().flat_map(|v| v.iter()).collect(); + + if all_matches.is_empty() { + for key in matches_by_purl.keys() { + result.insert(key.clone(), Vec::new()); + } + return Ok(result); + } + + let mut advisory_ids = HashSet::new(); + let mut purl_status_ids = HashSet::new(); + for m in &all_matches { + advisory_ids.insert(m.advisory_id); + if let Some(id) = m.purl_status_id { + purl_status_ids.insert(id); + } + } + + let advisory_id_vec: Vec = advisory_ids.into_iter().collect(); + + let (advisory_models, remediation_map) = tokio::try_join!( + load_advisories(&advisory_id_vec, connection), + load_purl_remediations(&purl_status_ids, connection), + )?; + + let advisory_date_map: HashMap> = advisory_models + .into_iter() + .map(|a| (a.id, a.modified.or(a.published))) + .collect(); + + for (purl_string, matches) in &matches_by_purl { + if matches.is_empty() { + result.insert(purl_string.clone(), Vec::new()); + continue; + } + + // Dedup by vulnerability: keep match from most recent advisory + let mut best_by_vuln: HashMap<&str, &PurlCorrelationMatch> = HashMap::new(); + for m in matches { + best_by_vuln + .entry(m.vulnerability_id.as_ref()) + .and_modify(|existing| { + let existing_date = advisory_date_map + .get(&existing.advisory_id) + .copied() + .flatten(); + let new_date = advisory_date_map.get(&m.advisory_id).copied().flatten(); + if new_date > existing_date { + *existing = m; + } + }) + .or_insert(m); + } + + let vulnerabilities = best_by_vuln + .into_values() + .map(|m| { + let slug = statuses + .get(&m.status_id) + .map(|s| s.as_ref()) + .unwrap_or("unknown"); + + let vex_status = match slug { + "affected" => VexStatus::Affected, + "fixed" => VexStatus::Fixed, + "not_affected" => VexStatus::NotAffected, + "under_investigation" => VexStatus::UnderInvestigation, + "recommended" => VexStatus::Recommended, + other => VexStatus::Other(other.to_string()), + }; + + let remediations = m + .purl_status_id + .and_then(|id| remediation_map.get(&id)) + .cloned() + .unwrap_or_default(); + + VulnerabilityStatus { + id: m.vulnerability_id.as_ref().to_string(), + status: Some(vex_status), + justification: None, + remediations, + } + }) + .collect(); + + let entry = RecommendEntry { + package: purl_string.clone(), + vulnerabilities, + }; + + result.insert(purl_string.clone(), vec![entry]); + } + + Ok(result) +} diff --git a/modules/correlation/src/service/load.rs b/modules/correlation/src/service/load.rs new file mode 100644 index 000000000..ed8c4df9d --- /dev/null +++ b/modules/correlation/src/service/load.rs @@ -0,0 +1,732 @@ +use crate::model::{ + AdvisoryIndex, AdvisoryPatch, CorrelationState, ProductStatusEntry, PurlKey, PurlStatusEntry, + SbomIndex, SbomPackageEntry, SbomPatch, SeverityIndex, VersionRangeData, VulnEntrySource, + VulnIndexEntry, +}; +use futures::TryStreamExt; +use sea_orm::{ + ColumnTrait, ConnectionTrait, EntityTrait, FromQueryResult, JoinType, QueryFilter, QuerySelect, + RelationTrait, StreamTrait, +}; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use tracing::{Instrument, info_span, instrument}; +use trustify_common::db::ReadOnly; +use trustify_entity::advisory_vulnerability_score::Severity; +use trustify_entity::version_scheme::VersionScheme; +use trustify_entity::{ + advisory, advisory_vulnerability_score, base_purl, product_status, purl_status, qualified_purl, + sbom_node_purl_ref, status, version_range, +}; +use uuid::Uuid; + +/// Deduplicates strings into `Arc` to reduce heap allocations. +/// +/// Strings that appear across many rows (purl types, namespaces, vulnerability IDs, +/// version bounds) are interned so identical values share a single allocation. +struct StringInterner(HashMap>); + +impl StringInterner { + fn new() -> Self { + Self(HashMap::new()) + } + + /// Interns a string, returning a shared reference. + fn intern(&mut self, s: String) -> Arc { + if let Some(existing) = self.0.get(s.as_str()) { + Arc::clone(existing) + } else { + let arc: Arc = Arc::from(s.as_str()); + self.0.insert(s, Arc::clone(&arc)); + arc + } + } + + /// Interns an optional string. + fn intern_opt(&mut self, s: Option) -> Option> { + s.map(|s| self.intern(s)) + } +} + +/// Loads the complete correlation state from the database. +/// +/// Advisory and SBOM indexes are loaded sequentially to avoid doubling peak +/// memory from parallel materialization. Each uses streaming cursors to avoid +/// intermediate Vecs. +#[instrument(skip_all, err(level = tracing::Level::INFO))] +pub async fn load_all(db: &ReadOnly) -> Result { + let txn = db.begin().await?; + + let advisory_index = load_advisory_index(&txn) + .instrument(info_span!("load advisory index")) + .await?; + + let sbom_index = load_sbom_index(&txn) + .instrument(info_span!("load sbom index")) + .await?; + + tracing::info!( + purl_entries = advisory_index.by_purl.len(), + statuses = advisory_index.statuses.len(), + product_entries = advisory_index.product_by_name.len(), + sboms = sbom_index.by_sbom.len(), + cpe_sboms = sbom_index.describing_cpes.len(), + "correlation state loaded" + ); + + Ok(CorrelationState { + advisory_index, + sbom_index, + }) +} + +/// Raw row for purl_status + version_range + base_purl join. +#[derive(Debug, FromQueryResult)] +struct PurlStatusRow { + purl_status_id: Uuid, + advisory_id: Uuid, + vulnerability_id: String, + status_id: Uuid, + purl_type: String, + purl_namespace: Option, + purl_name: String, + context_cpe_id: Option, + version_scheme_id: VersionScheme, + low_version: Option, + low_inclusive: Option, + high_version: Option, + high_inclusive: Option, +} + +/// Raw row for advisory_vulnerability_score (severity lookup). +#[derive(Debug, FromQueryResult)] +struct SeverityRow { + advisory_id: Uuid, + vulnerability_id: String, + severity: Severity, +} + +/// Raw row for SBOM describing CPEs. +#[derive(Debug, FromQueryResult)] +struct SbomCpeRow { + sbom_id: Uuid, + cpe_id: Uuid, +} + +/// Loads the full advisory index using streaming cursors. +/// +/// Queries run sequentially within a single transaction to share the server-side cursor. +/// Deprecated advisories are filtered out at the SQL level. +pub(crate) async fn load_advisory_index( + txn: &(impl ConnectionTrait + StreamTrait), +) -> Result { + let mut interner = StringInterner::new(); + + // Stream purl_status rows and build the by_purl index incrementally + let mut by_purl: HashMap> = HashMap::new(); + let mut purl_row_count: u64 = 0; + + let mut stream = purl_status::Entity::find() + .join( + JoinType::InnerJoin, + purl_status::Relation::VersionRange.def(), + ) + .join(JoinType::InnerJoin, purl_status::Relation::BasePurl.def()) + .join(JoinType::InnerJoin, purl_status::Relation::Advisory.def()) + .filter(advisory::Column::Deprecated.eq(false)) + .select_only() + .column_as(purl_status::Column::Id, "purl_status_id") + .column(purl_status::Column::AdvisoryId) + .column(purl_status::Column::VulnerabilityId) + .column(purl_status::Column::StatusId) + .column(purl_status::Column::ContextCpeId) + .column_as(base_purl::Column::Type, "purl_type") + .column_as(base_purl::Column::Namespace, "purl_namespace") + .column_as(base_purl::Column::Name, "purl_name") + .column(version_range::Column::VersionSchemeId) + .column(version_range::Column::LowVersion) + .column(version_range::Column::LowInclusive) + .column(version_range::Column::HighVersion) + .column(version_range::Column::HighInclusive) + .into_model::() + .stream(txn) + .await?; + + while let Some(row) = stream.try_next().await? { + purl_row_count += 1; + let key = PurlKey { + ty: interner.intern(row.purl_type), + namespace: interner.intern_opt(row.purl_namespace), + name: interner.intern(row.purl_name), + }; + by_purl.entry(key).or_default().push(PurlStatusEntry { + purl_status_id: row.purl_status_id, + advisory_id: row.advisory_id, + vulnerability_id: interner.intern(row.vulnerability_id), + status_id: row.status_id, + version_range: build_version_range( + &mut interner, + row.version_scheme_id, + row.low_version, + row.low_inclusive.unwrap_or(true), + row.high_version, + row.high_inclusive.unwrap_or(false), + ), + context_cpe_id: row.context_cpe_id, + }); + } + drop(stream); + + tracing::info!( + purl_keys = by_purl.len(), + purl_status_rows = purl_row_count, + interned_strings = interner.0.len(), + "advisory purl_status loaded" + ); + + // Status table is small — load all at once + let status_rows = status::Entity::find() + .all(txn) + .instrument(info_span!("load statuses")) + .await?; + + let statuses: HashMap<_, _> = status_rows + .into_iter() + .map(|r| (r.id, interner.intern(r.slug))) + .collect(); + + // Stream product_status rows and build the product_by_name index + let mut product_by_name: HashMap, Vec> = HashMap::new(); + let mut product_row_count: u64 = 0; + + let mut stream = product_status::Entity::find() + .join( + JoinType::InnerJoin, + product_status::Relation::Advisory.def(), + ) + .filter(advisory::Column::Deprecated.eq(false)) + .filter(product_status::Column::Package.is_not_null()) + .stream(txn) + .await?; + + while let Some(row) = stream.try_next().await? { + if let Some(pkg) = row.package { + product_row_count += 1; + product_by_name + .entry(interner.intern(pkg)) + .or_default() + .push(ProductStatusEntry { + product_status_id: row.id, + advisory_id: row.advisory_id, + vulnerability_id: interner.intern(row.vulnerability_id), + status_id: row.status_id, + context_cpe_id: row.context_cpe_id, + }); + } + } + drop(stream); + + tracing::info!( + product_keys = product_by_name.len(), + product_rows = product_row_count, + "advisory product_status loaded" + ); + + // Load severity index from advisory_vulnerability_score + let severity = load_severity_index(&mut interner, txn) + .instrument(info_span!("load severity index")) + .await?; + + tracing::info!(severity_entries = severity.len(), "severity index loaded"); + + // Build reverse vulnerability index from by_purl and product_by_name + let by_vulnerability = build_vulnerability_index(&by_purl, &product_by_name); + + tracing::info!( + vulnerability_keys = by_vulnerability.len(), + "vulnerability reverse index built" + ); + + Ok(AdvisoryIndex { + by_purl, + product_by_name, + statuses, + severity, + by_vulnerability, + }) +} + +/// Loads the SBOM index by joining sbom_node_purl_ref with qualified_purl, +/// building per-SBOM package vectors directly. +/// The CPE query uses a CTE with a self-join and `split_part()` — kept as raw SQL. +pub(crate) async fn load_sbom_index( + txn: &(impl ConnectionTrait + StreamTrait), +) -> Result { + let mut interner = StringInterner::new(); + + let mut by_sbom_build: HashMap> = HashMap::new(); + let mut seen_per_sbom: HashMap> = HashMap::new(); + let mut ref_count: u64 = 0; + + let rows: Vec<(sbom_node_purl_ref::Model, Option)> = + sbom_node_purl_ref::Entity::find() + .find_also_related(qualified_purl::Entity) + .all(txn) + .instrument(info_span!("load sbom purl refs")) + .await?; + + for (snpr, qp_opt) in rows { + if let Some(qp) = qp_opt + && let Some(version) = qp.purl.version + && !version.is_empty() + { + let seen = seen_per_sbom.entry(snpr.sbom_id).or_default(); + if !seen.insert(qp.id) { + continue; + } + ref_count += 1; + by_sbom_build + .entry(snpr.sbom_id) + .or_default() + .push(SbomPackageEntry { + ty: interner.intern(qp.purl.ty), + name: interner.intern(qp.purl.name), + namespace: interner.intern_opt(qp.purl.namespace), + version: interner.intern(version), + }); + } + } + drop(seen_per_sbom); + + let by_sbom: HashMap> = by_sbom_build + .into_iter() + .map(|(id, pkgs)| (id, Arc::from(pkgs.into_boxed_slice()))) + .collect(); + + tracing::info!( + sboms = by_sbom.len(), + purl_refs = ref_count, + interned_strings = interner.0.len(), + "per-SBOM package index built" + ); + + // Load CPE IDs per SBOM (direct + generalized, matching v3a SQL logic). + // Kept as raw SQL — CTE with self-join and split_part() doesn't map to SeaORM. + let mut describing_cpes: HashMap> = HashMap::new(); + + let mut stream = SbomCpeRow::find_by_statement(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + r#" + WITH filtered AS ( + SELECT sdc.sbom_id, cpe.id AS cpe_id, cpe.vendor, cpe.product, cpe.version + FROM sbom_describing_cpe sdc + JOIN cpe ON sdc.cpe_id = cpe.id + ), + generalized AS ( + SELECT f.sbom_id, c.id AS cpe_id + FROM filtered f + JOIN cpe c ON c.vendor = f.vendor + AND c.product = f.product + AND c.version = split_part(f.version, '.', 1) + AND (c.edition IS NULL OR c.edition = '*') + ) + SELECT sbom_id, cpe_id FROM filtered + UNION + SELECT sbom_id, cpe_id FROM generalized + "# + .to_string(), + )) + .stream(txn) + .await?; + + while let Some(row) = stream.try_next().await? { + describing_cpes + .entry(row.sbom_id) + .or_default() + .insert(row.cpe_id); + } + drop(stream); + + tracing::info!(cpe_sboms = describing_cpes.len(), "sbom cpes loaded"); + + // Build reverse PurlKey → sbom_ids index + let mut by_purl_key: HashMap> = HashMap::new(); + for (&sbom_id, packages) in &by_sbom { + for pkg in packages.iter() { + let key = PurlKey { + ty: Arc::clone(&pkg.ty), + namespace: pkg.namespace.as_ref().map(Arc::clone), + name: Arc::clone(&pkg.name), + }; + by_purl_key.entry(key).or_default().push(sbom_id); + } + } + + tracing::info!( + purl_key_entries = by_purl_key.len(), + "sbom by_purl_key reverse index built" + ); + + Ok(SbomIndex { + by_sbom, + describing_cpes, + by_purl_key, + }) +} + +/// Loads advisory patches for a batch of advisory IDs. +/// +/// Returns one AdvisoryPatch per advisory that has data. Uses SeaORM query builder +/// with `.is_in()` for parameter binding. +#[instrument(skip_all, fields(count = ids.len()), err(level = tracing::Level::INFO))] +pub(crate) async fn load_advisory_patches( + ids: &[Uuid], + txn: &impl ConnectionTrait, +) -> Result, anyhow::Error> { + if ids.is_empty() { + return Ok(HashMap::new()); + } + + let mut interner = StringInterner::new(); + + let purl_rows = purl_status::Entity::find() + .join( + JoinType::InnerJoin, + purl_status::Relation::VersionRange.def(), + ) + .join(JoinType::InnerJoin, purl_status::Relation::BasePurl.def()) + .join(JoinType::InnerJoin, purl_status::Relation::Advisory.def()) + .filter(purl_status::Column::AdvisoryId.is_in(ids.iter().copied())) + .filter(advisory::Column::Deprecated.eq(false)) + .select_only() + .column_as(purl_status::Column::Id, "purl_status_id") + .column(purl_status::Column::AdvisoryId) + .column(purl_status::Column::VulnerabilityId) + .column(purl_status::Column::StatusId) + .column(purl_status::Column::ContextCpeId) + .column_as(base_purl::Column::Type, "purl_type") + .column_as(base_purl::Column::Namespace, "purl_namespace") + .column_as(base_purl::Column::Name, "purl_name") + .column(version_range::Column::VersionSchemeId) + .column(version_range::Column::LowVersion) + .column(version_range::Column::LowInclusive) + .column(version_range::Column::HighVersion) + .column(version_range::Column::HighInclusive) + .into_model::() + .all(txn) + .instrument(info_span!("load advisory purl_status patches")) + .await?; + + let product_rows = product_status::Entity::find() + .join( + JoinType::InnerJoin, + product_status::Relation::Advisory.def(), + ) + .filter(product_status::Column::AdvisoryId.is_in(ids.iter().copied())) + .filter(advisory::Column::Deprecated.eq(false)) + .filter(product_status::Column::Package.is_not_null()) + .all(txn) + .instrument(info_span!("load advisory product_status patches")) + .await?; + + let mut patches: HashMap = HashMap::new(); + + for row in purl_rows { + let key = PurlKey { + ty: interner.intern(row.purl_type), + namespace: interner.intern_opt(row.purl_namespace), + name: interner.intern(row.purl_name), + }; + patches + .entry(row.advisory_id) + .or_default() + .purl_statuses + .entry(key) + .or_default() + .push(PurlStatusEntry { + purl_status_id: row.purl_status_id, + advisory_id: row.advisory_id, + vulnerability_id: interner.intern(row.vulnerability_id), + status_id: row.status_id, + version_range: build_version_range( + &mut interner, + row.version_scheme_id, + row.low_version, + row.low_inclusive.unwrap_or(true), + row.high_version, + row.high_inclusive.unwrap_or(false), + ), + context_cpe_id: row.context_cpe_id, + }); + } + + for row in product_rows { + if let Some(pkg) = row.package { + patches + .entry(row.advisory_id) + .or_default() + .product_statuses + .entry(interner.intern(pkg)) + .or_default() + .push(ProductStatusEntry { + product_status_id: row.id, + advisory_id: row.advisory_id, + vulnerability_id: interner.intern(row.vulnerability_id), + status_id: row.status_id, + context_cpe_id: row.context_cpe_id, + }); + } + } + + // Load severity data for these advisories + let severity_rows = advisory_vulnerability_score::Entity::find() + .filter(advisory_vulnerability_score::Column::AdvisoryId.is_in(ids.iter().copied())) + .select_only() + .column(advisory_vulnerability_score::Column::AdvisoryId) + .column(advisory_vulnerability_score::Column::VulnerabilityId) + .column(advisory_vulnerability_score::Column::Severity) + .into_model::() + .all(txn) + .instrument(info_span!("load advisory severity patches")) + .await?; + + for row in severity_rows { + let vuln_id = interner.intern(row.vulnerability_id); + let affected = crate::model::severity_to_affected(row.severity); + let patch = patches.entry(row.advisory_id).or_default(); + patch + .severity + .entry((row.advisory_id, vuln_id)) + .and_modify(|existing| { + if affected > *existing { + *existing = affected; + } + }) + .or_insert(affected); + } + + Ok(patches) +} + +/// Loads SBOM patches for a batch of SBOM IDs. +/// +/// Returns one SbomPatch per SBOM that has data. Uses SeaORM query builder +/// for packages and raw SQL for the CPE CTE query. +#[instrument(skip_all, fields(count = ids.len()), err(level = tracing::Level::INFO))] +pub(crate) async fn load_sbom_patches( + ids: &[Uuid], + txn: &impl ConnectionTrait, +) -> Result, anyhow::Error> { + if ids.is_empty() { + return Ok(HashMap::new()); + } + + let mut interner = StringInterner::new(); + + let pkg_rows: Vec<(sbom_node_purl_ref::Model, Option)> = + sbom_node_purl_ref::Entity::find() + .filter(sbom_node_purl_ref::Column::SbomId.is_in(ids.iter().copied())) + .find_also_related(qualified_purl::Entity) + .all(txn) + .instrument(info_span!("load sbom package patches")) + .await?; + + let placeholders = build_placeholders(ids.len()); + let values: Vec = ids.iter().copied().map(Into::into).collect(); + + let cpe_rows = SbomCpeRow::find_by_statement(sea_orm::Statement::from_sql_and_values( + sea_orm::DatabaseBackend::Postgres, + format!( + r#" + WITH filtered AS ( + SELECT sdc.sbom_id, cpe.id AS cpe_id, cpe.vendor, cpe.product, cpe.version + FROM sbom_describing_cpe sdc + JOIN cpe ON sdc.cpe_id = cpe.id + WHERE sdc.sbom_id IN ({placeholders}) + ), + generalized AS ( + SELECT f.sbom_id, c.id AS cpe_id + FROM filtered f + JOIN cpe c ON c.vendor = f.vendor + AND c.product = f.product + AND c.version = split_part(f.version, '.', 1) + AND (c.edition IS NULL OR c.edition = '*') + ) + SELECT sbom_id, cpe_id FROM filtered + UNION + SELECT sbom_id, cpe_id FROM generalized + "#, + ), + values, + )) + .all(txn) + .instrument(info_span!("load sbom cpe patches")) + .await?; + + let mut patches: HashMap = HashMap::new(); + + for (snpr, qp_opt) in pkg_rows { + if let Some(qp) = qp_opt + && let Some(version) = qp.purl.version + && !version.is_empty() + { + patches + .entry(snpr.sbom_id) + .or_default() + .packages + .push(SbomPackageEntry { + ty: interner.intern(qp.purl.ty), + name: interner.intern(qp.purl.name), + namespace: interner.intern_opt(qp.purl.namespace), + version: interner.intern(version), + }); + } + } + + for row in cpe_rows { + patches + .entry(row.sbom_id) + .or_default() + .describing_cpes + .insert(row.cpe_id); + } + + Ok(patches) +} + +/// Loads max severity per (advisory_id, vulnerability_id) from advisory_vulnerability_score. +/// +/// For each pair, keeps the highest severity using the CVSS ranking order. +async fn load_severity_index( + interner: &mut StringInterner, + txn: &(impl ConnectionTrait + StreamTrait), +) -> Result { + let mut severity: SeverityIndex = HashMap::new(); + + let mut stream = advisory_vulnerability_score::Entity::find() + .select_only() + .column(advisory_vulnerability_score::Column::AdvisoryId) + .column(advisory_vulnerability_score::Column::VulnerabilityId) + .column(advisory_vulnerability_score::Column::Severity) + .into_model::() + .stream(txn) + .await?; + + while let Some(row) = stream.try_next().await? { + let vuln_id = interner.intern(row.vulnerability_id); + let affected = crate::model::severity_to_affected(row.severity); + let key = (row.advisory_id, vuln_id); + severity + .entry(key) + .and_modify(|existing| { + if affected > *existing { + *existing = affected; + } + }) + .or_insert(affected); + } + drop(stream); + + Ok(severity) +} + +/// Builds the reverse vulnerability index from the purl and product indexes. +fn build_vulnerability_index( + by_purl: &HashMap>, + product_by_name: &HashMap, Vec>, +) -> HashMap, Vec> { + let mut by_vulnerability: HashMap, Vec> = HashMap::new(); + + for (purl_key, entries) in by_purl { + for entry in entries { + by_vulnerability + .entry(Arc::clone(&entry.vulnerability_id)) + .or_default() + .push(VulnIndexEntry { + advisory_id: entry.advisory_id, + status_id: entry.status_id, + context_cpe_id: entry.context_cpe_id, + source: VulnEntrySource::Purl { + purl_key: purl_key.clone(), + version_range: entry.version_range.clone(), + }, + }); + } + } + + for (package_name, entries) in product_by_name { + for entry in entries { + by_vulnerability + .entry(Arc::clone(&entry.vulnerability_id)) + .or_default() + .push(VulnIndexEntry { + advisory_id: entry.advisory_id, + status_id: entry.status_id, + context_cpe_id: entry.context_cpe_id, + source: VulnEntrySource::Product { + package_name: Arc::clone(package_name), + }, + }); + } + } + + by_vulnerability +} + +/// Builds a comma-separated placeholder list ($1, $2, ..., $n) for raw SQL queries. +fn build_placeholders(count: usize) -> String { + (1..=count) + .map(|i| format!("${i}")) + .collect::>() + .join(", ") +} + +/// Builds a `VersionRangeData` with pre-parsed semver boundaries when applicable. +fn build_version_range( + interner: &mut StringInterner, + scheme: VersionScheme, + low_version: Option, + low_inclusive: bool, + high_version: Option, + high_inclusive: bool, +) -> VersionRangeData { + let (low_parsed, high_parsed) = if is_semver_family(scheme) { + ( + low_version + .as_deref() + .and_then(|v| lenient_semver::parse(v).ok()), + high_version + .as_deref() + .and_then(|v| lenient_semver::parse(v).ok()), + ) + } else { + (None, None) + }; + + VersionRangeData { + version_scheme: scheme, + low_version: interner.intern_opt(low_version), + low_inclusive, + high_version: interner.intern_opt(high_version), + high_inclusive, + low_parsed, + high_parsed, + } +} + +/// Returns true for version schemes that use semver-style comparison. +fn is_semver_family(scheme: VersionScheme) -> bool { + matches!( + scheme, + VersionScheme::Semver + | VersionScheme::Npm + | VersionScheme::Gem + | VersionScheme::NuGet + | VersionScheme::Packagist + | VersionScheme::Hex + | VersionScheme::Swift + | VersionScheme::Pub + | VersionScheme::Cargo + | VersionScheme::Golang + ) +} diff --git a/modules/correlation/src/service/mod.rs b/modules/correlation/src/service/mod.rs new file mode 100644 index 000000000..017e2edfd --- /dev/null +++ b/modules/correlation/src/service/mod.rs @@ -0,0 +1,668 @@ +pub mod hydrate; +mod load; + +#[cfg(test)] +mod test; + +use crate::{ + Error, + config::CorrelationConfig, + model::{ + AdvisoryIndex, CorrelationMatch, CorrelationState, PurlCorrelationMatch, PurlKey, + SbomIndex, VulnCorrelationMatch, VulnEntrySource, VulnIndexEntry, + }, +}; +use arc_swap::ArcSwap; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::Duration; +use tokio::{sync::mpsc, task::JoinHandle}; +use tracing::{Instrument, info_span, instrument}; +use trustify_common::db::change::ChangeListener; +use trustify_common::db::{ReadOnly, ReadWrite, change::ChangeEntity}; +use trustify_common::purl::Purl; +use trustify_module_fundamental::sbom::model::{AffectedSeverity, SbomAdvisorySummary}; +use uuid::Uuid; + +/// Events that trigger incremental state updates in the background loader. +#[derive(Debug)] +pub enum CorrelationEvent { + /// A specific advisory was ingested, updated, or deleted. + AdvisoryChanged(Uuid), + /// A specific SBOM was ingested, updated, or deleted. + SbomChanged(Uuid), +} + +/// In-memory correlation service for fast advisory-SBOM matching. +/// +/// Advisory and SBOM indexes are stored in separate `ArcSwap` instances so that +/// incremental updates only clone the index that changed. +#[derive(Clone)] +pub struct CorrelationService { + advisory_state: Arc>, + sbom_state: Arc>, + _db: ReadOnly, + tx: mpsc::UnboundedSender, + _loader: Arc>, + _listener: Arc>, +} + +impl CorrelationService { + /// Creates and starts the correlation service, loading initial state from the database. + /// + /// The `db_rw` parameter provides the PostgreSQL connection for LISTEN/NOTIFY; + /// it fails fast at startup if the backend is not PostgreSQL. + pub async fn new( + config: &CorrelationConfig, + db_ro: ReadOnly, + db_rw: &ReadWrite, + ) -> Result { + let (tx, rx) = mpsc::unbounded_channel(); + + // Initial full load + let initial = load::load_all(&db_ro) + .instrument(info_span!("correlation initial load")) + .await?; + + tracing::info!("correlation service initial load complete"); + + let advisory_state = Arc::new(ArcSwap::from_pointee(initial.advisory_index)); + let sbom_state = Arc::new(ArcSwap::from_pointee(initial.sbom_index)); + + let loader_advisory = advisory_state.clone(); + let loader_sbom = sbom_state.clone(); + let loader_db = db_ro.clone(); + + let debounce = Duration::from_secs(config.correlation_debounce_secs); + let _loader = Arc::new(tokio::spawn(Self::background_loader( + loader_advisory, + loader_sbom, + loader_db, + rx, + debounce, + ))); + + // Spawn the change listener (LISTEN/NOTIFY + polling fallback) + let change_listener = ChangeListener::new(db_rw, Duration::from_secs(86400))?; + let poll_interval = Duration::from_secs(config.correlation_poll_interval_secs); + let listener_tx = tx.clone(); + + let _listener = Arc::new(tokio::spawn(async move { + change_listener + .with_poll_interval(poll_interval) + .run(move |entries| { + for entry in entries { + if let Some(id) = entry.id { + let event = match entry.r#type { + ChangeEntity::Advisory => { + CorrelationEvent::AdvisoryChanged(id) + } + ChangeEntity::Sbom => CorrelationEvent::SbomChanged(id), + }; + let _ = listener_tx.send(event); + } + } + }) + .await; + })); + + Ok(Self { + advisory_state, + sbom_state, + _db: db_ro, + tx, + _loader, + _listener, + }) + } + + /// Returns the current correlation state for inspection. + pub fn state(&self) -> CorrelationState { + let advisory_index = self.advisory_state.load(); + let sbom_index = self.sbom_state.load(); + CorrelationState { + advisory_index: (**advisory_index).clone(), + sbom_index: (**sbom_index).clone(), + } + } + + /// Sends a local event (for tests or manual triggers). + pub fn notify_local(&self, event: CorrelationEvent) { + if self.tx.send(event).is_err() { + tracing::warn!("correlation event channel closed"); + } + } + + /// Returns the current advisory status slug map (status_id → slug). + pub fn status_slugs(&self) -> HashMap> { + self.advisory_state.load().statuses.clone() + } + + /// Finds all advisories that affect the given SBOM. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub fn correlate_sbom(&self, sbom_id: Uuid) -> Result, Error> { + let advisory = self.advisory_state.load(); + let sbom = self.sbom_state.load(); + + let packages = sbom + .by_sbom + .get(&sbom_id) + .ok_or_else(|| Error::SbomNotFound(sbom_id.to_string()))?; + + let sbom_cpes = sbom.describing_cpes.get(&sbom_id); + let sbom_has_cpes = sbom_cpes.is_some_and(|c| !c.is_empty()); + let has_product_index = !advisory.product_by_name.is_empty(); + let mut matches = Vec::with_capacity(packages.len()); + + for pkg in packages.iter() { + let key = PurlKey { + ty: Arc::clone(&pkg.ty), + namespace: pkg.namespace.as_ref().map(Arc::clone), + name: Arc::clone(&pkg.name), + }; + + // Path 1: purl_status matching (version range based) + if let Some(statuses) = advisory.by_purl.get(&key) { + for entry in statuses { + if !check_cpe_context(entry.context_cpe_id, sbom_cpes, sbom_has_cpes) { + continue; + } + + if crate::model::version::version_matches(&pkg.version, &entry.version_range) { + matches.push(CorrelationMatch { + advisory_id: entry.advisory_id, + vulnerability_id: Arc::clone(&entry.vulnerability_id), + status_id: entry.status_id, + context_cpe_id: entry.context_cpe_id, + purl_key: key.clone(), + version: Arc::clone(&pkg.version), + }); + } + } + } + + // Path 2: product_status matching (name based) + if has_product_index { + Self::check_product_status( + &advisory, + &pkg.name, + &key, + &pkg.version, + sbom_cpes, + sbom_has_cpes, + &mut matches, + ); + if let Some(ns) = &pkg.namespace { + let full_name = format!("{}/{}", ns, pkg.name); + Self::check_product_status( + &advisory, + &full_name, + &key, + &pkg.version, + sbom_cpes, + sbom_has_cpes, + &mut matches, + ); + } + } + } + + Ok(matches) + } + + /// Correlates standalone PURLs against the advisory index without SBOM context. + /// + /// For each parsed PURL, looks up by_purl entries and applies version matching. + /// No CPE context filtering is applied (standalone PURLs have no SBOM context). + /// Returns matches grouped by the original PURL string. + #[instrument(skip_all, fields(purl_count = purls.len()), err(level = tracing::Level::INFO))] + pub fn correlate_purls( + &self, + purls: &[Purl], + ) -> Result>, Error> { + let advisory = self.advisory_state.load(); + let mut results: HashMap> = HashMap::new(); + + for purl in purls { + let key = PurlKey { + ty: Arc::from(purl.ty.as_str()), + namespace: purl.namespace.as_deref().map(Arc::from), + name: Arc::from(purl.name.as_str()), + }; + + let purl_str = purl.to_string(); + let version = match &purl.version { + Some(v) => v.as_str(), + None => { + results.entry(purl_str).or_default(); + continue; + } + }; + + let matches = results.entry(purl_str).or_default(); + + if let Some(statuses) = advisory.by_purl.get(&key) { + for entry in statuses { + if crate::model::version::version_matches(version, &entry.version_range) { + matches.push(PurlCorrelationMatch { + purl_status_id: Some(entry.purl_status_id), + product_status_id: None, + advisory_id: entry.advisory_id, + vulnerability_id: Arc::clone(&entry.vulnerability_id), + status_id: entry.status_id, + context_cpe_id: entry.context_cpe_id, + version_range: Some(entry.version_range.clone()), + }); + } + } + } + + // product_status matches (CSAF name-based, no version range) + if !advisory.product_by_name.is_empty() { + let mut seen: HashSet<(Uuid, Arc, Uuid, Option)> = HashSet::new(); + + for package_name in Self::product_lookup_names(purl) { + if let Some(entries) = advisory.product_by_name.get(package_name.as_str()) { + for entry in entries { + let is_new = seen.insert(( + entry.advisory_id, + Arc::clone(&entry.vulnerability_id), + entry.status_id, + entry.context_cpe_id, + )); + if is_new { + matches.push(PurlCorrelationMatch { + purl_status_id: None, + product_status_id: Some(entry.product_status_id), + advisory_id: entry.advisory_id, + vulnerability_id: Arc::clone(&entry.vulnerability_id), + status_id: entry.status_id, + context_cpe_id: entry.context_cpe_id, + version_range: None, + }); + } + } + } + } + } + } + + Ok(results) + } + + /// Computes advisory severity counts for a batch of SBOMs. + /// + /// For each SBOM, runs in-memory correlation to find affected vulnerabilities, + /// then looks up the pre-computed max severity from the severity index. + /// Returns per-SBOM counts grouped by severity level. + #[instrument(skip_all, fields(sbom_count = sbom_ids.len()))] + pub fn batch_severity_counts(&self, sbom_ids: &[Uuid]) -> HashMap { + let advisory = self.advisory_state.load(); + let sbom = self.sbom_state.load(); + let mut result = HashMap::with_capacity(sbom_ids.len()); + + for &sbom_id in sbom_ids { + let Some(packages) = sbom.by_sbom.get(&sbom_id) else { + continue; + }; + + let sbom_cpes = sbom.describing_cpes.get(&sbom_id); + let sbom_has_cpes = sbom_cpes.is_some_and(|c| !c.is_empty()); + + // Track unique (advisory_id, vulnerability_id) pairs for dedup + let mut seen: HashSet<(Uuid, Arc)> = HashSet::new(); + let mut severity_counts: SbomAdvisorySummary = HashMap::new(); + + for pkg in packages.iter() { + let key = PurlKey { + ty: Arc::clone(&pkg.ty), + namespace: pkg.namespace.as_ref().map(Arc::clone), + name: Arc::clone(&pkg.name), + }; + + // Check purl_status matches + if let Some(statuses) = advisory.by_purl.get(&key) { + for entry in statuses { + if !check_cpe_context(entry.context_cpe_id, sbom_cpes, sbom_has_cpes) { + continue; + } + let status_slug = advisory.statuses.get(&entry.status_id); + if status_slug.is_none_or(|s| s.as_ref() != "affected") { + continue; + } + if !crate::model::version::version_matches( + &pkg.version, + &entry.version_range, + ) { + continue; + } + + let pair = (entry.advisory_id, Arc::clone(&entry.vulnerability_id)); + if seen.insert(pair) { + let severity = advisory + .severity + .get(&(entry.advisory_id, Arc::clone(&entry.vulnerability_id))) + .copied() + .unwrap_or(AffectedSeverity::Unknown); + *severity_counts.entry(severity).or_default() += 1; + } + } + } + + // Check product_status matches + Self::count_product_severity( + &advisory, + &pkg.name, + sbom_cpes, + sbom_has_cpes, + &mut seen, + &mut severity_counts, + ); + if let Some(ns) = &pkg.namespace { + let full_name = format!("{}/{}", ns, pkg.name); + Self::count_product_severity( + &advisory, + &full_name, + sbom_cpes, + sbom_has_cpes, + &mut seen, + &mut severity_counts, + ); + } + } + + if !severity_counts.is_empty() { + result.insert(sbom_id, severity_counts); + } + } + + result + } + + /// Returns the raw vulnerability index entries for the purl fallback path. + /// + /// When `correlate_vulnerability()` finds no SBOM matches, these entries + /// are used to build the legacy `purls` field in VulnerabilityAdvisorySummary. + pub fn vulnerability_entries(&self, vulnerability_id: &str) -> Vec { + let advisory = self.advisory_state.load(); + advisory + .by_vulnerability + .get(vulnerability_id) + .cloned() + .unwrap_or_default() + } + + /// Correlates a vulnerability against the SBOM index. + /// + /// Looks up the vulnerability in the reverse index to find all advisory entries, + /// then for each purl-based entry, finds matching SBOMs via the purl key reverse + /// index and applies version matching. Filters out `not_affected` statuses. + #[instrument(skip_all, fields(vulnerability_id), err(level = tracing::Level::INFO))] + pub fn correlate_vulnerability( + &self, + vulnerability_id: &str, + ) -> Result, Error> { + let advisory = self.advisory_state.load(); + let sbom = self.sbom_state.load(); + + let entries = match advisory.by_vulnerability.get(vulnerability_id) { + Some(entries) => entries, + None => return Ok(Vec::new()), + }; + + let mut matches = Vec::new(); + + for entry in entries { + let status_slug = advisory + .statuses + .get(&entry.status_id) + .map(|s| s.as_ref()) + .unwrap_or("unknown"); + + if status_slug == "not_affected" { + continue; + } + + match &entry.source { + VulnEntrySource::Purl { + purl_key, + version_range, + } => { + let Some(sbom_ids) = sbom.by_purl_key.get(purl_key) else { + continue; + }; + for &sbom_id in sbom_ids { + let Some(packages) = sbom.by_sbom.get(&sbom_id) else { + continue; + }; + let sbom_cpes = sbom.describing_cpes.get(&sbom_id); + let sbom_has_cpes = sbom_cpes.is_some_and(|c| !c.is_empty()); + + if !check_cpe_context(entry.context_cpe_id, sbom_cpes, sbom_has_cpes) { + continue; + } + + for pkg in packages.iter() { + if pkg.ty != purl_key.ty + || pkg.namespace != purl_key.namespace + || pkg.name != purl_key.name + { + continue; + } + if crate::model::version::version_matches(&pkg.version, version_range) { + matches.push(VulnCorrelationMatch { + advisory_id: entry.advisory_id, + status_id: entry.status_id, + context_cpe_id: entry.context_cpe_id, + sbom_id, + purl_key: purl_key.clone(), + version: Arc::clone(&pkg.version), + }); + } + } + } + } + VulnEntrySource::Product { package_name: _ } => { + // Product-based matching is more complex and less common. + // Skip for now — the hydration fallback covers this path. + } + } + } + + Ok(matches) + } + + /// Counts product_status severity matches for a single package name. + fn count_product_severity( + advisory: &AdvisoryIndex, + package_name: &str, + sbom_cpes: Option<&HashSet>, + sbom_has_cpes: bool, + seen: &mut HashSet<(Uuid, Arc)>, + severity_counts: &mut SbomAdvisorySummary, + ) { + if let Some(entries) = advisory.product_by_name.get(package_name) { + for entry in entries { + if !check_cpe_context(entry.context_cpe_id, sbom_cpes, sbom_has_cpes) { + continue; + } + let status_slug = advisory.statuses.get(&entry.status_id); + if status_slug.is_none_or(|s| s.as_ref() != "affected") { + continue; + } + + let pair = (entry.advisory_id, Arc::clone(&entry.vulnerability_id)); + if seen.insert(pair) { + let severity = advisory + .severity + .get(&(entry.advisory_id, Arc::clone(&entry.vulnerability_id))) + .copied() + .unwrap_or(AffectedSeverity::Unknown); + *severity_counts.entry(severity).or_default() += 1; + } + } + } + } + + /// Returns the product_by_name lookup keys for a PURL: bare name and namespace/name. + fn product_lookup_names(purl: &Purl) -> Vec { + let mut names = vec![purl.name.clone()]; + if let Some(ns) = &purl.namespace { + names.push(format!("{}/{}", ns, purl.name)); + } + names + } + + /// Checks product_status entries for a package name match. + fn check_product_status( + advisory: &AdvisoryIndex, + package_name: &str, + purl_key: &PurlKey, + version: &Arc, + sbom_cpes: Option<&std::collections::HashSet>, + sbom_has_cpes: bool, + matches: &mut Vec, + ) { + if let Some(entries) = advisory.product_by_name.get(package_name) { + for entry in entries { + if !check_cpe_context(entry.context_cpe_id, sbom_cpes, sbom_has_cpes) { + continue; + } + + matches.push(CorrelationMatch { + advisory_id: entry.advisory_id, + vulnerability_id: Arc::clone(&entry.vulnerability_id), + status_id: entry.status_id, + context_cpe_id: entry.context_cpe_id, + purl_key: purl_key.clone(), + version: Arc::clone(version), + }); + } + } + } + + /// Background task that processes events with debouncing and applies incremental updates. + async fn background_loader( + advisory_state: Arc>, + sbom_state: Arc>, + db: ReadOnly, + mut rx: mpsc::UnboundedReceiver, + debounce: Duration, + ) { + while let Some(event) = rx.recv().await { + // Debounce: wait then drain accumulated events + let mut pending = PendingChanges::new(); + pending.add(event); + + tokio::time::sleep(debounce).await; + while let Ok(event) = rx.try_recv() { + pending.add(event); + } + + tracing::info!( + advisories = pending.advisory_ids.len(), + sboms = pending.sbom_ids.len(), + "applying incremental correlation updates" + ); + + if let Err(err) = Self::apply_changes(&advisory_state, &sbom_state, &db, &pending).await + { + tracing::error!(%err, "failed to apply incremental correlation updates"); + } + } + } + + /// Loads patches for the changed entities and applies them to the state. + /// + /// Only clones the index that actually has changes, avoiding unnecessary + /// deep-clones of the unaffected side. + async fn apply_changes( + advisory_state: &ArcSwap, + sbom_state: &ArcSwap, + db: &ReadOnly, + pending: &PendingChanges, + ) -> Result<(), anyhow::Error> { + let txn = db.begin().await?; + + // Apply advisory patches — only clone advisory index if needed + if !pending.advisory_ids.is_empty() { + let ids: Vec = pending.advisory_ids.iter().copied().collect(); + let mut patches = load::load_advisory_patches(&ids, &txn) + .instrument(info_span!("load advisory patches")) + .await?; + + let old = advisory_state.load(); + let mut new_advisory = (**old).clone(); + for &id in &ids { + let patch = patches.remove(&id).unwrap_or_default(); + new_advisory.apply_patch(id, patch); + } + advisory_state.store(Arc::new(new_advisory)); + } + + // Apply SBOM patches — only clone sbom index if needed + if !pending.sbom_ids.is_empty() { + let ids: Vec = pending.sbom_ids.iter().copied().collect(); + let mut patches = load::load_sbom_patches(&ids, &txn) + .instrument(info_span!("load sbom patches")) + .await?; + + let old = sbom_state.load(); + let mut new_sbom = (**old).clone(); + for &id in &ids { + let patch = patches.remove(&id).unwrap_or_default(); + new_sbom.apply_patch(id, patch); + } + sbom_state.store(Arc::new(new_sbom)); + } + + tracing::info!("correlation state updated incrementally"); + Ok(()) + } +} + +/// Accumulates changed entity IDs during the debounce window. +struct PendingChanges { + advisory_ids: HashSet, + sbom_ids: HashSet, +} + +impl PendingChanges { + fn new() -> Self { + Self { + advisory_ids: HashSet::new(), + sbom_ids: HashSet::new(), + } + } + + fn add(&mut self, event: CorrelationEvent) { + match event { + CorrelationEvent::AdvisoryChanged(id) => { + self.advisory_ids.insert(id); + } + CorrelationEvent::SbomChanged(id) => { + self.sbom_ids.insert(id); + } + } + } +} + +/// Checks the CPE context filter, matching the v3a SQL logic: +/// - NULL context_cpe_id always matches +/// - If the SBOM has no describing CPEs, everything matches +/// - Otherwise the context_cpe_id must be in the SBOM's CPE set +fn check_cpe_context( + context_cpe_id: Option, + sbom_cpes: Option<&std::collections::HashSet>, + sbom_has_cpes: bool, +) -> bool { + match context_cpe_id { + None => true, + Some(cpe_id) => { + if !sbom_has_cpes { + return true; + } + sbom_cpes.is_some_and(|cpes| cpes.contains(&cpe_id)) + } + } +} diff --git a/modules/correlation/src/service/test.rs b/modules/correlation/src/service/test.rs new file mode 100644 index 000000000..0035d7002 --- /dev/null +++ b/modules/correlation/src/service/test.rs @@ -0,0 +1,67 @@ +use crate::model::{ + CorrelationState, PurlKey, PurlStatusEntry, SbomPackageEntry, VersionRangeData, +}; +use std::collections::HashMap; +use std::sync::Arc; +use trustify_entity::version_scheme::VersionScheme; +use uuid::Uuid; + +#[test] +fn correlate_basic_match() { + let advisory_id = Uuid::new_v4(); + let status_id = Uuid::new_v4(); + let sbom_id = Uuid::new_v4(); + + let purl_key = PurlKey { + ty: Arc::from("maven"), + namespace: Some(Arc::from("org.example")), + name: Arc::from("test-pkg"), + }; + + let pkg = SbomPackageEntry { + ty: Arc::from("maven"), + version: Arc::from("1.5.0"), + name: Arc::from("test-pkg"), + namespace: Some(Arc::from("org.example")), + }; + + let state = CorrelationState { + advisory_index: crate::model::AdvisoryIndex { + by_purl: HashMap::from([( + purl_key.clone(), + vec![PurlStatusEntry { + purl_status_id: Uuid::new_v4(), + advisory_id, + vulnerability_id: Arc::from("CVE-2024-0001"), + status_id, + version_range: VersionRangeData { + version_scheme: VersionScheme::Semver, + low_parsed: lenient_semver::parse("1.0.0").ok(), + high_parsed: lenient_semver::parse("2.0.0").ok(), + low_version: Some(Arc::from("1.0.0")), + low_inclusive: true, + high_version: Some(Arc::from("2.0.0")), + high_inclusive: false, + }, + context_cpe_id: None, + }], + )]), + product_by_name: HashMap::new(), + statuses: HashMap::from([(status_id, Arc::from("affected"))]), + severity: HashMap::new(), + by_vulnerability: HashMap::new(), + }, + sbom_index: crate::model::SbomIndex { + by_sbom: HashMap::from([(sbom_id, Arc::from(vec![pkg].into_boxed_slice()))]), + describing_cpes: HashMap::new(), + by_purl_key: HashMap::new(), + }, + }; + + let pkg = &state.sbom_index.by_sbom[&sbom_id][0]; + let entry = &state.advisory_index.by_purl[&purl_key][0]; + assert!(crate::model::version::version_matches( + &pkg.version, + &entry.version_range + )); +} diff --git a/modules/correlation/tests/benchmark.rs b/modules/correlation/tests/benchmark.rs new file mode 100644 index 000000000..d4783dcbb --- /dev/null +++ b/modules/correlation/tests/benchmark.rs @@ -0,0 +1,493 @@ +#![recursion_limit = "512"] +#![allow(clippy::unwrap_used)] +#![allow(clippy::expect_used)] + +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; +use std::time::Instant; +use test_context::test_context; +use test_log::test; +use trustify_common::{db::pagination_cache::PaginationCache, id::Id, purl::Purl}; +use trustify_module_correlation::{config::CorrelationConfig, service::CorrelationService}; +use trustify_module_fundamental::{ + purl::service::PurlService, sbom::service::SbomService, + vulnerability::service::VulnerabilityService, +}; +use trustify_module_ingestor::common::Deprecation; +use trustify_test_context::{Dataset, TrustifyContext}; + +/// Benchmark: compare v3a (SQL) vs v3 (in-memory) correlation for quarkus-bom. +/// +/// Ingests the DS3 dataset, then runs both the v3a SQL-based correlation +/// and the v3 in-memory correlation on the quarkus-bom SBOM, timing each. +/// Also verifies that both produce the same advisory count. +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn benchmark_quarkus_bom(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + assert!(result.warnings.is_empty()); + + let sbom = &result.files["spdx/quarkus-bom-2.13.8.Final-redhat-00004.json.bz2"]; + let sbom_id = Id::parse_uuid(&sbom.id)?; + + // --- v3a baseline (SQL) --- + let sbom_service = SbomService::new(PaginationCache::for_test()); + + let start_v3a = Instant::now(); + let v3a_details = sbom_service + .fetch_sbom_details(sbom_id.clone(), vec![], &ctx.db) + .await? + .expect("SBOM should exist"); + let v3a_time = start_v3a.elapsed(); + let v3a_count = v3a_details.advisories.len(); + + log::info!( + "v3a quarkus-bom: {} advisories in {}", + v3a_count, + humantime::Duration::from(v3a_time), + ); + + // --- v3 correlation (in-memory) --- + let db_ro = trustify_common::db::ReadOnly::new(ctx.db.clone()); + let db_rw = trustify_common::db::ReadWrite::new(ctx.db.clone()); + let config = CorrelationConfig { + correlation_poll_interval_secs: 30, + correlation_debounce_secs: 2, + }; + let correlation = CorrelationService::new(&config, db_ro, &db_rw).await?; + + let sbom_uuid = match sbom_id { + Id::Uuid(u) => u, + _ => panic!("expected UUID"), + }; + + let start_v3_correlate = Instant::now(); + let v3_matches = correlation.correlate_sbom(sbom_uuid)?; + let v3_correlate_time = start_v3_correlate.elapsed(); + let match_count = v3_matches.len(); + + // Hydrate: convert matches to Vec (includes DB lookups) + let statuses = correlation.status_slugs(); + let txn = ctx.db.begin().await?; + + let start_v3_hydrate = Instant::now(); + let v3_advisories = + trustify_module_correlation::service::hydrate::hydrate_matches(v3_matches, &statuses, &txn) + .await?; + let v3_hydrate_time = start_v3_hydrate.elapsed(); + + let v3_total_time = v3_correlate_time + v3_hydrate_time; + let v3_count = v3_advisories.len(); + + log::info!( + "v3 quarkus-bom: {} advisories ({} matches) — correlate={}, hydrate={}, total={}", + v3_count, + match_count, + humantime::Duration::from(v3_correlate_time), + humantime::Duration::from(v3_hydrate_time), + humantime::Duration::from(v3_total_time), + ); + + log::info!( + "speedup: {:.1}x (v3a={}, v3={})", + v3a_time.as_secs_f64() / v3_total_time.as_secs_f64(), + humantime::Duration::from(v3a_time), + humantime::Duration::from(v3_total_time), + ); + + // Verify both find the same advisory count + assert_eq!( + v3a_count, v3_count, + "v3a found {} advisories but v3 found {} — mismatch!", + v3a_count, v3_count, + ); + + // Known DS3 ground truth: quarkus-bom should have 22 advisories + assert_eq!(v3a_count, 22, "expected 22 advisories for quarkus-bom"); + + Ok(()) +} + +/// Benchmark: ubi8 SBOM correlation (fewer matches). +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn benchmark_ubi8(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + assert!(result.warnings.is_empty()); + + let sbom = &result.files["spdx/ubi8-8.8-1067.json.bz2"]; + let sbom_id = Id::parse_uuid(&sbom.id)?; + + // --- v3a baseline --- + let sbom_service = SbomService::new(PaginationCache::for_test()); + + let start_v3a = Instant::now(); + let v3a_details = sbom_service + .fetch_sbom_details(sbom_id.clone(), vec![], &ctx.db) + .await? + .expect("SBOM should exist"); + let v3a_time = start_v3a.elapsed(); + let v3a_count = v3a_details.advisories.len(); + + log::info!( + "v3a ubi8: {} advisories in {}", + v3a_count, + humantime::Duration::from(v3a_time), + ); + + // --- v3 correlation --- + let db_ro = trustify_common::db::ReadOnly::new(ctx.db.clone()); + let db_rw = trustify_common::db::ReadWrite::new(ctx.db.clone()); + let config = CorrelationConfig { + correlation_poll_interval_secs: 30, + correlation_debounce_secs: 2, + }; + let correlation = CorrelationService::new(&config, db_ro, &db_rw).await?; + + let sbom_uuid = match sbom_id { + Id::Uuid(u) => u, + _ => panic!("expected UUID"), + }; + + let start_v3_correlate = Instant::now(); + let v3_matches = correlation.correlate_sbom(sbom_uuid)?; + let v3_correlate_time = start_v3_correlate.elapsed(); + let match_count = v3_matches.len(); + + let statuses = correlation.status_slugs(); + let txn = ctx.db.begin().await?; + + let start_v3_hydrate = Instant::now(); + let v3_advisories = + trustify_module_correlation::service::hydrate::hydrate_matches(v3_matches, &statuses, &txn) + .await?; + let v3_hydrate_time = start_v3_hydrate.elapsed(); + + let v3_total_time = v3_correlate_time + v3_hydrate_time; + let v3_count = v3_advisories.len(); + + log::info!( + "v3 ubi8: {} advisories ({} matches) — correlate={}, hydrate={}, total={}", + v3_count, + match_count, + humantime::Duration::from(v3_correlate_time), + humantime::Duration::from(v3_hydrate_time), + humantime::Duration::from(v3_total_time), + ); + + log::info!( + "speedup: {:.1}x (v3a={}, v3={})", + v3a_time.as_secs_f64() / v3_total_time.as_secs_f64(), + humantime::Duration::from(v3a_time), + humantime::Duration::from(v3_total_time), + ); + + // Verify counts match + assert_eq!( + v3a_count, v3_count, + "v3a found {} advisories but v3 found {} — mismatch!", + v3a_count, v3_count, + ); + + // Known DS3 ground truth: ubi8 should have 1 advisory (CVE-2024-28834) + assert_eq!(v3a_count, 1, "expected 1 advisory for ubi8"); + + Ok(()) +} + +/// Helper: creates a CorrelationService from a TrustifyContext. +async fn create_correlation(ctx: &TrustifyContext) -> anyhow::Result { + let db_ro = trustify_common::db::ReadOnly::new(ctx.db.clone()); + let db_rw = trustify_common::db::ReadWrite::new(ctx.db.clone()); + let config = CorrelationConfig { + correlation_poll_interval_secs: 30, + correlation_debounce_secs: 2, + }; + CorrelationService::new(&config, db_ro, &db_rw).await +} + +/// Benchmark: compare v3a (SQL) vs v3 (in-memory) vulnerability details. +/// +/// Uses CVE-2023-4853 from DS3, which has CSAF advisory data and affects +/// multiple packages in the quarkus-bom SBOM. +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn benchmark_vulnerability(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + assert!(result.warnings.is_empty()); + + let vuln_id = "CVE-2023-4853"; + + // --- v3a baseline (SQL) --- + let vuln_service = VulnerabilityService::new(PaginationCache::for_test()); + + let start_v3a = Instant::now(); + let v3a_details = vuln_service + .fetch_vulnerability(vuln_id, Deprecation::Ignore, true, &ctx.db) + .await? + .expect("vulnerability should exist"); + let v3a_time = start_v3a.elapsed(); + let v3a_count = v3a_details.advisories.len(); + + log::info!( + "v3a vulnerability {}: {} advisories in {}", + vuln_id, + v3a_count, + humantime::Duration::from(v3a_time), + ); + + // --- v3 correlation (in-memory) --- + let correlation = create_correlation(&ctx).await?; + let txn = ctx.db.begin().await?; + + let start_v3 = Instant::now(); + let matches = correlation.correlate_vulnerability(vuln_id)?; + let vuln_entries = correlation.vulnerability_entries(vuln_id); + let statuses = correlation.status_slugs(); + + let vuln = trustify_entity::vulnerability::Entity::find_by_id(vuln_id) + .one(&txn) + .await? + .expect("vulnerability should exist"); + + let (advisory_vulns, vuln_scores) = tokio::try_join!( + trustify_entity::advisory_vulnerability::Entity::find() + .filter(trustify_entity::advisory_vulnerability::Column::VulnerabilityId.eq(vuln_id),) + .all(&txn), + trustify_entity::advisory_vulnerability_score::Entity::find() + .filter( + trustify_entity::advisory_vulnerability_score::Column::VulnerabilityId.eq(vuln_id), + ) + .all(&txn), + )?; + + let advisories = + trustify_module_correlation::service::hydrate::hydrate_vulnerability_advisories( + &vuln, + &advisory_vulns, + &vuln_scores, + matches, + &vuln_entries, + &statuses, + &txn, + ) + .await?; + let v3_time = start_v3.elapsed(); + let v3_count = advisories.len(); + + log::info!( + "v3 vulnerability {}: {} advisories in {}", + vuln_id, + v3_count, + humantime::Duration::from(v3_time), + ); + + log::info!( + "speedup: {:.1}x (v3a={}, v3={})", + v3a_time.as_secs_f64() / v3_time.as_secs_f64(), + humantime::Duration::from(v3a_time), + humantime::Duration::from(v3_time), + ); + + assert_eq!( + v3a_count, v3_count, + "v3a found {} advisories but v3 found {} — mismatch!", + v3a_count, v3_count, + ); + + Ok(()) +} + +/// Benchmark: compare v3a (SQL) vs v3 (in-memory) PURL detail lookup. +/// +/// Uses a quarkus PURL from DS3 that has known advisory matches. +/// Correctness is verified in `correctness::purl_advisories_subset_of_sql`. +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn benchmark_purl(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + assert!(result.warnings.is_empty()); + + let purl_str = "pkg:maven/io.quarkus/quarkus-vertx-http@2.13.8.Final-redhat-00004?repository_url=https://maven.repository.redhat.com/ga/&type=jar"; + let purl = Purl::try_from(purl_str)?; + + // --- v3a baseline (SQL) --- + let purl_service = PurlService::new(PaginationCache::for_test()); + + let start_v3a = Instant::now(); + let v3a_details = purl_service + .purl_by_purl(&purl, Deprecation::Ignore, &ctx.db) + .await? + .expect("PURL should exist"); + let v3a_time = start_v3a.elapsed(); + + log::info!( + "v3a purl: {} advisories in {}", + v3a_details.advisories.len(), + humantime::Duration::from(v3a_time), + ); + + // --- v3 correlation (in-memory) --- + let correlation = create_correlation(&ctx).await?; + let txn = ctx.db.begin().await?; + + let start_v3 = Instant::now(); + let matches = correlation.correlate_purls(&[purl])?; + let statuses = correlation.status_slugs(); + let purl_matches = matches.into_values().next().unwrap_or_default(); + let advisories = trustify_module_correlation::service::hydrate::hydrate_purl_advisories( + purl_matches, + &statuses, + &txn, + ) + .await?; + let v3_time = start_v3.elapsed(); + + log::info!( + "v3 purl: {} advisories in {}", + advisories.len(), + humantime::Duration::from(v3_time), + ); + + log::info!( + "speedup: {:.1}x (v3a={}, v3={})", + v3a_time.as_secs_f64() / v3_time.as_secs_f64(), + humantime::Duration::from(v3a_time), + humantime::Duration::from(v3_time), + ); + + Ok(()) +} + +/// Benchmark: compare v3a (SQL) vs v3 (in-memory) vulnerability analysis. +/// +/// Sends a batch of PURLs from the quarkus-bom through the analyze endpoint +/// and compares SQL-based analysis with in-memory correlation. +/// Correctness is verified in `correctness::analyze_vulnerability_ids_match_sql`. +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn benchmark_analyze(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + assert!(result.warnings.is_empty()); + + let purls = [ + "pkg:maven/io.quarkus/quarkus-vertx-http@2.13.8.Final-redhat-00004?type=jar", + "pkg:maven/io.netty/netty-handler@4.1.86.Final-redhat-00001?type=jar", + "pkg:maven/org.apache.james/apache-mime4j-core@0.8.9-redhat-00001?type=jar", + ]; + + // --- v3a baseline (SQL) --- + let vuln_service = VulnerabilityService::new(PaginationCache::for_test()); + + let start_v3a = Instant::now(); + let v3a_response = vuln_service + .analyze_purls_v3(purls.iter().copied(), &ctx.db) + .await?; + let v3a_time = start_v3a.elapsed(); + let v3a_detail_count: usize = v3a_response.0.values().map(|r| r.details.len()).sum(); + + log::info!( + "v3a analyze: {} purls, {} vuln details in {}", + v3a_response.0.len(), + v3a_detail_count, + humantime::Duration::from(v3a_time), + ); + + // --- v3 correlation (in-memory) --- + let correlation = create_correlation(&ctx).await?; + let txn = ctx.db.begin().await?; + + let parsed: Vec<_> = purls + .iter() + .filter_map(|p| Purl::try_from(*p).ok()) + .collect(); + + let start_v3 = Instant::now(); + let matches = correlation.correlate_purls(&parsed)?; + let statuses = correlation.status_slugs(); + let v3_response = + trustify_module_correlation::service::hydrate::hydrate_analysis(matches, &statuses, &txn) + .await?; + let v3_time = start_v3.elapsed(); + let v3_detail_count: usize = v3_response.0.values().map(|r| r.details.len()).sum(); + + log::info!( + "v3 analyze: {} purls, {} vuln details in {}", + v3_response.0.len(), + v3_detail_count, + humantime::Duration::from(v3_time), + ); + + log::info!( + "speedup: {:.1}x (v3a={}, v3={})", + v3a_time.as_secs_f64() / v3_time.as_secs_f64(), + humantime::Duration::from(v3a_time), + humantime::Duration::from(v3_time), + ); + + Ok(()) +} + +/// Benchmark: compare v3a (SQL) vs v3 (in-memory) PURL recommend. +/// +/// Sends PURLs with known Red Hat patch versions through the recommend +/// path and compares the SQL-based and in-memory correlation results. +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn benchmark_recommend(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + assert!(result.warnings.is_empty()); + + let purl_strs = ["pkg:maven/io.quarkus/quarkus-vertx-http@2.13.8.Final-redhat-00004?type=jar"]; + let purls: Vec<_> = purl_strs + .iter() + .filter_map(|p| Purl::try_from(*p).ok()) + .collect(); + + // --- v3a baseline (SQL) --- + let purl_service = PurlService::new(PaginationCache::for_test()); + + let start_v3a = Instant::now(); + let v3a_result = purl_service.recommend_purls(&purls, &ctx.db).await?; + let v3a_time = start_v3a.elapsed(); + let v3a_count: usize = v3a_result.values().map(|v| v.len()).sum(); + + log::info!( + "v3a recommend: {} entries across {} purls in {}", + v3a_count, + v3a_result.len(), + humantime::Duration::from(v3a_time), + ); + + // --- v3 correlation (in-memory) --- + let correlation = create_correlation(&ctx).await?; + let txn = ctx.db.begin().await?; + + let start_v3 = Instant::now(); + let matches = correlation.correlate_purls(&purls)?; + let statuses = correlation.status_slugs(); + + // The recommend endpoint remaps winner PURLs; here we just benchmark + // the hydrate_recommend_matches path directly. + let v3_result = trustify_module_correlation::service::hydrate::hydrate_recommend_matches( + matches, &statuses, &txn, + ) + .await?; + let v3_time = start_v3.elapsed(); + let v3_count: usize = v3_result.values().map(|v| v.len()).sum(); + + log::info!( + "v3 recommend: {} entries across {} purls in {}", + v3_count, + v3_result.len(), + humantime::Duration::from(v3_time), + ); + + log::info!( + "speedup: {:.1}x (v3a={}, v3={})", + v3a_time.as_secs_f64() / v3_time.as_secs_f64(), + humantime::Duration::from(v3a_time), + humantime::Duration::from(v3_time), + ); + + Ok(()) +} diff --git a/modules/correlation/tests/correctness.rs b/modules/correlation/tests/correctness.rs new file mode 100644 index 000000000..f69312aa0 --- /dev/null +++ b/modules/correlation/tests/correctness.rs @@ -0,0 +1,352 @@ +#![recursion_limit = "512"] +#![allow(clippy::unwrap_used)] +#![allow(clippy::expect_used)] + +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; +use std::collections::HashSet; +use test_context::test_context; +use test_log::test; +use trustify_common::{db::pagination_cache::PaginationCache, id::Id, purl::Purl}; +use trustify_module_correlation::{config::CorrelationConfig, service::CorrelationService}; +use trustify_module_fundamental::{ + purl::service::PurlService, sbom::service::SbomService, + vulnerability::service::VulnerabilityService, +}; +use trustify_module_ingestor::common::Deprecation; +use trustify_test_context::{Dataset, TrustifyContext}; + +/// Helper: creates a CorrelationService from a TrustifyContext. +async fn create_correlation(ctx: &TrustifyContext) -> anyhow::Result { + let db_ro = trustify_common::db::ReadOnly::new(ctx.db.clone()); + let db_rw = trustify_common::db::ReadWrite::new(ctx.db.clone()); + let config = CorrelationConfig { + correlation_poll_interval_secs: 30, + correlation_debounce_secs: 2, + }; + CorrelationService::new(&config, db_ro, &db_rw).await +} + +/// Verify SBOM advisory correlation matches the SQL-based result exactly. +/// +/// The quarkus-bom SBOM should produce 22 advisories from both the SQL and +/// in-memory paths, because SBOM correlation uses both purl_status and +/// product_status matching. +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn sbom_advisory_count_matches_sql(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + assert!(result.warnings.is_empty()); + + let sbom = &result.files["spdx/quarkus-bom-2.13.8.Final-redhat-00004.json.bz2"]; + let sbom_id = Id::parse_uuid(&sbom.id)?; + + // SQL baseline + let sbom_service = SbomService::new(PaginationCache::for_test()); + let v3a = sbom_service + .fetch_sbom_details(sbom_id.clone(), vec![], &ctx.db) + .await? + .expect("SBOM should exist"); + + // In-memory correlation + let correlation = create_correlation(&ctx).await?; + let sbom_uuid = match sbom_id { + Id::Uuid(u) => u, + _ => panic!("expected UUID"), + }; + let matches = correlation.correlate_sbom(sbom_uuid)?; + let statuses = correlation.status_slugs(); + let txn = ctx.db.begin().await?; + let v3 = + trustify_module_correlation::service::hydrate::hydrate_matches(matches, &statuses, &txn) + .await?; + + assert_eq!( + v3a.advisories.len(), + v3.len(), + "SBOM advisory count must match: SQL={}, correlation={}", + v3a.advisories.len(), + v3.len(), + ); + assert_eq!(v3.len(), 22, "quarkus-bom should have 22 advisories"); + + Ok(()) +} + +/// Verify vulnerability details match between SQL and in-memory paths. +/// +/// CVE-2023-4853 should produce the same advisory count from both paths. +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn vulnerability_advisory_count_matches_sql(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + assert!(result.warnings.is_empty()); + + let vuln_id = "CVE-2023-4853"; + + // SQL baseline + let vuln_service = VulnerabilityService::new(PaginationCache::for_test()); + let v3a = vuln_service + .fetch_vulnerability(vuln_id, Deprecation::Ignore, true, &ctx.db) + .await? + .expect("vulnerability should exist"); + + // In-memory correlation + let correlation = create_correlation(&ctx).await?; + let txn = ctx.db.begin().await?; + + let matches = correlation.correlate_vulnerability(vuln_id)?; + let vuln_entries = correlation.vulnerability_entries(vuln_id); + let statuses = correlation.status_slugs(); + + let vuln = trustify_entity::vulnerability::Entity::find_by_id(vuln_id) + .one(&txn) + .await? + .expect("vulnerability should exist"); + + let (advisory_vulns, vuln_scores) = tokio::try_join!( + trustify_entity::advisory_vulnerability::Entity::find() + .filter(trustify_entity::advisory_vulnerability::Column::VulnerabilityId.eq(vuln_id),) + .all(&txn), + trustify_entity::advisory_vulnerability_score::Entity::find() + .filter( + trustify_entity::advisory_vulnerability_score::Column::VulnerabilityId.eq(vuln_id), + ) + .all(&txn), + )?; + + let v3 = trustify_module_correlation::service::hydrate::hydrate_vulnerability_advisories( + &vuln, + &advisory_vulns, + &vuln_scores, + matches, + &vuln_entries, + &statuses, + &txn, + ) + .await?; + + assert_eq!( + v3a.advisories.len(), + v3.len(), + "vulnerability advisory count must match: SQL={}, correlation={}", + v3a.advisories.len(), + v3.len(), + ); + + Ok(()) +} + +/// Verify PURL advisory results match between SQL and in-memory paths. +/// +/// The SQL path (`PurlService::purl_by_purl`) queries both `purl_status` and +/// `product_status` tables. The in-memory correlation must produce the same +/// vulnerability set. +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn purl_advisory_count_matches_sql(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + assert!(result.warnings.is_empty()); + + let purl_str = "pkg:maven/io.quarkus/quarkus-vertx-http@2.13.8.Final-redhat-00004?repository_url=https://maven.repository.redhat.com/ga/&type=jar"; + let purl = Purl::try_from(purl_str)?; + + // SQL baseline + let purl_service = PurlService::new(PaginationCache::for_test()); + let v3a = purl_service + .purl_by_purl(&purl, Deprecation::Ignore, &ctx.db) + .await? + .expect("PURL should exist"); + + // In-memory correlation + let correlation = create_correlation(&ctx).await?; + let txn = ctx.db.begin().await?; + let matches = correlation.correlate_purls(&[purl])?; + let statuses = correlation.status_slugs(); + let purl_matches = matches.into_values().next().unwrap_or_default(); + let v3 = trustify_module_correlation::service::hydrate::hydrate_purl_advisories( + purl_matches, + &statuses, + &txn, + ) + .await?; + + let v3a_vuln_ids: HashSet<_> = v3a + .advisories + .iter() + .flat_map(|a| a.status.iter().map(|s| s.vulnerability.identifier.clone())) + .collect(); + let v3_vuln_ids: HashSet<_> = v3 + .iter() + .flat_map(|a| a.status.iter().map(|s| s.vulnerability.identifier.clone())) + .collect(); + + assert_eq!( + v3a_vuln_ids, v3_vuln_ids, + "vulnerability IDs must match: SQL={:?}, correlation={:?}", + v3a_vuln_ids, v3_vuln_ids, + ); + + assert_eq!( + v3a.advisories.len(), + v3.len(), + "advisory count must match: SQL={}, correlation={}", + v3a.advisories.len(), + v3.len(), + ); + + Ok(()) +} + +/// Verify analyze results match between SQL and in-memory paths. +/// +/// Both paths must find the same set of vulnerabilities for the given PURLs. +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn analyze_vulnerability_ids_match_sql(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + assert!(result.warnings.is_empty()); + + let purls = [ + "pkg:maven/io.quarkus/quarkus-vertx-http@2.13.8.Final-redhat-00004?type=jar", + "pkg:maven/io.netty/netty-handler@4.1.86.Final-redhat-00001?type=jar", + "pkg:maven/org.apache.james/apache-mime4j-core@0.8.9-redhat-00001?type=jar", + ]; + + // SQL baseline + let vuln_service = VulnerabilityService::new(PaginationCache::for_test()); + let v3a = vuln_service + .analyze_purls_v3(purls.iter().copied(), &ctx.db) + .await?; + + // In-memory correlation + let correlation = create_correlation(&ctx).await?; + let txn = ctx.db.begin().await?; + let parsed: Vec<_> = purls + .iter() + .filter_map(|p| Purl::try_from(*p).ok()) + .collect(); + let matches = correlation.correlate_purls(&parsed)?; + let statuses = correlation.status_slugs(); + let v3 = + trustify_module_correlation::service::hydrate::hydrate_analysis(matches, &statuses, &txn) + .await?; + + let v3a_vuln_ids: HashSet<_> = v3a + .0 + .values() + .flat_map(|r| r.details.iter().map(|d| d.head.identifier.clone())) + .collect(); + let v3_vuln_ids: HashSet<_> = + v3.0.values() + .flat_map(|r| r.details.iter().map(|d| d.head.identifier.clone())) + .collect(); + + assert_eq!( + v3a_vuln_ids, v3_vuln_ids, + "vulnerability IDs must match: SQL={:?}, correlation={:?}", + v3a_vuln_ids, v3_vuln_ids, + ); + + let v3a_ps_count: usize = v3a + .0 + .values() + .flat_map(|r| r.details.iter()) + .map(|d| d.purl_statuses.len()) + .sum(); + let v3_ps_count: usize = + v3.0.values() + .flat_map(|r| r.details.iter()) + .map(|d| d.purl_statuses.len()) + .sum(); + assert_eq!( + v3a_ps_count, v3_ps_count, + "purl_status count must match: SQL={}, correlation={}", + v3a_ps_count, v3_ps_count, + ); + + Ok(()) +} + +/// Verify recommend results are non-empty for a known Red Hat patched PURL. +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn recommend_returns_results(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + assert!(result.warnings.is_empty()); + + let purl_strs = ["pkg:maven/io.quarkus/quarkus-vertx-http@2.13.8.Final-redhat-00004?type=jar"]; + let purls: Vec<_> = purl_strs + .iter() + .filter_map(|p| Purl::try_from(*p).ok()) + .collect(); + + // SQL baseline + let purl_service = PurlService::new(PaginationCache::for_test()); + let v3a = purl_service.recommend_purls(&purls, &ctx.db).await?; + let v3a_count: usize = v3a.values().map(|v| v.len()).sum(); + + // In-memory correlation + let correlation = create_correlation(&ctx).await?; + let txn = ctx.db.begin().await?; + let matches = correlation.correlate_purls(&purls)?; + let statuses = correlation.status_slugs(); + let v3 = trustify_module_correlation::service::hydrate::hydrate_recommend_matches( + matches, &statuses, &txn, + ) + .await?; + let v3_count: usize = v3.values().map(|v| v.len()).sum(); + + // Both should return at least one recommendation + assert!(v3a_count > 0, "SQL recommend should return entries"); + assert!(v3_count > 0, "correlation recommend should return entries"); + + log::info!( + "recommend: SQL={} entries, correlation={} entries", + v3a_count, + v3_count, + ); + + Ok(()) +} + +/// Verify SBOM ubi8 correlation finds 1 advisory matching the SQL result. +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn sbom_ubi8_advisory_count_matches_sql(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + assert!(result.warnings.is_empty()); + + let sbom = &result.files["spdx/ubi8-8.8-1067.json.bz2"]; + let sbom_id = Id::parse_uuid(&sbom.id)?; + + // SQL baseline + let sbom_service = SbomService::new(PaginationCache::for_test()); + let v3a = sbom_service + .fetch_sbom_details(sbom_id.clone(), vec![], &ctx.db) + .await? + .expect("SBOM should exist"); + + // In-memory correlation + let correlation = create_correlation(&ctx).await?; + let sbom_uuid = match sbom_id { + Id::Uuid(u) => u, + _ => panic!("expected UUID"), + }; + let matches = correlation.correlate_sbom(sbom_uuid)?; + let statuses = correlation.status_slugs(); + let txn = ctx.db.begin().await?; + let v3 = + trustify_module_correlation::service::hydrate::hydrate_matches(matches, &statuses, &txn) + .await?; + + assert_eq!( + v3a.advisories.len(), + v3.len(), + "ubi8 advisory count must match: SQL={}, correlation={}", + v3a.advisories.len(), + v3.len(), + ); + assert_eq!(v3.len(), 1, "ubi8 should have 1 advisory"); + + Ok(()) +} diff --git a/modules/correlation/tests/diagnostic.rs b/modules/correlation/tests/diagnostic.rs new file mode 100644 index 000000000..de45cb034 --- /dev/null +++ b/modules/correlation/tests/diagnostic.rs @@ -0,0 +1,166 @@ +#![recursion_limit = "512"] +#![allow(clippy::unwrap_used)] +#![allow(clippy::expect_used)] + +use sea_orm::FromQueryResult; +use std::collections::HashSet; +use test_context::test_context; +use test_log::test; +use trustify_common::{db::pagination_cache::PaginationCache, id::Id}; +use trustify_module_correlation::{config::CorrelationConfig, service::CorrelationService}; +use trustify_module_fundamental::sbom::service::SbomService; +use trustify_test_context::{Dataset, TrustifyContext}; + +#[derive(Debug, FromQueryResult)] +struct MatchCheck { + package: String, + has_cpe_context: bool, + name_match: Option, + ns_name_match: Option, +} + +/// Diagnostic: show which advisories v3a finds vs v3 for quarkus-bom. +#[test_context(TrustifyContext, skip_teardown)] +#[test(tokio::test)] +async fn diagnostic_advisory_diff(ctx: TrustifyContext) -> anyhow::Result<()> { + let result = ctx.ingest_dataset(Dataset::DS3).await?; + + let sbom = &result.files["spdx/quarkus-bom-2.13.8.Final-redhat-00004.json.bz2"]; + let sbom_id = Id::parse_uuid(&sbom.id)?; + + // v3a + let sbom_service = SbomService::new(PaginationCache::for_test()); + let v3a_details = sbom_service + .fetch_sbom_details(sbom_id.clone(), vec![], &ctx.db) + .await? + .expect("SBOM should exist"); + + let v3a_advisory_ids: HashSet<_> = v3a_details + .advisories + .iter() + .map(|a| a.head.uuid.to_string()) + .collect(); + + // v3 + let db_ro = trustify_common::db::ReadOnly::new(ctx.db.clone()); + let db_rw = trustify_common::db::ReadWrite::new(ctx.db.clone()); + let config = CorrelationConfig { + correlation_poll_interval_secs: 30, + correlation_debounce_secs: 2, + }; + let correlation = CorrelationService::new(&config, db_ro, &db_rw).await?; + + let sbom_uuid = match sbom_id { + Id::Uuid(u) => u, + _ => panic!("expected UUID"), + }; + + let v3_matches = correlation.correlate_sbom(sbom_uuid)?; + let v3_advisory_ids: HashSet<_> = v3_matches + .iter() + .map(|m| m.advisory_id.to_string()) + .collect(); + + let only_v3a: Vec<_> = v3a_advisory_ids.difference(&v3_advisory_ids).collect(); + log::info!( + "v3a={}, v3={}, only_v3a={}", + v3a_advisory_ids.len(), + v3_advisory_ids.len(), + only_v3a.len() + ); + + // State summary + let state = correlation.state(); + log::info!( + "v3 state: {} purl_keys, {} product_by_name, {} sbom packages for this SBOM", + state.advisory_index.by_purl.len(), + state.advisory_index.product_by_name.len(), + state + .sbom_index + .by_sbom + .get(&sbom_uuid) + .map(|p| p.len()) + .unwrap_or(0), + ); + + // For one v3a-only advisory (CVE-2023-33201), check if product_status.package + // values match any SBOM base_purl name or namespace/name + let checks: Vec = MatchCheck::find_by_statement(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + format!( + r#" + WITH sbom_pkgs AS ( + SELECT DISTINCT bp.name, bp.namespace + FROM sbom_node_purl_ref snpr + JOIN qualified_purl qp ON qp.id = snpr.qualified_purl_id + JOIN versioned_purl vp ON vp.id = qp.versioned_purl_id + JOIN base_purl bp ON bp.id = vp.base_purl_id + WHERE snpr.sbom_id = '{}' + ) + SELECT DISTINCT + ps.package, + (ps.context_cpe_id IS NOT NULL) as has_cpe_context, + sp_name.name as name_match, + CONCAT(sp_ns.namespace, '/', sp_ns.name) as ns_name_match + FROM product_status ps + JOIN advisory_vulnerability av ON av.advisory_id = ps.advisory_id + AND av.vulnerability_id = ps.vulnerability_id + LEFT JOIN sbom_pkgs sp_name ON ps.package = sp_name.name + LEFT JOIN sbom_pkgs sp_ns ON sp_ns.namespace IS NOT NULL + AND ps.package = CONCAT(sp_ns.namespace, '/', sp_ns.name) + WHERE av.vulnerability_id = 'CVE-2023-33201' + AND ps.package IS NOT NULL + ORDER BY ps.package + "#, + sbom_uuid + ), + )) + .all(&ctx.db) + .await?; + + for c in &checks { + log::info!( + "CVE-2023-33201 product_status: package={:?} has_cpe={} name_match={:?} ns_match={:?}", + c.package, + c.has_cpe_context, + c.name_match, + c.ns_name_match + ); + } + + // Check: does v3 product_by_name have these package names? + for c in &checks { + let in_index = state + .advisory_index + .product_by_name + .contains_key(c.package.as_str()); + log::info!(" product_by_name[{:?}] exists: {}", c.package, in_index); + } + + // What SBOM packages would match these product_status entries? + let packages = state.sbom_index.by_sbom.get(&sbom_uuid).unwrap(); + let matching_pkgs: Vec<_> = packages + .iter() + .filter(|p| { + checks.iter().any(|c| { + c.package.as_str() == &*p.name + || p.namespace + .as_ref() + .is_some_and(|ns| c.package == format!("{}/{}", ns, p.name)) + }) + }) + .collect(); + log::info!( + "SBOM packages matching CVE-2023-33201 product_status names: {}", + matching_pkgs.len() + ); + for p in &matching_pkgs { + log::info!(" matched: name={:?} ns={:?}", p.name, p.namespace); + } + + // Check the CPE context for the SBOM + let sbom_cpes = state.sbom_index.describing_cpes.get(&sbom_uuid); + log::info!("SBOM describing CPEs: {:?}", sbom_cpes.map(|c| c.len())); + + Ok(()) +} diff --git a/modules/fundamental/src/advisory/endpoints/mod.rs b/modules/fundamental/src/advisory/endpoints/mod.rs index e72013e69..d21643518 100644 --- a/modules/fundamental/src/advisory/endpoints/mod.rs +++ b/modules/fundamental/src/advisory/endpoints/mod.rs @@ -19,6 +19,7 @@ use sea_orm::TransactionTrait; use std::str::FromStr; use time::OffsetDateTime; use trustify_auth::{CreateAdvisory, DeleteAdvisory, ReadAdvisory, authorizer::Require}; +use trustify_common::db::change::{ChangeEntity, ChangeOperation, record_change}; use trustify_common::{ db::{self, pagination_cache::PaginationCache, query::Query}, decompress::decompress_async, @@ -159,6 +160,13 @@ pub async fn delete( if let Some(v) = service.fetch_advisory(id, &tx).await? && service.delete_advisory(v.head.uuid, &tx).await? { + record_change( + &tx, + ChangeEntity::Advisory, + Some(v.head.uuid), + ChangeOperation::Deleted, + ) + .await?; tx.commit().await?; if let Err(e) = delete_doc(&v.source_document, i.storage()).await { log::error!("Ignoring {e}"); diff --git a/modules/fundamental/src/common/test.rs b/modules/fundamental/src/common/test.rs index 650fe890a..9617cb527 100644 --- a/modules/fundamental/src/common/test.rs +++ b/modules/fundamental/src/common/test.rs @@ -227,7 +227,7 @@ impl UpdateAssignments { let initial_etag = self.etag.clone(); let request = TestRequest::put() - .uri(&format!("/api/v3/group/sbom-assignment/{}", &self.sbom_id)) + .uri(&format!("/api/v3/group/sbom-assignment/{}", self.sbom_id)) .set_json(&self.group_ids); let request = match self.etag { diff --git a/modules/fundamental/src/purl/endpoints/mod.rs b/modules/fundamental/src/purl/endpoints/mod.rs index 57d9d4d3a..7577ae61a 100644 --- a/modules/fundamental/src/purl/endpoints/mod.rs +++ b/modules/fundamental/src/purl/endpoints/mod.rs @@ -44,7 +44,7 @@ pub fn configure( } #[utoipa::path( - operation_id = "getPurl", + operation_id = "getPurlV3a", tag = "purl", params( Deprecation, @@ -54,7 +54,7 @@ pub fn configure( (status = 200, description = "Details for the qualified PURL", body = PurlDetails), ), )] -#[get("/v3/purl/{key}")] +#[get("/v3a/purl/{key}")] /// Retrieve details of a fully-qualified pURL pub async fn get( service: web::Data, @@ -130,14 +130,14 @@ mod v3 { use super::*; #[utoipa::path( - operation_id = "recommend", + operation_id = "recommendV3a", tag = "purl", request_body = RecommendRequest, responses( (status = 200, description = "Get recommendations and remediations for provided purls", body = RecommendResponse) ) )] - #[post("/v3/purl/recommend")] + #[post("/v3a/purl/recommend")] pub async fn recommend( purl_service: web::Data, db: web::Data, diff --git a/modules/fundamental/src/sbom/endpoints/mod.rs b/modules/fundamental/src/sbom/endpoints/mod.rs index f9063b108..cbe43b7f5 100644 --- a/modules/fundamental/src/sbom/endpoints/mod.rs +++ b/modules/fundamental/src/sbom/endpoints/mod.rs @@ -35,6 +35,7 @@ use trustify_auth::{ authenticator::user::UserInformation, authorizer::{Authorizer, Require}, }; +use trustify_common::db::change::{ChangeEntity, ChangeOperation, record_change}; use trustify_common::{ db::{self, pagination_cache::PaginationCache, query::Query}, decompress::decompress_async, @@ -231,7 +232,7 @@ mod v3 { /// List SBOMs #[utoipa::path( tag = "sbom", - operation_id = "listSboms", + operation_id = "listSbomsV3a", params( Query, Paginated, @@ -242,7 +243,7 @@ mod v3 { (status = 200, description = "Matching SBOMs", body = PaginatedResults>), ), )] - #[get("/v3/sbom")] + #[get("/v3a/sbom")] #[allow(clippy::too_many_arguments)] pub async fn all( fetch: web::Data, @@ -367,10 +368,10 @@ pub async fn get( } } -/// Get advisories for an SBOM +/// Get advisories for an SBOM (SQL-based, replaced by in-memory correlation on /v3) #[utoipa::path( tag = "sbom", - operation_id = "getSbomAdvisories", + operation_id = "getSbomAdvisoriesV3a", params( ("id" = Id, Path), ), @@ -379,7 +380,7 @@ pub async fn get( (status = 404, description = "The SBOM could not be found"), ), )] -#[get("/v3/sbom/{id}/advisory")] +#[get("/v3a/sbom/{id}/advisory")] pub async fn get_sbom_advisories( fetcher: web::Data, db: web::Data, @@ -438,6 +439,13 @@ pub async fn delete( && let digests = service.delete_sboms(vec![v.sbom_id], &tx).await? && !digests.is_empty() { + record_change( + &tx, + ChangeEntity::Sbom, + Some(v.sbom_id), + ChangeOperation::Deleted, + ) + .await?; tx.commit().await?; delete_blobs(&digests, i.storage()).await; } @@ -467,14 +475,17 @@ pub async fn delete_many( ) -> Result { let tx = db.begin().await?; - let ids = body + let ids: Vec = body .into_iter() .filter_map(|x| Uuid::try_parse(&x).ok()) .collect(); - let digests = service.delete_sboms(ids, &tx).await?; + let digests = service.delete_sboms(ids.clone(), &tx).await?; if !digests.is_empty() { + for &id in &ids { + record_change(&tx, ChangeEntity::Sbom, Some(id), ChangeOperation::Deleted).await?; + } tx.commit().await?; delete_blobs(&digests, i.storage()).await; } diff --git a/modules/fundamental/src/sbom/model/mod.rs b/modules/fundamental/src/sbom/model/mod.rs index 06d3b1ea2..8d0c0e704 100644 --- a/modules/fundamental/src/sbom/model/mod.rs +++ b/modules/fundamental/src/sbom/model/mod.rs @@ -44,7 +44,7 @@ pub enum AffectedSeverity { impl From> for AffectedSeverity { fn from(value: Option) -> Self { match value { - Option::None => AffectedSeverity::Unknown, + None => AffectedSeverity::Unknown, Some(Severity::None) => AffectedSeverity::None, Some(Severity::Low) => AffectedSeverity::Low, Some(Severity::Medium) => AffectedSeverity::Medium, diff --git a/modules/fundamental/src/sbom_group/endpoints/test/mod.rs b/modules/fundamental/src/sbom_group/endpoints/test/mod.rs index 6c32e66e1..85de19d5e 100644 --- a/modules/fundamental/src/sbom_group/endpoints/test/mod.rs +++ b/modules/fundamental/src/sbom_group/endpoints/test/mod.rs @@ -76,7 +76,7 @@ impl Update { } let request = TestRequest::put() - .uri(&format!("/api/v3/group/sbom/{}", &self.id)) + .uri(&format!("/api/v3/group/sbom/{}", self.id)) .set_json(update_body); let request = add_if_match(request, self.if_match_type, &self.etag); diff --git a/modules/fundamental/src/vulnerability/endpoints/mod.rs b/modules/fundamental/src/vulnerability/endpoints/mod.rs index a9617b8ce..8419eedbb 100644 --- a/modules/fundamental/src/vulnerability/endpoints/mod.rs +++ b/modules/fundamental/src/vulnerability/endpoints/mod.rs @@ -91,7 +91,7 @@ pub async fn all( #[utoipa::path( tag = "vulnerability", - operation_id = "getVulnerability", + operation_id = "getVulnerabilityV3a", params( ("id", Path, description = "ID of the vulnerability"), VulnerabilityGetParams, @@ -101,7 +101,7 @@ pub async fn all( (status = 404, description = "The vulnerability could not be found"), ), )] -#[get("/v3/vulnerability/{id}")] +#[get("/v3a/vulnerability/{id}")] /// Retrieve vulnerability details pub async fn get( state: web::Data, @@ -146,14 +146,14 @@ pub async fn analyze( } #[utoipa::path( - operation_id = "analyze_v3", + operation_id = "analyze_v3a", tag = "vulnerability", request_body = AnalysisRequest, responses( (status = 200, description = "Analyze the provided purls to search for known vulnerabilities", body = AnalysisResponseV3), ), )] -#[post("/v3/vulnerability/analyze")] +#[post("/v3a/vulnerability/analyze")] pub async fn analyze_v3( service: web::Data, db: web::Data, diff --git a/modules/fundamental/src/vulnerability/model/mod.rs b/modules/fundamental/src/vulnerability/model/mod.rs index 41b151810..9cb341c98 100644 --- a/modules/fundamental/src/vulnerability/model/mod.rs +++ b/modules/fundamental/src/vulnerability/model/mod.rs @@ -1,4 +1,4 @@ -mod analyze; +pub mod analyze; mod details; mod summary; pub mod v2; diff --git a/modules/ingestor/src/service/mod.rs b/modules/ingestor/src/service/mod.rs index ca2640306..9585ea7e6 100644 --- a/modules/ingestor/src/service/mod.rs +++ b/modules/ingestor/src/service/mod.rs @@ -24,6 +24,7 @@ use sea_orm::{ConnectionTrait, TransactionTrait}; use std::{fmt::Debug, sync::Arc, time::Instant}; use tokio::task::JoinError; use tracing::instrument; +use trustify_common::db::change::{ChangeEntity, ChangeOperation, record_change}; use trustify_common::{db::DatabaseErrors, error::ErrorInformation, id::IdError}; use trustify_entity::labels::Labels; use trustify_module_analysis::service::AnalysisService; @@ -240,6 +241,22 @@ impl IngestorService { .load(&self.graph, labels.into(), issuer, &result.digests, tx) .await?; + let change_entity = match fmt { + Format::CSAF | Format::CVE | Format::OSV => Some(ChangeEntity::Advisory), + Format::SPDX | Format::CycloneDX => Some(ChangeEntity::Sbom), + _ => None, + }; + if let Some(entity_type) = change_entity { + record_change( + tx, + entity_type, + uuid::Uuid::try_parse(&result.id).ok(), + ChangeOperation::Added, + ) + .await + .map_err(|err| Error::Storage(anyhow!("{err}")))?; + } + if let Some(wait) = cache.into() { self.load_graph_cache(fmt, &result, wait).await; } diff --git a/modules/notification/Cargo.toml b/modules/notification/Cargo.toml new file mode 100644 index 000000000..b42a0905c --- /dev/null +++ b/modules/notification/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "trustify-module-notification" +version.workspace = true +edition.workspace = true +publish.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +trustify-auth = { workspace = true } +trustify-common = { workspace = true } +trustify-infrastructure = { workspace = true } + +clap = { workspace = true } +humantime = { workspace = true } + +actix-http = { workspace = true } +actix-web = { workspace = true } +actix-ws = { workspace = true } +anyhow = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["sync"] } +tracing = { workspace = true } +utoipa-actix-web = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +test-context = { workspace = true } +test-log = { workspace = true, features = ["log", "trace"] } +trustify-test-context = { workspace = true } diff --git a/modules/notification/README.md b/modules/notification/README.md new file mode 100644 index 000000000..bb4fc7637 --- /dev/null +++ b/modules/notification/README.md @@ -0,0 +1,95 @@ +# Notification Module + +WebSocket endpoint that streams `change_log` events (advisory/SBOM additions and deletions) in real time. + +## Protocol + +- **Endpoint**: `GET /api/v3/notifications?after=` +- **Auth**: Bearer token via `Authorization` header or `?token=` query parameter. Requires at least one of `read.sbom` or `read.advisory` — events are filtered per-entity by the caller's permissions +- **`after`**: last known cursor; omit on first connect (the server sends a `connection` message with the current cursor), provide on reconnect to replay missed events +- **Heartbeat**: server sends a WebSocket ping every 30 s; clients should respond with pong (most WebSocket libraries do this automatically) + +### Connection message + +On first connect (no `after` parameter), the server sends a control message: + +```json +{"type":"connection","cursor":"019577ab-..."} +``` + +Save this `cursor` value. If you disconnect before any change events arrive, pass it as `?after=` on reconnect for gap-free delivery. + +### Change events + +Server → client, JSON text frames: + +```json +{"cursor":"019577ab-...","type":"sbom","id":"550e8400-...","operation":"added"} +``` + +| Field | Type | Description | +|-------------|-----------------------------|----------------------------------------------------------| +| `cursor` | `string` | Monotonically increasing event cursor (pass as `after` on reconnect) | +| `type` | `"sbom"` \| `"advisory"` | What kind of entity changed | +| `id` | `string` \| `null` | The entity that changed (null for bulk operations) | +| `operation` | `"added"` \| `"deleted"` | What happened | + +Track the `cursor` field from each message and pass it as `?after=` on reconnect for gap-free delivery. + +## Example: websocat + +```sh +websocat -H "Authorization: Bearer $TOKEN" \ + ws://localhost:8080/api/v3/notifications +``` + +To resume from a known cursor: + +```sh +websocat -H "Authorization: Bearer $TOKEN" \ + "ws://localhost:8080/api/v3/notifications?after=019577ab-0000-7000-8000-000000000000" +``` + +## Example: HTML + +Streams events with reconnect. Enter your access token before connecting. + +```html + + + + + + 

+  
+
+
+```
diff --git a/modules/notification/src/config.rs b/modules/notification/src/config.rs
new file mode 100644
index 000000000..e7c0e20ba
--- /dev/null
+++ b/modules/notification/src/config.rs
@@ -0,0 +1,11 @@
+/// Configuration for the notification service.
+#[derive(clap::Args, Debug, Clone)]
+pub struct NotificationConfig {
+    /// Retention period for change_log entries (humantime, e.g. "1h", "30m", "1d")
+    #[arg(
+        long,
+        env = "TRUSTD_CHANGE_LOG_RETENTION",
+        default_value = "1d"
+    )]
+    pub change_log_retention: humantime::Duration,
+}
diff --git a/modules/notification/src/endpoints.rs b/modules/notification/src/endpoints.rs
new file mode 100644
index 000000000..56ee2fc7d
--- /dev/null
+++ b/modules/notification/src/endpoints.rs
@@ -0,0 +1,190 @@
+use actix_web::{HttpRequest, HttpResponse, web};
+use futures::StreamExt;
+use serde::Deserialize;
+use std::sync::Arc;
+use tokio::sync::broadcast;
+use trustify_auth::{
+    Permission, authenticator::Authenticator, authenticator::user::UserInformation,
+    authorizer::Authorizer,
+};
+use trustify_common::db::change::{ChangeBroadcaster, ChangeEntity, ChangeEntry};
+use trustify_infrastructure::app::new_auth;
+use utoipa_actix_web::service_config::ServiceConfig;
+use uuid::Uuid;
+
+use crate::inject_token::QueryTokenInjector;
+
+#[derive(Debug, Deserialize)]
+pub struct NotificationQuery {
+    pub after: Option,
+    pub token: Option,
+}
+
+#[derive(serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+enum Message {
+    Connection,
+}
+
+#[derive(serde::Serialize)]
+struct ConnectionMessage {
+    r#type: Message,
+    cursor: Uuid,
+}
+
+pub fn configure(
+    config: &mut ServiceConfig,
+    broadcaster: ChangeBroadcaster,
+    auth: Option>,
+) {
+    config.app_data(web::Data::new(broadcaster)).map(|svc| {
+        svc.service(
+            web::resource("/api/v3/notifications")
+                .wrap(new_auth(auth))
+                .wrap(QueryTokenInjector)
+                .route(web::get().to(ws_handler)),
+        )
+    });
+}
+
+async fn ws_handler(
+    req: HttpRequest,
+    body: web::Payload,
+    query: web::Query,
+    broadcaster: web::Data,
+    user: UserInformation,
+) -> Result {
+    let authorizer = req
+        .app_data::>()
+        .cloned()
+        .unwrap_or_default();
+
+    let can_read_sbom = authorizer.require(&user, Permission::ReadSbom).is_ok();
+    let can_read_advisory = authorizer.require(&user, Permission::ReadAdvisory).is_ok();
+
+    if !can_read_sbom && !can_read_advisory {
+        return Ok(HttpResponse::Forbidden().finish());
+    }
+
+    let (response, session, msg_stream) = actix_ws::handle(&req, body)?;
+
+    let broadcaster = broadcaster.into_inner();
+    let after = query.into_inner().after;
+
+    actix_web::rt::spawn(async move {
+        if let Err(err) = run_ws_session(
+            session,
+            msg_stream,
+            &broadcaster,
+            after,
+            can_read_sbom,
+            can_read_advisory,
+        )
+        .await
+        {
+            tracing::warn!(%err, "WebSocket notification session error");
+        }
+    });
+
+    Ok(response)
+}
+
+pub(crate) fn is_allowed(
+    entry: &ChangeEntry,
+    can_read_sbom: bool,
+    can_read_advisory: bool,
+) -> bool {
+    match entry.r#type {
+        ChangeEntity::Sbom => can_read_sbom,
+        ChangeEntity::Advisory => can_read_advisory,
+    }
+}
+
+async fn run_ws_session(
+    mut session: actix_ws::Session,
+    mut msg_stream: actix_ws::MessageStream,
+    broadcaster: &ChangeBroadcaster,
+    after: Option,
+    can_read_sbom: bool,
+    can_read_advisory: bool,
+) -> Result<(), anyhow::Error> {
+    // Subscribe before backfill/cursor fetch to avoid gaps.
+    let mut rx = broadcaster.subscribe();
+
+    if let Some(cursor) = after {
+        match broadcaster.fetch_after(&cursor).await {
+            Ok(entries) => {
+                for entry in entries {
+                    if !is_allowed(&entry, can_read_sbom, can_read_advisory) {
+                        continue;
+                    }
+                    let json = serde_json::to_string(&entry)?;
+                    if session.text(json).await.is_err() {
+                        return Ok(());
+                    }
+                }
+            }
+            Err(err) => {
+                tracing::warn!(%err, "notification backfill query failed");
+            }
+        }
+    } else {
+        let cursor = broadcaster.fetch_latest_cursor().await;
+        let msg = ConnectionMessage {
+            r#type: Message::Connection,
+            cursor,
+        };
+        let json = serde_json::to_string(&msg)?;
+        if session.text(json).await.is_err() {
+            return Ok(());
+        }
+    }
+
+    let mut heartbeat = tokio::time::interval(std::time::Duration::from_secs(30));
+
+    loop {
+        tokio::select! {
+            event = rx.recv() => {
+                match event {
+                    Ok(entry) => {
+                        if !is_allowed(&entry, can_read_sbom, can_read_advisory) {
+                            continue;
+                        }
+                        let json = serde_json::to_string(&entry)?;
+                        if session.text(json).await.is_err() {
+                            break;
+                        }
+                    }
+                    Err(broadcast::error::RecvError::Lagged(n)) => {
+                        tracing::warn!(n, "WebSocket notification client lagged");
+                    }
+                    Err(broadcast::error::RecvError::Closed) => {
+                        break;
+                    }
+                }
+            }
+
+            msg = msg_stream.next() => {
+                match msg {
+                    Some(Ok(actix_ws::Message::Ping(data))) => {
+                        let _ = session.pong(&data).await;
+                    }
+                    Some(Ok(actix_ws::Message::Close(reason))) => {
+                        let _ = session.close(reason).await;
+                        break;
+                    }
+                    Some(Err(_)) | None => break,
+                    _ => {}
+                }
+            }
+
+            _ = heartbeat.tick() => {
+                if session.ping(b"").await.is_err() {
+                    break;
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
diff --git a/modules/notification/src/inject_token.rs b/modules/notification/src/inject_token.rs
new file mode 100644
index 000000000..accf1f821
--- /dev/null
+++ b/modules/notification/src/inject_token.rs
@@ -0,0 +1,57 @@
+use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform};
+use actix_web::http::header;
+use futures::future::{LocalBoxFuture, Ready, ok};
+use std::task::{Context, Poll};
+
+pub struct QueryTokenInjector;
+
+impl Transform for QueryTokenInjector
+where
+    S: Service, Error = actix_web::Error> + 'static,
+{
+    type Response = ServiceResponse;
+    type Error = actix_web::Error;
+    type Transform = QueryTokenInjectorMiddleware;
+    type InitError = ();
+    type Future = Ready>;
+
+    fn new_transform(&self, service: S) -> Self::Future {
+        ok(QueryTokenInjectorMiddleware { service })
+    }
+}
+
+pub struct QueryTokenInjectorMiddleware {
+    service: S,
+}
+
+impl Service for QueryTokenInjectorMiddleware
+where
+    S: Service, Error = actix_web::Error> + 'static,
+{
+    type Response = ServiceResponse;
+    type Error = actix_web::Error;
+    type Future = LocalBoxFuture<'static, Result>;
+
+    fn poll_ready(&self, cx: &mut Context<'_>) -> Poll> {
+        self.service.poll_ready(cx)
+    }
+
+    fn call(&self, mut req: ServiceRequest) -> Self::Future {
+        if req.headers().get(header::AUTHORIZATION).is_none()
+            && let Some(token) = extract_token(req.query_string())
+            && let Ok(value) = format!("Bearer {token}").parse()
+        {
+            req.headers_mut().insert(header::AUTHORIZATION, value);
+        }
+
+        let fut = self.service.call(req);
+        Box::pin(fut)
+    }
+}
+
+pub(crate) fn extract_token(query: &str) -> Option<&str> {
+    query.split('&').find_map(|pair| {
+        let (key, value) = pair.split_once('=')?;
+        (key == "token").then_some(value)
+    })
+}
diff --git a/modules/notification/src/lib.rs b/modules/notification/src/lib.rs
new file mode 100644
index 000000000..c3ec8b685
--- /dev/null
+++ b/modules/notification/src/lib.rs
@@ -0,0 +1,6 @@
+pub mod config;
+pub mod endpoints;
+pub(crate) mod inject_token;
+
+#[cfg(test)]
+mod test;
diff --git a/modules/notification/src/test.rs b/modules/notification/src/test.rs
new file mode 100644
index 000000000..87ac451f1
--- /dev/null
+++ b/modules/notification/src/test.rs
@@ -0,0 +1,336 @@
+#![cfg(test)]
+
+use actix_web::{App, HttpRequest, HttpResponse, http::StatusCode, test as actix, web};
+use std::time::Duration;
+use test_context::test_context;
+use test_log::test;
+use trustify_auth::{
+    authenticator::user::UserDetails,
+    authorizer::{Authorizer, AuthorizerConfig},
+};
+use trustify_common::db;
+use trustify_common::db::change::{
+    ChangeBroadcaster, ChangeEntity, ChangeEntry, ChangeOperation, record_change,
+};
+use trustify_test_context::TrustifyContext;
+use trustify_test_context::auth::TestAuthentication;
+use utoipa_actix_web::AppExt;
+use uuid::Uuid;
+
+// -- Group A: extract_token -------------------------------------------------
+
+#[test]
+fn extract_token_basic() {
+    assert_eq!(
+        crate::inject_token::extract_token("token=abc123"),
+        Some("abc123")
+    );
+}
+
+#[test]
+fn extract_token_with_other_params() {
+    assert_eq!(
+        crate::inject_token::extract_token("after=xxx&token=jwt.val&foo=bar"),
+        Some("jwt.val")
+    );
+}
+
+#[test]
+fn extract_token_missing() {
+    assert_eq!(
+        crate::inject_token::extract_token("after=xxx&foo=bar"),
+        None
+    );
+}
+
+#[test]
+fn extract_token_empty() {
+    assert_eq!(crate::inject_token::extract_token(""), None);
+}
+
+// -- Group B: is_allowed ----------------------------------------------------
+
+fn dummy_entry(r#type: ChangeEntity) -> ChangeEntry {
+    ChangeEntry {
+        cursor: Uuid::now_v7(),
+        r#type,
+        id: Some(Uuid::now_v7()),
+        operation: ChangeOperation::Added,
+    }
+}
+
+#[test]
+fn is_allowed_sbom_with_perm() {
+    assert!(crate::endpoints::is_allowed(
+        &dummy_entry(ChangeEntity::Sbom),
+        true,
+        false
+    ));
+}
+
+#[test]
+fn is_allowed_sbom_without_perm() {
+    assert!(!crate::endpoints::is_allowed(
+        &dummy_entry(ChangeEntity::Sbom),
+        false,
+        true
+    ));
+}
+
+#[test]
+fn is_allowed_advisory_with_perm() {
+    assert!(crate::endpoints::is_allowed(
+        &dummy_entry(ChangeEntity::Advisory),
+        false,
+        true
+    ));
+}
+
+#[test]
+fn is_allowed_advisory_without_perm() {
+    assert!(!crate::endpoints::is_allowed(
+        &dummy_entry(ChangeEntity::Advisory),
+        true,
+        false
+    ));
+}
+
+#[test]
+fn is_allowed_no_perms() {
+    assert!(!crate::endpoints::is_allowed(
+        &dummy_entry(ChangeEntity::Sbom),
+        false,
+        false
+    ));
+    assert!(!crate::endpoints::is_allowed(
+        &dummy_entry(ChangeEntity::Advisory),
+        false,
+        false
+    ));
+}
+
+// -- Group C: QueryTokenInjector middleware ----------------------------------
+
+async fn echo_auth(req: HttpRequest) -> HttpResponse {
+    match req.headers().get("Authorization") {
+        Some(val) => HttpResponse::Ok().body(val.to_str().unwrap_or("bad").to_string()),
+        None => HttpResponse::NoContent().finish(),
+    }
+}
+
+#[test(actix_web::test)]
+async fn injector_copies_token() {
+    let app = actix::init_service(
+        App::new().service(
+            web::resource("/test")
+                .wrap(crate::inject_token::QueryTokenInjector)
+                .route(web::get().to(echo_auth)),
+        ),
+    )
+    .await;
+
+    let req = actix::TestRequest::get()
+        .uri("/test?token=mytoken")
+        .to_request();
+    let resp = actix::call_service(&app, req).await;
+    assert_eq!(resp.status(), StatusCode::OK);
+    let body = actix::read_body(resp).await;
+    assert_eq!(body, "Bearer mytoken");
+}
+
+#[test(actix_web::test)]
+async fn injector_preserves_existing_header() {
+    let app = actix::init_service(
+        App::new().service(
+            web::resource("/test")
+                .wrap(crate::inject_token::QueryTokenInjector)
+                .route(web::get().to(echo_auth)),
+        ),
+    )
+    .await;
+
+    let req = actix::TestRequest::get()
+        .uri("/test?token=other")
+        .append_header(("Authorization", "Bearer existing"))
+        .to_request();
+    let resp = actix::call_service(&app, req).await;
+    let body = actix::read_body(resp).await;
+    assert_eq!(body, "Bearer existing");
+}
+
+#[test(actix_web::test)]
+async fn injector_no_token_no_header() {
+    let app = actix::init_service(
+        App::new().service(
+            web::resource("/test")
+                .wrap(crate::inject_token::QueryTokenInjector)
+                .route(web::get().to(echo_auth)),
+        ),
+    )
+    .await;
+
+    let req = actix::TestRequest::get().uri("/test").to_request();
+    let resp = actix::call_service(&app, req).await;
+    assert_eq!(resp.status(), StatusCode::NO_CONTENT);
+}
+
+// -- Group D: endpoint permission tests -------------------------------------
+
+fn user_with_permissions(perms: &[&str]) -> UserDetails {
+    UserDetails {
+        id: "test-user".into(),
+        permissions: perms.iter().map(|s| s.to_string()).collect(),
+    }
+}
+
+#[test_context(TrustifyContext, skip_teardown)]
+#[test(actix_web::test)]
+async fn ws_anonymous_forbidden(ctx: TrustifyContext) {
+    let db_rw = db::ReadWrite::new(ctx.db.clone());
+    let broadcaster = ChangeBroadcaster::new(&db_rw, Duration::from_secs(86400)).expect("broadcaster");
+    let authorizer = Authorizer::new(Some(AuthorizerConfig {}));
+
+    let app = actix::init_service(
+        App::new()
+            .into_utoipa_app()
+            .app_data(web::Data::new(authorizer))
+            .configure(|svc| {
+                crate::endpoints::configure(svc, broadcaster, None);
+            })
+            .into_app(),
+    )
+    .await;
+
+    let req = actix::TestRequest::get()
+        .uri("/api/v3/notifications")
+        .to_request();
+    let resp = actix::call_service(&app, req).await;
+    assert_eq!(resp.status(), StatusCode::FORBIDDEN);
+}
+
+#[test_context(TrustifyContext, skip_teardown)]
+#[test(actix_web::test)]
+async fn ws_no_permissions_forbidden(ctx: TrustifyContext) {
+    let db_rw = db::ReadWrite::new(ctx.db.clone());
+    let broadcaster = ChangeBroadcaster::new(&db_rw, Duration::from_secs(86400)).expect("broadcaster");
+    let authorizer = Authorizer::new(Some(AuthorizerConfig {}));
+
+    let app = actix::init_service(
+        App::new()
+            .into_utoipa_app()
+            .app_data(web::Data::new(authorizer))
+            .configure(|svc| {
+                crate::endpoints::configure(svc, broadcaster, None);
+            })
+            .into_app(),
+    )
+    .await;
+
+    let req = actix::TestRequest::get()
+        .uri("/api/v3/notifications")
+        .to_request()
+        .test_auth_details(user_with_permissions(&[]));
+    let resp = actix::call_service(&app, req).await;
+    assert_eq!(resp.status(), StatusCode::FORBIDDEN);
+}
+
+#[test_context(TrustifyContext, skip_teardown)]
+#[test(actix_web::test)]
+async fn ws_read_sbom_accepted(ctx: TrustifyContext) {
+    let db_rw = db::ReadWrite::new(ctx.db.clone());
+    let broadcaster = ChangeBroadcaster::new(&db_rw, Duration::from_secs(86400)).expect("broadcaster");
+    let authorizer = Authorizer::new(Some(AuthorizerConfig {}));
+
+    let app = actix::init_service(
+        App::new()
+            .into_utoipa_app()
+            .app_data(web::Data::new(authorizer))
+            .configure(|svc| {
+                crate::endpoints::configure(svc, broadcaster, None);
+            })
+            .into_app(),
+    )
+    .await;
+
+    let req = actix::TestRequest::get()
+        .uri("/api/v3/notifications")
+        .to_request()
+        .test_auth_details(user_with_permissions(&["read.sbom"]));
+    let resp = actix::call_service(&app, req).await;
+    // Not 403 — passed the permission gate (will fail at WS upgrade since no upgrade headers)
+    assert_ne!(resp.status(), StatusCode::FORBIDDEN);
+}
+
+#[test_context(TrustifyContext, skip_teardown)]
+#[test(actix_web::test)]
+async fn ws_read_advisory_accepted(ctx: TrustifyContext) {
+    let db_rw = db::ReadWrite::new(ctx.db.clone());
+    let broadcaster = ChangeBroadcaster::new(&db_rw, Duration::from_secs(86400)).expect("broadcaster");
+    let authorizer = Authorizer::new(Some(AuthorizerConfig {}));
+
+    let app = actix::init_service(
+        App::new()
+            .into_utoipa_app()
+            .app_data(web::Data::new(authorizer))
+            .configure(|svc| {
+                crate::endpoints::configure(svc, broadcaster, None);
+            })
+            .into_app(),
+    )
+    .await;
+
+    let req = actix::TestRequest::get()
+        .uri("/api/v3/notifications")
+        .to_request()
+        .test_auth_details(user_with_permissions(&["read.advisory"]));
+    let resp = actix::call_service(&app, req).await;
+    assert_ne!(resp.status(), StatusCode::FORBIDDEN);
+}
+
+// -- Group E: ChangeBroadcaster::fetch_after DB test ------------------------
+
+#[test_context(TrustifyContext, skip_teardown)]
+#[test(tokio::test)]
+async fn fetch_after_returns_newer_entries(ctx: TrustifyContext) {
+    let db_rw = db::ReadWrite::new(ctx.db.clone());
+    let broadcaster = ChangeBroadcaster::new(&db_rw, Duration::from_secs(86400)).expect("broadcaster");
+
+    // Insert 3 entries with small delays so UUIDv7 ordering is preserved
+    record_change(
+        &ctx.db,
+        ChangeEntity::Sbom,
+        Some(Uuid::now_v7()),
+        ChangeOperation::Added,
+    )
+    .await
+    .unwrap();
+    tokio::time::sleep(Duration::from_millis(2)).await;
+    record_change(
+        &ctx.db,
+        ChangeEntity::Advisory,
+        Some(Uuid::now_v7()),
+        ChangeOperation::Added,
+    )
+    .await
+    .unwrap();
+    tokio::time::sleep(Duration::from_millis(2)).await;
+    record_change(
+        &ctx.db,
+        ChangeEntity::Sbom,
+        Some(Uuid::now_v7()),
+        ChangeOperation::Deleted,
+    )
+    .await
+    .unwrap();
+
+    let all = broadcaster.fetch_after(&Uuid::nil()).await.unwrap();
+    assert!(all.len() >= 3);
+
+    let first_cursor = all[all.len() - 3].cursor;
+    let after_first = broadcaster.fetch_after(&first_cursor).await.unwrap();
+    assert_eq!(after_first.len(), 2);
+
+    let last_cursor = all.last().unwrap().cursor;
+    let after_last = broadcaster.fetch_after(&last_cursor).await.unwrap();
+    assert!(after_last.is_empty());
+}
diff --git a/openapi.yaml b/openapi.yaml
index 653e46cfe..996ad11fd 100644
--- a/openapi.yaml
+++ b/openapi.yaml
@@ -1005,6 +1005,19 @@ paths:
           description: The user did not provide valid authentication credentials
         '403':
           description: The user lacks the required permission
+  /api/v3/correlation/status:
+    get:
+      tags:
+      - correlation
+      summary: Get the status of the correlation service.
+      operationId: getCorrelationStatus
+      responses:
+        '200':
+          description: Correlation service status
+        '401':
+          description: The user did not provide valid authentication credentials
+        '403':
+          description: The user lacks the required permission
   /api/v3/dataset:
     post:
       tags:
@@ -2528,7 +2541,12 @@ paths:
   /api/v3/purl/recommend:
     post:
       tags:
-      - purl
+      - correlation
+      summary: Recommend Red Hat patched versions using in-memory correlation.
+      description: |-
+        Finds the highest Red Hat patch version for each input PURL (same major.minor.patch
+        with a `redhat-NNNNN` suffix), then uses in-memory correlation to determine
+        which vulnerabilities affect those patched versions.
       operationId: recommend
       requestBody:
         content:
@@ -2538,26 +2556,27 @@ paths:
         required: true
       responses:
         '200':
-          description: Get recommendations and remediations for provided purls
+          description: Recommendations and remediations for provided PURLs
           content:
             application/json:
               schema:
                 $ref: '#/components/schemas/RecommendResponse'
+        '401':
+          description: The user did not provide valid authentication credentials
+        '403':
+          description: The user lacks the required permission
+        '503':
+          description: Correlation service not ready
   /api/v3/purl/{key}:
     get:
       tags:
-      - purl
-      summary: Retrieve details of a fully-qualified pURL
+      - correlation
+      summary: Retrieve PURL details with in-memory advisory correlation.
+      description: |-
+        Loads PURL head/version/base/license data from the database, then replaces
+        the advisory matching with in-memory correlation results.
       operationId: getPurl
       parameters:
-      - name: deprecated
-        in: query
-        required: false
-        schema:
-          type: string
-          enum:
-          - Ignore
-          - Consider
       - name: key
         in: path
         description: opaque identifier for a fully-qualified PURL, or URL-encoded pURL itself
@@ -2571,11 +2590,19 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/PurlDetails'
+        '401':
+          description: The user did not provide valid authentication credentials
+        '403':
+          description: The user lacks the required permission
+        '404':
+          description: PURL not found
+        '503':
+          description: Correlation service not ready
   /api/v3/sbom:
     get:
       tags:
-      - sbom
-      summary: List SBOMs
+      - correlation
+      summary: List SBOMs with in-memory severity counts replacing the SQL-based advisory summary.
       operationId: listSboms
       parameters:
       - name: q
@@ -2682,9 +2709,6 @@ paths:
           type: boolean
       - name: group
         in: query
-        description: |-
-          Filter by group IDs. Only SBOMs assigned to any of the provided groups will be returned.
-          Can be specified multiple times. Malformed IDs are silently ignored.
         required: false
         schema:
           type: array
@@ -2692,7 +2716,7 @@ paths:
             type: string
       - name: advisories
         in: query
-        description: Include advisory severity summary per SBOM
+        description: Include advisory severity summary per SBOM.
         required: false
         schema:
           type: boolean
@@ -2703,6 +2727,10 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/PaginatedResults_SbomSummary_SbomPackageSummary'
+        '401':
+          description: The user did not provide valid authentication credentials
+        '403':
+          description: The user lacks the required permission
     post:
       tags:
       - sbom
@@ -3160,26 +3188,28 @@ paths:
   /api/v3/sbom/{id}/advisory:
     get:
       tags:
-      - sbom
-      summary: Get advisories for an SBOM
-      operationId: getSbomAdvisories
+      - correlation
+      summary: Find advisories affecting an SBOM using in-memory correlation.
+      operationId: getCorrelationSbomAdvisories
       parameters:
       - name: id
         in: path
+        description: SBOM ID
         required: true
         schema:
-          $ref: '#/components/schemas/Id'
+          type: string
+          format: uuid
       responses:
         '200':
-          description: Matching SBOM
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  $ref: '#/components/schemas/SbomAdvisory'
+          description: Advisories affecting this SBOM
+        '401':
+          description: The user did not provide valid authentication credentials
+        '403':
+          description: The user lacks the required permission
         '404':
-          description: The SBOM could not be found
+          description: SBOM not found
+        '503':
+          description: Correlation service not ready
   /api/v3/sbom/{id}/all-license-ids:
     get:
       tags:
@@ -3914,7 +3944,8 @@ paths:
   /api/v3/vulnerability/analyze:
     post:
       tags:
-      - vulnerability
+      - correlation
+      summary: Analyze PURLs for known vulnerabilities using in-memory correlation.
       operationId: analyze_v3
       requestBody:
         content:
@@ -3924,16 +3955,23 @@ paths:
         required: true
       responses:
         '200':
-          description: Analyze the provided purls to search for known vulnerabilities
+          description: Vulnerability analysis results
           content:
             application/json:
               schema:
                 $ref: '#/components/schemas/AnalysisResponseV3'
+        '401':
+          description: The user did not provide valid authentication credentials
+        '403':
+          description: The user lacks the required permission
   /api/v3/vulnerability/{id}:
     get:
       tags:
-      - vulnerability
-      summary: Retrieve vulnerability details
+      - correlation
+      summary: Retrieve vulnerability details using in-memory correlation and DB hydration.
+      description: |-
+        Loads the vulnerability entity from the database, uses in-memory correlation
+        to identify affected SBOMs, then hydrates the response from the database.
       operationId: getVulnerability
       parameters:
       - name: id
@@ -3955,8 +3993,14 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/VulnerabilityDetails'
+        '401':
+          description: The user did not provide valid authentication credentials
+        '403':
+          description: The user lacks the required permission
         '404':
           description: The vulnerability could not be found
+        '503':
+          description: Correlation service not ready
   /api/v3/weakness:
     get:
       tags:
@@ -4094,6 +4138,253 @@ paths:
                 $ref: '#/components/schemas/LicenseSummary'
         '404':
           description: The weakness could not be found
+  /api/v3a/purl/recommend:
+    post:
+      tags:
+      - purl
+      operationId: recommendV3a
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/RecommendRequest'
+        required: true
+      responses:
+        '200':
+          description: Get recommendations and remediations for provided purls
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RecommendResponse'
+  /api/v3a/purl/{key}:
+    get:
+      tags:
+      - purl
+      summary: Retrieve details of a fully-qualified pURL
+      operationId: getPurlV3a
+      parameters:
+      - name: deprecated
+        in: query
+        required: false
+        schema:
+          type: string
+          enum:
+          - Ignore
+          - Consider
+      - name: key
+        in: path
+        description: opaque identifier for a fully-qualified PURL, or URL-encoded pURL itself
+        required: true
+        schema:
+          type: string
+      responses:
+        '200':
+          description: Details for the qualified PURL
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PurlDetails'
+  /api/v3a/sbom:
+    get:
+      tags:
+      - sbom
+      summary: List SBOMs
+      operationId: listSbomsV3a
+      parameters:
+      - name: q
+        in: query
+        description: |
+          EBNF grammar for the _q_ parameter:
+          ```text
+              q = ( values | filter ) { '&' q }
+              values = value { '|', values }
+              filter = field, operator, values
+              operator = "=" | "!=" | "~" | "!~" | ">=" | ">" | "<=" | "<"
+              value = (* any text but escape special characters with '\' *)
+              field = (* must match an entity attribute name *)
+          ```
+          Any values in a _q_ will result in a case-insensitive "full
+          text search", effectively producing an OR clause of LIKE
+          clauses for every string-ish field in the resource being
+          queried.
+
+          Examples:
+          - `foo` - any field containing 'foo'
+          - `foo|bar` - any field containing either 'foo' OR 'bar'
+          - `foo&bar` - some field contains 'foo' AND some field contains 'bar'
+
+          A _filter_ may also be used to constrain the results. The
+          filter's field name must correspond to one of the resource's
+          attributes. If it doesn't, an error will be returned
+          containing a list of the valid fields for that resource.
+
+          An ASCII value of `NUL`, percent-encoded as `%00`, may be used
+          to find resources on which a particular field isn't set. For
+          example, `name=%00` and `name!=%00` yield the WHERE clauses,
+          'NAME IS NULL' and 'NAME IS NOT NULL', respectively.
+
+          Examples:
+          - `name=foo` - entity's _name_ matches 'foo' exactly
+          - `name~foo` - entity's _name_ contains 'foo', case-insensitive
+          - `name~foo|bar` - entity's _name_ contains either 'foo' OR 'bar', case-insensitive
+          - `name=` - entity's _name_ is the empty string, ''
+          - `name=%00` - entity's _name_ isn't set
+          - `published>3 days ago` - date values can be "human time"
+
+          Multiple full text searches and/or filters should be
+          '&'-delimited -- they are logically AND'd together.
+
+          - `red hat|fedora&labels:type=cve|osv&published>last wednesday 17:00`
+
+          Fields corresponding to JSON objects in the database may use a
+          ':' to delimit the column name and the object key,
+          e.g. `purl:qualifiers:type=pom`
+
+          Any operator or special character, e.g. '|', '&', within a
+          value should be escaped by prefixing it with a backslash.
+        required: false
+        schema:
+          type: string
+      - name: sort
+        in: query
+        description: |
+          EBNF grammar for the _sort_ parameter:
+          ```text
+              sort = field [ ':', order ] { ',' sort }
+              order = ( "asc" | "desc" )
+              field = (* must match the name of entity's attributes *)
+          ```
+          The optional _order_ should be one of "asc" or "desc". If
+          omitted, the order defaults to "asc".
+
+          Each _field_ name must correspond to one of the columns of the
+          table holding the entities being queried. Those corresponding
+          to JSON objects in the database may use a ':' to delimit the
+          column name and the object key,
+          e.g. `purl:qualifiers:type:desc`
+        required: false
+        schema:
+          type: string
+      - name: offset
+        in: query
+        description: |-
+          The first item to return, skipping all that come before it.
+
+          NOTE: The order of items is defined by the API being called.
+        required: false
+        schema:
+          type: integer
+          format: int64
+          minimum: 0
+      - name: limit
+        in: query
+        description: |-
+          The maximum number of entries to return.
+
+          Zero means: return no items (the total count is still computed if requested).
+        required: false
+        schema:
+          type: integer
+          format: int64
+          minimum: 0
+      - name: total
+        in: query
+        description: Whether to compute and return the total count of matching items.
+        required: false
+        schema:
+          type: boolean
+      - name: group
+        in: query
+        description: |-
+          Filter by group IDs. Only SBOMs assigned to any of the provided groups will be returned.
+          Can be specified multiple times. Malformed IDs are silently ignored.
+        required: false
+        schema:
+          type: array
+          items:
+            type: string
+      - name: advisories
+        in: query
+        description: Include advisory severity summary per SBOM
+        required: false
+        schema:
+          type: boolean
+      responses:
+        '200':
+          description: Matching SBOMs
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PaginatedResults_SbomSummary_SbomPackageSummary'
+  /api/v3a/sbom/{id}/advisory:
+    get:
+      tags:
+      - sbom
+      summary: Get advisories for an SBOM (SQL-based, replaced by in-memory correlation on /v3)
+      operationId: getSbomAdvisoriesV3a
+      parameters:
+      - name: id
+        in: path
+        required: true
+        schema:
+          $ref: '#/components/schemas/Id'
+      responses:
+        '200':
+          description: Matching SBOM
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/SbomAdvisory'
+        '404':
+          description: The SBOM could not be found
+  /api/v3a/vulnerability/analyze:
+    post:
+      tags:
+      - vulnerability
+      operationId: analyze_v3a
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/AnalysisRequest'
+        required: true
+      responses:
+        '200':
+          description: Analyze the provided purls to search for known vulnerabilities
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/AnalysisResponseV3'
+  /api/v3a/vulnerability/{id}:
+    get:
+      tags:
+      - vulnerability
+      summary: Retrieve vulnerability details
+      operationId: getVulnerabilityV3a
+      parameters:
+      - name: id
+        in: path
+        description: ID of the vulnerability
+        required: true
+        schema:
+          type: string
+      - name: scores
+        in: query
+        description: Include the full scores array from the advisory that contributed the base_score.
+        required: false
+        schema:
+          type: boolean
+      responses:
+        '200':
+          description: Specified vulnerability
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/VulnerabilityDetails'
+        '404':
+          description: The vulnerability could not be found
 components:
   schemas:
     AdvisoryDetails:
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 3db494c9d..def45e51b 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -12,9 +12,11 @@ trustify-common = { workspace = true }
 trustify-db = { workspace = true }
 trustify-infrastructure = { workspace = true }
 trustify-module-analysis = { workspace = true }
+trustify-module-correlation = { workspace = true }
 trustify-module-fundamental = { workspace = true }
 trustify-module-importer = { workspace = true }
 trustify-module-ingestor = { workspace = true }
+trustify-module-notification = { workspace = true }
 trustify-module-storage = { workspace = true }
 trustify-module-ui = { workspace = true }
 trustify-module-user = { workspace = true }
diff --git a/server/src/openapi.rs b/server/src/openapi.rs
index 4e8bf4348..c20aa4adc 100644
--- a/server/src/openapi.rs
+++ b/server/src/openapi.rs
@@ -1,16 +1,21 @@
 use crate::profile::api::{Config, ModuleConfig, configure, default_openapi_info};
 use actix_web::App;
-use trustify_common::db::{self, pagination_cache::PaginationCache};
+use std::time::Duration;
+use trustify_common::db::{self, change::ChangeBroadcaster, pagination_cache::PaginationCache};
 use trustify_module_analysis::{config::AnalysisConfig, service::AnalysisService};
+use trustify_module_correlation::{config::CorrelationConfig, service::CorrelationService};
 use trustify_module_storage::service::fs::FileSystemBackend;
 use utoipa_actix_web::AppExt;
 
 pub async fn create_openapi() -> anyhow::Result {
-    let (db, _) = trustify_db::embedded::create().await?;
+    let (db, _guard) = trustify_db::embedded::create().await?;
     let (storage, _temp) = FileSystemBackend::for_test().await?;
     let db_rw = db::ReadWrite::new(db.clone());
     let db_ro = db::ReadOnly::new(db.clone());
     let analysis = AnalysisService::new(AnalysisConfig::default(), db_ro.clone());
+    let correlation =
+        CorrelationService::new(&CorrelationConfig::default(), db_ro.clone(), &db_rw).await?;
+    let broadcaster = ChangeBroadcaster::new(&db_rw, Duration::from_secs(86400))?;
 
     let (_, mut openapi) = App::new()
         .into_utoipa_app()
@@ -25,6 +30,8 @@ pub async fn create_openapi() -> anyhow::Result {
                     storage: storage.into(),
                     auth: None,
                     analysis,
+                    correlation,
+                    broadcaster,
                     read_only: false,
                 },
             );
diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs
index 7758159c9..57a6b0aa8 100644
--- a/server/src/profile/api.rs
+++ b/server/src/profile/api.rs
@@ -17,6 +17,7 @@ use trustify_common::{
     config::{Database, DatabaseReadOnly},
     db::{
         self,
+        change::ChangeBroadcaster,
         pagination_cache::{PaginationCache, PaginationConfig},
     },
     middleware::ReadOnlyState,
@@ -32,6 +33,8 @@ use trustify_infrastructure::{
     otel::{Metrics as OtelMetrics, Tracing},
 };
 use trustify_module_analysis::{config::AnalysisConfig, service::AnalysisService};
+use trustify_module_correlation::{config::CorrelationConfig, service::CorrelationService};
+use trustify_module_notification::config::NotificationConfig;
 use trustify_module_ingestor::graph::Graph;
 use trustify_module_storage::{config::StorageConfig, service::dispatch::DispatchBackend};
 use trustify_module_ui::{UI, endpoints::UiResources};
@@ -98,6 +101,14 @@ pub struct Run {
     #[command(flatten)]
     pub analysis: AnalysisConfig,
 
+    /// Correlation configuration
+    #[command(flatten)]
+    pub correlation: CorrelationConfig,
+
+    /// Notification configuration
+    #[command(flatten)]
+    pub notification: NotificationConfig,
+
     /// Database configuration
     #[command(flatten)]
     pub database: Database,
@@ -193,6 +204,8 @@ struct InitData {
     ui: UI,
     config: ModuleConfig,
     analysis: AnalysisService,
+    correlation: CorrelationService,
+    broadcaster: ChangeBroadcaster,
     read_only: bool,
 }
 
@@ -298,8 +311,13 @@ impl InitData {
             },
         };
 
+        let correlation = CorrelationService::new(&run.correlation, db_ro.clone(), &db_rw).await?;
+        let broadcaster = ChangeBroadcaster::new(&db_rw, *run.notification.change_log_retention)?;
+
         Ok(InitData {
             analysis: AnalysisService::new(run.analysis, db_ro.clone()),
+            correlation,
+            broadcaster,
             authenticator,
             authorizer,
             db_rw,
@@ -340,6 +358,8 @@ impl InitData {
                             storage: self.storage.clone(),
                             auth: self.authenticator.clone(),
                             analysis: self.analysis.clone(),
+                            correlation: self.correlation.clone(),
+                            broadcaster: self.broadcaster.clone(),
                             read_only: self.read_only,
                         },
                     );
@@ -389,6 +409,8 @@ pub(crate) struct Config {
     pub(crate) cache: PaginationCache,
     pub(crate) storage: DispatchBackend,
     pub(crate) analysis: AnalysisService,
+    pub(crate) correlation: CorrelationService,
+    pub(crate) broadcaster: ChangeBroadcaster,
     pub(crate) auth: Option>,
     pub(crate) read_only: bool,
 }
@@ -407,6 +429,8 @@ pub(crate) fn configure(svc: &mut utoipa_actix_web::service_config::ServiceConfi
         storage,
         auth,
         analysis,
+        correlation,
+        broadcaster,
         read_only,
     } = config;
 
@@ -417,8 +441,11 @@ pub(crate) fn configure(svc: &mut utoipa_actix_web::service_config::ServiceConfi
     svc.app_data(web::PayloadConfig::default().limit(limit));
     svc.app_data(graph);
 
+    // Outside the `/api` scope: browser WebSocket clients cannot set HTTP headers,
+    // so this endpoint handles auth via `?token=` query parameter injection.
     svc.configure(|svc| {
         endpoints::configure(svc, auth.clone(), read_only);
+        trustify_module_notification::endpoints::configure(svc, broadcaster, auth.clone());
     });
 
     svc.service(
@@ -440,9 +467,15 @@ pub(crate) fn configure(svc: &mut utoipa_actix_web::service_config::ServiceConfi
                     db_ro.clone(),
                     storage,
                     analysis.clone(),
-                    cache,
+                    cache.clone(),
                 );
                 trustify_module_analysis::endpoints::configure(svc, db_ro.clone(), analysis);
+                trustify_module_correlation::endpoints::configure(
+                    svc,
+                    db_ro.clone(),
+                    correlation,
+                    cache,
+                );
                 trustify_module_user::endpoints::configure(svc);
                 trustify_module_ui::endpoints::configure(svc, ui)
             }),
@@ -475,9 +508,10 @@ mod test {
     };
     use clap::{Args, Command, FromArgMatches};
     use rstest::rstest;
-    use std::sync::Arc;
+    use std::{sync::Arc, time::Duration};
     use test_context::test_context;
     use test_log::test;
+    use trustify_common::db::change::ChangeBroadcaster;
     use trustify_infrastructure::app::http::ApplyOpenApi;
     use trustify_module_ui::{UI, endpoints::UiResources};
     use trustify_test_context::{TrustifyContext, app::TestApp, call, call::CallService};
@@ -503,8 +537,13 @@ mod test {
     #[test(actix_web::test)]
     async fn routing(ctx: TrustifyContext) -> Result<(), anyhow::Error> {
         let ui = Arc::new(UiResources::new(&UI::default())?);
-        let analysis =
-            AnalysisService::new(AnalysisConfig::default(), db::ReadOnly::new(ctx.db.clone()));
+        let db_ro = db::ReadOnly::new(ctx.db.clone());
+        let db_rw = db::ReadWrite::new(ctx.db.clone());
+        let analysis = AnalysisService::new(AnalysisConfig::default(), db_ro.clone());
+        let correlation =
+            CorrelationService::new(&CorrelationConfig::default(), db_ro.clone(), &db_rw).await?;
+        let broadcaster =
+            ChangeBroadcaster::new(&db_rw, Duration::from_secs(86400))?;
         let app = actix_web::test::init_service(
             App::new()
                 .into_utoipa_app()
@@ -520,6 +559,8 @@ mod test {
                             storage: ctx.storage.clone().into(),
                             auth: None,
                             analysis,
+                            correlation,
+                            broadcaster,
                             read_only: false,
                         },
                     );
@@ -580,8 +621,15 @@ mod test {
 
     /// Creates a fully configured test app with all server endpoints and standard middleware.
     async fn caller(ctx: &TrustifyContext, read_only: bool) -> impl CallService {
-        let analysis =
-            AnalysisService::new(AnalysisConfig::default(), db::ReadOnly::new(ctx.db.clone()));
+        let db_ro = db::ReadOnly::new(ctx.db.clone());
+        let db_rw = db::ReadWrite::new(ctx.db.clone());
+        let analysis = AnalysisService::new(AnalysisConfig::default(), db_ro.clone());
+        let correlation =
+            CorrelationService::new(&CorrelationConfig::default(), db_ro.clone(), &db_rw)
+                .await
+                .expect("failed to create correlation service");
+        let broadcaster = ChangeBroadcaster::new(&db_rw, Duration::from_secs(86400))
+            .expect("failed to create change broadcaster");
         call::caller_app(move |svc| {
             configure(
                 svc,
@@ -593,6 +641,8 @@ mod test {
                     cache: PaginationCache::for_test(),
                     auth: None,
                     analysis,
+                    correlation,
+                    broadcaster,
                     read_only,
                 },
             );
diff --git a/xtask/src/openapi.rs b/xtask/src/openapi.rs
index 4b31b48d9..1cedffc3d 100644
--- a/xtask/src/openapi.rs
+++ b/xtask/src/openapi.rs
@@ -57,7 +57,7 @@ pub async fn generate_openapi(base: Option<&Path>) -> anyhow::Result<()> {
 
     // write
 
-    println!("Writing openapi to {:?}", &path);
+    println!("Writing openapi to {:?}", path);
 
     fs::write(path, doc).context("Failed to write openapi spec")?;