Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 38 additions & 7 deletions crates/sp42-citation/src/citation/body_classifier.rs
Original file line number Diff line number Diff line change
Expand Up @@ -171,11 +171,18 @@ const PAYWALL_VENDOR_HOSTS: &[&str] = &[
"wallkit.net",
];

/// Hosts whose pages are viewer/embed shells under generic extraction (no readable
/// article body). Matched as a case-insensitive substring of the URL host. Extension point: add hosts here.
const SPECIAL_CASE_HOSTS: &[&str] = &["books.google."];

/// `true` if the URL's host matches a special-case viewer-shell host.
/// Domains whose pages are viewer/embed shells under generic extraction (no
/// readable article body), as label sequences (excluding the TLD). Matched by
/// domain-suffix against the host, ignoring the final TLD label — so
/// `["books", "google"]` matches `books.google.com` and
/// `www.books.google.de` but NOT `books.google.com.evil.example` or
/// `xbooks.google.com`. Extension point: add domains here.
const SPECIAL_CASE_DOMAINS: &[&[&str]] = &[&["books", "google"]];

/// `true` if the URL's host is (a subdomain of) a special-case viewer-shell
/// domain, compared by DNS label boundaries rather than raw substring — a
/// substring match would misclassify any host merely embedding the text
/// (e.g. `books.google.com.attacker.example`).
fn is_special_case_host(source_url: &str) -> bool {
let Ok(parsed) = Url::parse(source_url) else {
return false;
Expand All @@ -184,9 +191,15 @@ fn is_special_case_host(source_url: &str) -> bool {
return false;
};
let host = host.to_ascii_lowercase();
SPECIAL_CASE_HOSTS
let labels: Vec<&str> = host.split('.').collect();
// Drop the final TLD label; a special-case domain must sit immediately
// before it (as the host itself or a subdomain suffix).
let Some((_tld, without_tld)) = labels.split_last() else {
return false;
};
SPECIAL_CASE_DOMAINS
.iter()
.any(|needle| host.contains(needle))
.any(|domain| without_tld.ends_with(domain))
Comment on lines +197 to +202

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve matches for multi-label Google TLDs

Because this drops exactly one label before matching, Google Books country hosts with multi-label public suffixes such as https://books.google.co.uk/... leave without_tld as ['books', 'google', 'co'], which does not end with ['books', 'google']. These hosts were covered by the previous books.google. rule and by the intended books.google.* viewer-shell class, so they now fall through to normal body classification instead of returning ViewerShell.

Useful? React with 👍 / 👎.

}

/// The first `n` characters of `text` (a char-boundary-safe prefix window).
Expand Down Expand Up @@ -524,13 +537,31 @@ mod tests {
for url in [
"https://books.google.com/books?id=abc123",
"https://books.google.es/books?id=xyz",
// Subdomains of the viewer-shell domain still match.
"https://www.books.google.de/books?id=q",
] {
let r = classify_source_usability(url, "text/html", None, Some("irrelevant body text"));
assert!(!r.usable, "{url}");
assert_eq!(r.reason, BodyUsabilityReason::ViewerShell, "{url}");
}
}

#[test]
fn host_embedding_the_special_case_text_is_not_matched() {
// A domain-suffix match, not a raw substring: an attacker-controlled
// host that merely embeds "books.google." must NOT be misclassified
// as the Google Books viewer shell.
let prose = "The history of the bridge spans more than a century. ".repeat(10);
for url in [
"https://books.google.com.attacker.example/x",
"https://xbooks.google.com/x",
"https://notbooks.google.com/x",
] {
let r = classify_source_usability(url, "text/html", Some(&prose), Some(&prose));
assert!(r.usable, "{url} must not be flagged as a viewer shell");
}
}

#[test]
fn non_special_host_is_not_viewer_shell() {
let prose = "The history of the bridge spans more than a century. ".repeat(10);
Expand Down
Loading