Skip to content

Escape CSS idents #358

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kepano wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Escape CSS idents #358

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9f11c9b
Escape ids when building CSS selectors fix #352
kepano Aug 3, 2026
5756353
Stop whole-<body> fallbacks from winning retry comparisons
kepano Aug 3, 2026
a7911c2
Compare footnote ids in JS instead of interpolating into a selector
kepano Aug 3, 2026
6fdb351
Add unit tests for escapeCssIdent
kepano Aug 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 92 additions & 34 deletions src/defuddle.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { removeBySelector } from './removals/selectors';
import { removeByContentPattern, removeEyebrowLabel } from './removals/content-patterns';
import { removeMetadataBlock } from './removals/metadata-block';
import { getComputedStyle, textPreview, countWords } from './utils';
import { parseHTML, serializeHTML, decodeHTMLEntities, isDangerousUrl, getClassName } from './utils/dom';
import { parseHTML, serializeHTML, decodeHTMLEntities, isDangerousUrl, getClassName, escapeCssIdent } from './utils/dom';

interface StyleChange {
selector: string;
Expand All @@ -29,10 +29,11 @@ interface StyleChange {
/** Keys from extractor variables that map to top-level DefuddleResponse fields */
const STANDARD_VARIABLE_KEYS = new Set(['title', 'author', 'published', 'site', 'description', 'image', 'language']);

// CSS-special characters that make class names invalid in selectors (Tailwind utilities like sm:pt-[131px])
// Classes carrying CSS syntax (Tailwind utilities like sm:pt-[131px]) are dropped
// from generated selectors rather than escaped. Unlike ids, this is a deliberate
// trade: escaping them would change which elements long-standing selectors match.
const UNSAFE_CSS_CLASS_RE = /[:\[\]()#>~+,]/;


export class Defuddle {
// Reassigned briefly during the schema.org fallback so re-extraction runs
// against a sanitized clone instead of the caller's live document.
Expand All @@ -47,6 +48,10 @@ export class Defuddle {
private _smallImages: Set<string> | undefined;
private _inExtractorPipelineRun = false;

// Results produced by the whole-<body> fallback rather than by real extraction.
// Tracked out-of-band so the marker never reaches callers via DefuddleResponse.
private _degradedResults = new WeakSet<DefuddleResponse>();

/**
* Create a new Defuddle instance
* @param doc - The document to parse
Expand Down Expand Up @@ -85,7 +90,7 @@ export class Defuddle {
let result = this.parseInternal();

// If result has very little content, try again without clutter removal
if (result.wordCount < 200) {
if (this._effectiveWordCount(result) < 200) {
this._log('Initial parse returned very little content, trying again');
const retryResult = this.parseInternal({
removePartialSelectors: false
Expand All @@ -95,7 +100,7 @@ export class Defuddle {
// A small increase likely means partial selectors correctly removed
// clutter (author blocks, related articles, etc.) from a short article.
// A large increase (2x+) suggests partial selectors were too aggressive.
if (retryResult.wordCount > result.wordCount * 2) {
if (this._preferCandidate(result, retryResult, () => retryResult.wordCount > result.wordCount * 2)) {
this._log('Retry produced more content');
result = retryResult;
}
Expand All @@ -104,12 +109,12 @@ export class Defuddle {
// If still very little content, the page may be an index/listing page
// or a page that reveals content at runtime from a hidden wrapper.
// Retry once with hidden-element removal disabled.
if (result.wordCount < 50) {
if (this._effectiveWordCount(result) < 50) {
this._log('Still very little content, retrying without hidden-element removal');
const hiddenRetry = this.parseInternal({
removeHiddenElements: false
});
if (hiddenRetry.wordCount > result.wordCount * 2) {
if (this._preferCandidate(result, hiddenRetry, () => hiddenRetry.wordCount > result.wordCount * 2)) {
this._log('Hidden-element retry produced more content');
result = hiddenRetry;
}
Expand All @@ -124,13 +129,13 @@ export class Defuddle {
removePartialSelectors: false,
contentSelector: hiddenSelector
});
if (
if (this._preferCandidate(result, hiddenSelectorRetry, () =>
hiddenSelectorRetry.wordCount > result.wordCount ||
(
hiddenSelectorRetry.wordCount > Math.max(20, result.wordCount * 0.7) &&
hiddenSelectorRetry.content.length < result.content.length
)
) {
)) {
this._log('Hidden-selector retry produced better focused content');
result = hiddenSelectorRetry;
}
Expand All @@ -140,14 +145,14 @@ export class Defuddle {
// If still very little content, the page may be an index/listing page
// where card elements were scored as non-content or removed by partial
// selectors (e.g. "post-preview"). Retry with both disabled.
if (result.wordCount < 50) {
if (this._effectiveWordCount(result) < 50) {
this._log('Still very little content, retrying without scoring/partial selectors (possible index page)');
const indexRetry = this.parseInternal({
removeLowScoring: false,
removePartialSelectors: false,
removeContentPatterns: false
});
if (indexRetry.wordCount > result.wordCount) {
if (this._preferCandidate(result, indexRetry, () => indexRetry.wordCount > result.wordCount)) {
this._log('Index page retry produced more content');
result = indexRetry;
}
Expand All @@ -158,7 +163,7 @@ export class Defuddle {
// Use a 1.5x threshold to avoid triggering when the difference is small
// (e.g. just related-content link text removed).
const schemaText = this._getSchemaText(result.schemaOrgData);
if (schemaText && this.countHtmlWords(schemaText) > result.wordCount * 1.5) {
if (schemaText && this.countHtmlWords(schemaText) > this._effectiveWordCount(result) * 1.5) {
// Re-extract from a sanitized clone so dangerous elements and URI
// attributes (e.g. data:text/html in an img src) in the matched
// element are stripped, without mutating the caller's live document.
Expand All @@ -176,11 +181,18 @@ export class Defuddle {
const selector = this.getElementSelector(bestMatch);
this._log('Schema.org suggests a better content element, retrying with selector:', selector);
const schemaRetry = this.parseInternal({ contentSelector: selector });
result = schemaRetry;
// Trusted over the current result — schema.org named this element as
// the article body — but a degraded retry is still a whole-page dump
// and must not replace a real extraction.
if (this._preferCandidate(result, schemaRetry, () => true)) {
result = schemaRetry;
}
} else {
this._log('Using schema.org text as content (DOM element not found)');
result.content = schemaText;
result.wordCount = this.countHtmlWords(schemaText);
// No longer a whole-<body> dump: content now comes from schema.org.
this._degradedResults.delete(result);
}
} finally {
this.doc = liveDoc;
Expand Down Expand Up @@ -227,6 +239,53 @@ export class Defuddle {
* extraction pipeline already removes script/style/etc. via EXACT_SELECTORS,
* so only these raw-body paths need to sanitize here.
*/
/**
* Build the response for a path that failed to extract anything and is falling
* back to the whole <body>. Marked degraded: its word count reflects the entire
* page (nav, footer, sidebars) and would otherwise beat every real extraction
* in parse()'s retry comparisons, and suppress the gates that trigger them.
*/
private _degradedResponse(startTime: number): DefuddleResponse {
const content = this._serializeFallbackBody();
const response: DefuddleResponse = {
content,
...this._metadata,
wordCount: this.countHtmlWords(content),
parseTime: Math.round(Date.now() - startTime),
metaTags: this._metaTags
};
this._degradedResults.add(response);
return response;
}

/** True if the result came from the whole-<body> fallback, not real extraction. */
private _isDegraded(result: DefuddleResponse): boolean {
return this._degradedResults.has(result);
}

/**
* Word count for retry-gating purposes. A degraded result counts as zero so its
* inflated whole-page count can't suppress the retries that would find content.
*/
private _effectiveWordCount(result: DefuddleResponse): number {
return this._isDegraded(result) ? 0 : result.wordCount;
}

/**
* Whether a retry candidate should replace the current result. Degraded results
* never win; any real extraction beats a degraded current. Otherwise defer to
* the caller's predicate, which differs per retry.
*/
private _preferCandidate(
current: DefuddleResponse,
candidate: DefuddleResponse,
isBetter: () => boolean
): boolean {
if (this._isDegraded(candidate)) return false;
if (this._isDegraded(current)) return true;
return isBetter();
}

private _serializeFallbackBody(): string {
if (!this.doc.body) return '';
const safeBody = this.doc.body.cloneNode(true) as HTMLElement;
Expand Down Expand Up @@ -832,7 +891,7 @@ export class Defuddle {
removeHiddenElements: false,
});
const variables = this.getExtractorVariables(extracted.variables);
return {
const merged: DefuddleResponse = {
...pipelineResult,
title: extracted.variables?.title || pipelineResult.title,
description: extracted.variables?.description || pipelineResult.description,
Expand All @@ -843,6 +902,11 @@ export class Defuddle {
extractorType: extractor.constructor.name.replace('Extractor', '').toLowerCase(),
...(variables ? { variables } : {}),
};
// Spreading creates a new identity, so carry the marker across.
if (this._isDegraded(pipelineResult)) {
this._degradedResults.add(merged);
}
return merged;
} finally {
this._inExtractorPipelineRun = false;
}
Expand Down Expand Up @@ -887,7 +951,13 @@ export class Defuddle {
const mainContent = profileStep('findMainContent', (): Element | null => {
let found: Element | null = null;
if (options.contentSelector) {
found = clone.querySelector(options.contentSelector);
// contentSelector is public API input and may be unparseable.
// Degrade to auto-detection rather than failing the whole parse.
try {
found = clone.querySelector(options.contentSelector);
} catch (e) {
this._log('Invalid contentSelector, falling back to auto-detection:', options.contentSelector, e);
}
this._log('Using contentSelector:', options.contentSelector, found ? 'found' : 'not found');
}
if (!found) {
Expand Down Expand Up @@ -919,15 +989,7 @@ export class Defuddle {
});

if (!mainContent) {
const fallbackContent = this._serializeFallbackBody();
const endTime = Date.now();
return {
content: fallbackContent,
...metadata,
wordCount: this.countHtmlWords(fallbackContent),
parseTime: Math.round(endTime - startTime),
metaTags: pageMetaTags
};
return this._degradedResponse(startTime);
}

// Remove h1-adjacent date/author metadata blocks from the content.
Expand Down Expand Up @@ -1065,15 +1127,7 @@ export class Defuddle {
return result;
} catch (error) {
console.error('Defuddle', 'Error processing document:', error);
const errorContent = this._serializeFallbackBody();
const endTime = Date.now();
return {
content: errorContent,
...metadata,
wordCount: this.countHtmlWords(errorContent),
parseTime: Math.round(endTime - startTime),
metaTags: pageMetaTags
};
return this._degradedResponse(startTime);
}
}

Expand Down Expand Up @@ -1338,8 +1392,12 @@ export class Defuddle {
while (current && current !== this.doc.documentElement) {
let selector = current.tagName.toLowerCase();
if (current.id) {
selector += '#' + current.id;
// Escaping an id is lossless — it still matches the same one element.
selector += '#' + escapeCssIdent(current.id);
} else if (getClassName(current)) {
// Dropping a class is lossy by comparison: the selector widens, and if
// every class is unsafe this degrades to a bare tag name. See the note
// on UNSAFE_CSS_CLASS_RE for why that trade is kept.
const safe = getClassName(current).trim().split(/\s+/)
.filter(cls => !UNSAFE_CSS_CLASS_RE.test(cls));
if (safe.length) {
Expand Down
8 changes: 6 additions & 2 deletions src/elements/footnotes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,8 +402,12 @@ class FootnoteHandler {
const contentDiv = element.ownerDocument.createElement('div');
const clone = el.cloneNode(true);

// Remove empty/numeric ID anchors (e.g. <a id="r1"></a> or <a id="r1">1.</a>)
const idAnchor = clone.querySelector(`a[id="${id}"]`);
// Remove empty/numeric ID anchors (e.g. <a id="r1"></a> or <a id="r1">1.</a>).
// The id comes from the page, so it is compared in JS rather than
// interpolated into a selector — a quote or backslash in it would make the
// selector unparseable and throw. Matches the a[name] handling just below.
const idAnchor = Array.from(clone.querySelectorAll('a[id]'))
.find((a: any) => a.getAttribute('id') === id) as any;
if (idAnchor && (!idAnchor.textContent?.trim() || /^\d+[.)]*\s*$/.test(idAnchor.textContent.trim()))) {
idAnchor.remove();
}
Expand Down
34 changes: 34 additions & 0 deletions src/utils/dom.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,40 @@ export function escapeHtml(text: string): string {
.replace(/"/g, '&quot;');
}

// Characters valid in a CSS identifier as-is: letters, digits, `_`, `-`, non-ASCII
const CSS_IDENT_SAFE_RE = /[\w\u0080-\uffff-]/;

/**
* Escape a string for use as a CSS identifier, so values taken from a page
* can't turn a generated selector into invalid CSS. React streaming SSR is the
* common trigger: it emits ids like `S:a` and `B:0`, and an unescaped `#S:a`
* parses as a pseudo-class.
*
* Deliberately not delegating to `CSS.escape` where it exists: linkedom has no
* `CSS` global, so the Node/CLI/Worker environments need this anyway, and
* always using it keeps selectors identical across every environment.
* NUL is the one case this drops versus `CSS.escape` — the HTML tokenizer
* replaces it with U+FFFD before it can reach an attribute value.
*/
export function escapeCssIdent(value: string): string {
let result = '';
for (let i = 0; i < value.length; i++) {
const ch = value[i];
// Control characters, and a digit leading the identifier (or following a
// leading hyphen), have no inline `\x` form and need the hex escape.
const leadingDigit = ch >= '0' && ch <= '9'
&& (i === 0 || (i === 1 && value[0] === '-'));
if (leadingDigit || ch <= '\x1f' || ch === '\x7f') {
result += '\\' + ch.charCodeAt(0).toString(16) + ' ';
} else if (ch === '-' && value.length === 1) {
result += '\\-';
} else {
result += CSS_IDENT_SAFE_RE.test(ch) ? ch : '\\' + ch;
}
}
return result;
}

/**
* Safely get an element's class name as a string.
* Handles SVG elements where className is an SVGAnimatedString.
Expand Down
Loading
Loading