diff --git a/.changes/unreleased/Changed-20261107-092240.165899.yaml b/.changes/unreleased/Changed-20261107-092240.165899.yaml new file mode 100644 index 000000000..859c42970 --- /dev/null +++ b/.changes/unreleased/Changed-20261107-092240.165899.yaml @@ -0,0 +1,3 @@ +kind: Changed +body: 'The content blocker never accepts cookie consent for you: filter-list rules that would click an accept button or store an accepted answer are dropped, and on sites using Didomi, OneTrust, Cookiebot or Usercentrics, consent is refused through the platform itself while cookie notices are hidden' +time: 2026-11-07T09:22:40.165899+01:00 diff --git a/Lilo/ContentBlocker/FilterListConverter.swift b/Lilo/ContentBlocker/FilterListConverter.swift index 69b1512b2..074a0e507 100644 --- a/Lilo/ContentBlocker/FilterListConverter.swift +++ b/Lilo/ContentBlocker/FilterListConverter.swift @@ -369,7 +369,124 @@ enum FilterListConverter { let canonical = scriptletAliases[name] ?? name guard supportedScriptlets.contains(canonical), trusted || !(name.hasPrefix("trusted-") || canonical.hasPrefix("trusted-") || canonical == "replace-node-text") else { return nil } - return (canonical, Array(parts.dropFirst())) + let args = Array(parts.dropFirst()) + guard !acceptsConsent(canonical, args) else { return nil } + return (canonical, args) + } + + /// Stems of the words naming an accept, agree or allow answer, in the languages of the cookie lists; matched inside a word. + private static let acceptStems = [ + "accept", "agree", "allow", "approv", "grant", "optin", "akzept", "zustimm", "einverstanden", "einwillig", "accett", "acept", + "aceit", "concord", "akcept", "zgadzam", "godkänn", "godta", "hyväksy", "elfogad", "souhlas", "súhlas", "prija", "přij", + "prihvat", "sutinku", "piekrīt", "nõustu", "kabul", "принять", "соглас" + ] + /// Stems of the words naming a refusal, a necessary-only or partial answer, or a choice left to the user; matched as a word + /// prefix, as "not" would match "notice" anywhere in a word. They win over an accept stem: "accept only necessary" refuses. + private static let refuseStems = [ + "reject", "refus", "declin", "deny", "denied", "disagree", "disallow", "necessar", "nécessaire", "essential", "required", + "mandatory", "functional", "technical", "minimal", "without", "optout", "revoke", "withdraw", "select", "false", "ablehn", + "rifiut", "rechaz", "odrzu", "weiger", "avvis", "afvis", "hylkää", "notwendig", "nötig", "nødvendig", "nödvänd", "noodzak", + "nevyhnut", "niezbęd", "välttämät", "szükséges", "nezbytn", "valit", "seulement", "apenas", "somente", "alleen", "endast", + "tylko", "pouze", "senza", "zonder", "utan", "uden", "uten" + ] + private static let refuseWords: Set = ["only", "none", "not", "dont", "sans", "ohne", "nur", "kun", "vain", "solo", "sin", "sem", "bez"] + + /// Whether a scriptlet answers a consent prompt with accept on the user's behalf, which loads what the site holds back for + /// consent (ads, trackers): a click whose target names an accept answer, or a stored answer that does or cannot be read. Refusing, necessary-only, + /// continue-without and close clicks, and stored refusals, are kept. + static func acceptsConsent(_ name: String, _ args: [String]) -> Bool { + switch name { + case "trusted-click-element": + // A sequence accepts when any of its clicks does. + return args.first.map { splitTopLevel($0, at: ",").contains(where: clickAccepts) } ?? false + case "set-cookie", "trusted-set-cookie", "set-cookie-reload", "trusted-set-cookie-reload", "set-local-storage-item", + "trusted-set-local-storage-item", "set-session-storage-item", "trusted-set-session-storage-item": + guard args.count > 1 else { return false } + let key = consentWords(args[0]), value = args[1].lowercased(), valueWords = consentWords(args[1]) + if refuses(key + valueWords) || ["0", "no", "n", "off"].contains(value) || value.contains("[]") { return false } + // A saved consent record (an id or an encoded TCF string) may hold an acceptance nothing here can read. + let isSavedConsent = key.contains { $0.contains("consent") } && value.count >= 16 && value.contains(where: \.isNumber) + && value.allSatisfy { $0.isASCII && ($0.isLetter || $0.isNumber || "-_.".contains($0)) } + return isSavedConsent || accepts(valueWords) || (["true", "1", "yes", "y", "ok", "on", "all"].contains(value) && accepts(key)) + default: + return false + } + } + + /// Lowercased words, split at every non-letter and camelCase boundary. A long one (an opaque token) names nothing. + private static func consentWords(_ text: String) -> [String] { + var words: [String] = [], word = "", previousLower = false + for c in text + " " { + guard c.isLetter else { + if !word.isEmpty, word.count <= 40 { words.append(word) } + word = "" + previousLower = false + continue + } + if c.isUppercase, previousLower { + words.append(word) + word = "" + } + word += c.lowercased() + previousLower = c.isLowercase + } + return words + } + + private static func accepts(_ words: [String]) -> Bool { words.contains { word in acceptStems.contains { word.contains($0) } } } + + private static func refuses(_ words: [String]) -> Bool { + words.contains { word in refuseWords.contains(word) || refuseStems.contains { word.hasPrefix($0) } } + } + + /// Whether a click selector targets an accept control: the element it selects (its last compound) names no refusal, and the + /// selector names an accept answer, or excludes the refusing control with `:not()`. + private static func clickAccepts(_ selector: String) -> Bool { + var kept = "", negated = "", depth = 0, index = selector.startIndex + while index < selector.endIndex { + if depth == 0, selector[index...].hasPrefix(":not(") { + depth = 1 + index = selector.index(index, offsetBy: 5) + continue + } + let c = selector[index] + if depth > 0 { + depth += c == "(" ? 1 : c == ")" ? -1 : 0 + if depth > 0 { negated.append(c) } else { negated.append(" ") } + } else { + kept.append(c) + } + index = selector.index(after: index) + } + let target = splitTopLevel(kept, at: " >+~").last { !$0.isEmpty } ?? "" + if refuses(consentWords(target)) { return false } + let words = consentWords(kept) + // Sourcepoint's accept-all choice, and Quantcast's primary (agree) button. + return accepts(words) || words.contains { $0.contains("yes") } || refuses(consentWords(negated)) + || kept.contains("sp_choice_type_11") || kept.contains("mode=\"primary\"") + } + + /// `text` split at `separators` outside brackets, parentheses and quotes. + private static func splitTopLevel(_ text: String, at separators: String) -> [String] { + var parts: [String] = [], part = "", depth = 0, quote: Character? + for c in text { + if let open = quote { + if c == open { quote = nil } + } else if c == "\"" || c == "'" { + quote = c + } else if c == "[" || c == "(" { + depth += 1 + } else if c == "]" || c == ")" { + depth -= 1 + } else if depth == 0, separators.contains(c) { + parts.append(part) + part = "" + continue + } + part.append(c) + } + parts.append(part) + return parts } /// Splits generic cosmetic selectors: those keyed by a class, id or link host go to GenericCosmetics.js tables (native rules delay diff --git a/Lilo/ContentBlocker/FilterListService.swift b/Lilo/ContentBlocker/FilterListService.swift index aee33b8f9..406bf0e58 100644 --- a/Lilo/ContentBlocker/FilterListService.swift +++ b/Lilo/ContentBlocker/FilterListService.swift @@ -119,7 +119,7 @@ final class FilterListService { private static let partialRefreshAcceptAge: TimeInterval = 3 * refreshInterval /// Bumped whenever the conversion changes: a cache in another format is reconverted at launch /// instead of serving the old conversion until the next scheduled refresh. - private nonisolated static let cacheFormat = "10" + private nonisolated static let cacheFormat = "11" /// Names the conversion format and the lists it was made from: a cache stamped otherwise is redone. nonisolated static func cacheStamp(for sources: [URL]) -> String { @@ -349,6 +349,7 @@ final class FilterListService { let siteCap = Self.maxSiteCosmetics let scriptletCap = Self.maxScriptletEntries let proceduralCap = Self.maxProceduralSelectors + let refusesConsent = listSelection().cookieNotices let result = await Task.detached(priority: .utility) { [texts] () -> Conversion? in // One text, in source order: a `$badfilter` in one list must reach the rule it disables in another. let text = texts.joined(separator: "\n") @@ -384,6 +385,8 @@ final class FilterListService { ) let sitesJSON = (try? JSONSerialization.data(withJSONObject: siteTable, options: .sortedKeys)) .flatMap { String(data: $0, encoding: .utf8) } ?? "" + // With cookie notices hidden, the consent platform a page uses is answered with a refusal. First, so no cap drops it. + if refusesConsent { out.scriptlets.insert(.init(domains: ["*"], name: "refuse-consent", args: []), at: 0) } let table = FilterListConverter.scriptletTable(from: out.scriptlets, exceptions: out.scriptletExceptions, cap: scriptletCap) let scriptletsJSON = (try? JSONSerialization.data(withJSONObject: table, options: .sortedKeys)) .flatMap { String(data: $0, encoding: .utf8) } ?? "" diff --git a/Lilo/ContentBlocker/Scriptlets.js b/Lilo/ContentBlocker/Scriptlets.js index 61b49cee0..7fd0478c7 100644 --- a/Lilo/ContentBlocker/Scriptlets.js +++ b/Lilo/ContentBlocker/Scriptlets.js @@ -7,7 +7,7 @@ var nativeJSONStringify = JSON.stringify.bind(JSON); // Wrappers call these, not the page's Reflect, WeakMap and Promise methods, which a hook could replace to spot them. var reflectApply = Reflect.apply, weakGet = WeakMap.prototype.get, weakSet = WeakMap.prototype.set; - var promiseThen = Promise.prototype.then; + var promiseThen = Promise.prototype.then, functionBind = Function.prototype.bind; // Random per document: what a trap throws and the Trusted Types policy it creates carry no fixed name. var token = Math.random().toString(36).slice(2); @@ -1381,6 +1381,50 @@ } setTimeout(poll, Math.max(next - now, 100)); })(); + }, + // Refuses consent through the documented API of the consent platform the page loads, once it is ready and only while the + // visitor has not answered. Each platform is waited for on its own signal, so a page without one runs nothing more. + 'refuse-consent': function () { + // The platforms keep their API in the top document. + if (ancestors && ancestors.length) return; + function attempt(fn) { try { fn(); } catch (e) {} } + // A promise the platform returns goes through the native then, a plain value straight to fn. + function settle(value, fn) { + if (value && typeof value.then === 'function') reflectApply(promiseThen, value, [fn]); else fn(value); + } + // OneTrust dispatches it each time its script loads, and after each consent change. + self.addEventListener('OneTrustGroupsUpdated', function () { + attempt(function () { var ot = self.OneTrust; if (!ot.IsAlertBoxClosed()) ot.RejectAll(); }); + }); + self.addEventListener('CookiebotOnDialogDisplay', function () { + attempt(function () { + var cb = self.Cookiebot; + if (cb.hasResponse) return; + cb.submitCustomConsent(false, false, false); + cb.hide(); + }); + }); + // Usercentrics v3 registers __ucCmp, v2 UC_UI; both dispatch it once ready. + self.addEventListener('UC_UI_INITIALIZED', function () { + attempt(function () { + var v3 = self.__ucCmp, uc = v3 || self.UC_UI; + settle(uc.isConsentRequired(), function (required) { + if (required) settle(uc.denyAllConsents(), function () { attempt(function () { if (v3) uc.closeCmp(); else uc.closeCMP(); }); }); + }); + }); + }); + // Didomi has no event: its ready queue is joined once its script has loaded. Bound, the callback prints as native code. + var didomiAnswer = reflectApply(functionBind, function (didomi) { + attempt(function () { if (didomi.shouldUserStatusBeCollected()) didomi.setUserDisagreeToAll(); }); + }, [null]); + // Joined once even where the listener outlives its removal (a page hooking removeEventListener). + var joined = false; + document.addEventListener('load', function onLoad() { + if (joined || !self.Didomi) return; + joined = true; + document.removeEventListener('load', onLoad, true); + attempt(function () { (self.didomiOnReady = self.didomiOnReady || []).push(didomiAnswer); }); + }, true); } }; @@ -1454,6 +1498,8 @@ hosts.push(parts.slice(h).join('.'), parts.slice(h, n - 1).join('.') + '.*'); if (twoLabelSuffix && h < n - 2) hosts.push(parts.slice(h, n - 2).join('.') + '.*'); } + // `*` lists what runs on every site. + hosts.push('*'); // An entry starting with `@` excepts the same entry on this host and its subdomains, `@` alone every scriptlet. A rule // naming both the host and its entity runs once. var entries = new Set(), excepted = new Set(); diff --git a/Lilo/Extensions/bridge-src/src/scriptlets.test.ts b/Lilo/Extensions/bridge-src/src/scriptlets.test.ts index 519fa50e3..c67257f70 100644 --- a/Lilo/Extensions/bridge-src/src/scriptlets.test.ts +++ b/Lilo/Extensions/bridge-src/src/scriptlets.test.ts @@ -1104,6 +1104,80 @@ describe("consent scriptlets", () => { }); }); +describe("refuse-consent", () => { + const HOST = "www.example.com"; + const g = globalThis as Record; + const table = { "*": [["refuse-consent", []]] }; + let win: EventTarget; + + function page(cmp: Record, ancestorOrigins?: Record, allowlist: string[] = []) { + win = new EventTarget(); + g.document = new EventTarget(); + return run(table, HOST, "", allowlist, ancestorOrigins, { addEventListener: win.addEventListener.bind(win), ...cmp }).self; + } + + afterEach(() => { + delete g.document; + }); + + test("OneTrust rejects all once its script reports, unless the visitor has answered", () => { + const calls: string[] = []; + let closed = false; + page({ OneTrust: { IsAlertBoxClosed: () => closed, RejectAll: () => { calls.push("reject"); closed = true; } } }); + win.dispatchEvent(new Event("OneTrustGroupsUpdated")); + win.dispatchEvent(new Event("OneTrustGroupsUpdated")); + assert.deepEqual(calls, ["reject"]); + }); + + test("Cookiebot submits a refusal of every category and hides its dialog", () => { + const calls: unknown[] = []; + const self = page({ Cookiebot: { hasResponse: false, submitCustomConsent: (...a: boolean[]) => calls.push(a), hide: () => calls.push("hide") } }); + win.dispatchEvent(new Event("CookiebotOnDialogDisplay")); + (self.Cookiebot as { hasResponse: boolean }).hasResponse = true; + win.dispatchEvent(new Event("CookiebotOnDialogDisplay")); + assert.deepEqual(calls, [[false, false, false], "hide"]); + }); + + test("Usercentrics v3 and v2 deny all consents and close, only while consent is required", async () => { + const calls: string[] = []; + page({ __ucCmp: { isConsentRequired: async () => true, denyAllConsents: async () => calls.push("v3 deny"), closeCmp: async () => calls.push("v3 close") } }); + win.dispatchEvent(new Event("UC_UI_INITIALIZED")); + page({ UC_UI: { isConsentRequired: () => true, denyAllConsents: async () => calls.push("v2 deny"), closeCMP: () => calls.push("v2 close") } }); + win.dispatchEvent(new Event("UC_UI_INITIALIZED")); + page({ UC_UI: { isConsentRequired: () => false, denyAllConsents: async () => calls.push("answered deny") } }); + win.dispatchEvent(new Event("UC_UI_INITIALIZED")); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(calls.sort(), ["v2 close", "v2 deny", "v3 close", "v3 deny"]); + }); + + test("Didomi gets a callback in its ready queue once its script has loaded, printing as native code", () => { + const self = page({}); + const load = () => (g.document as EventTarget).dispatchEvent(new Event("load")); + load(); + assert.equal(self.didomiOnReady, undefined); + self.Didomi = {}; + load(); + load(); + const queue = self.didomiOnReady as unknown as ((d: unknown) => void)[]; + assert.equal(queue.length, 1); + assert.match(Function.prototype.toString.call(queue[0]), /\[native code\]/); + const calls: string[] = []; + queue[0]({ shouldUserStatusBeCollected: () => true, setUserDisagreeToAll: () => calls.push("disagree") }); + queue[0]({ shouldUserStatusBeCollected: () => false, setUserDisagreeToAll: () => calls.push("answered") }); + assert.deepEqual(calls, ["disagree"]); + }); + + test("nothing listens in a subframe or on an allowlisted site", () => { + const calls: string[] = []; + const OneTrust = { IsAlertBoxClosed: () => false, RejectAll: () => calls.push("reject") }; + page({ OneTrust }, { 0: "https://example.org", length: 1 }); + win.dispatchEvent(new Event("OneTrustGroupsUpdated")); + page({ OneTrust }, undefined, [HOST]); + win.dispatchEvent(new Event("OneTrustGroupsUpdated")); + assert.deepEqual(calls, []); + }); +}); + describe("anti-detection scriptlets", () => { test("bab-defuser skips BlockAdBlock's eval and its bait timer, nothing else", () => { const evaluated: string[] = []; diff --git a/LiloTests/ContentBlocker/BlockerLayersTests.swift b/LiloTests/ContentBlocker/BlockerLayersTests.swift index 1916333e9..227f77f6b 100644 --- a/LiloTests/ContentBlocker/BlockerLayersTests.swift +++ b/LiloTests/ContentBlocker/BlockerLayersTests.swift @@ -83,6 +83,45 @@ final class BlockerLayersTests: XCTestCase { XCTAssertTrue(paused.imageRequested) } + /// A page whose consent platform the lists answer: their accept click is dropped, their reject click kept, and the platform is + /// told to refuse, unless the user shows cookie notices. + func testConsentIsRefusedNeverAccepted() async throws { + let factory = WKEngineFactory() + let service = FilterListService(engineFactory: factory, allowlistProvider: { BlockerAllowlist() }) + let clicks = "example.com##+js(trusted-click-element, #accept-all)\nexample.com##+js(trusted-click-element, #reject-all)" + service.fetchList = { url in url.absoluteString.hasSuffix("/quick-fixes.txt") ? clicks : "" } + service.writeExtension = { _, _ in false } + // A made-up platform under OneTrust's documented API: what it was told lands in the title. + let page = """ + + + """ + func load() async -> String { + let tab = factory.createTab(configuration: .init(dataStore: factory.createDataStore(persistent: false, identifier: nil))) + defer { tab.teardown() } + tab.wkWebView.loadHTMLString(page, baseURL: URL(string: "https://example.com/")) + var title = "" + // The reject click lands a delay after the page; the accept one would have landed with it. + _ = await pollUntil(timeout: 5) { + title = (try? await tab.wkWebView.evaluateJavaScript("document.title") as? String) ?? "" + return title.contains("rejected") + } + try? await Task.sleep(for: .milliseconds(300)) + return (try? await tab.wkWebView.evaluateJavaScript("document.title") as? String) ?? title + } + + await service.refreshNow() + let refused = await load() + XCTAssertEqual(refused, "refused,rejected,") + service.listSelection = { FilterListService.ListSelection(cookieNotices: false) } + await service.refreshNow() + let shown = await load() + XCTAssertEqual(shown, "rejected,") + } + /// The built-in extension's declarativeNetRequest rules, written and loaded as the app does, apply in a private space and stand /// down for a paused site. func testTheBuiltInExtensionStandsDownForAPausedSite() async throws { diff --git a/LiloTests/ContentBlocker/FilterListConverterTests.swift b/LiloTests/ContentBlocker/FilterListConverterTests.swift index 74eb6c95e..8814dfc20 100644 --- a/LiloTests/ContentBlocker/FilterListConverterTests.swift +++ b/LiloTests/ContentBlocker/FilterListConverterTests.swift @@ -648,6 +648,37 @@ final class FilterListConverterTests: XCTestCase { XCTAssertTrue(isDropped("a.example##+js(this-scriptlet-does-not-exist, 1)")) } + func testConsentIsNeverAcceptedOnTheUsersBehalf() { + let accepting = [ + "#didomi-notice-agree-button", "button#onetrust-accept-btn-handler, , 1000", "button[onclick=\"Didomi.setUserAgreeToAll();\"]", + "button#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll", "button[title=\"Alle akzeptieren\" i]", + "#usercentrics-root >>> button[data-testid=\"uc-accept-all-button\"]", "button.sp_choice_type_11", + ".qc-cmp2-summary-buttons > button[mode=\"primary\"]", "div[data-project=\"cmp\"] button[class*=\"consent\"]:not([class*=\"reject\"])", + "'button.settings, button.save-all, a.cmptxt_btn_yes'", ".js-cookie-accept-all-button > button" + ].map { "a.example##+js(trusted-click-element, \($0))" } + [ + "set-cookie, cookiesAccepted, true", "set-local-storage-item, cookieConsent, granted", "trusted-set-cookie, consent, accept_all", + "set-cookie, cmplz_marketing, allow", "trusted-set-cookie, site_save_consent, iTTPgpSWqAGGcd3vV88zNDbHsABxE1hB", + "trusted-set-cookie, consentUUID, 4828f4be-a6c6-46d4-b46a-5a812a8c0f81_35" + ].map { "a.example##+js(\($0))" } + let kept = [ + "#didomi-notice-disagree-button", "button#onetrust-reject-all-handler", "button[aria-label=\"Continuer sans accepter\"]", + "button[aria-label=\"Accepter kun de nødvendige cookies\"]", "button#cm-acceptNone", "button.accept-necessary", + "#notice button.close", "[data-ga-action=\"disallow_all_cookies\"]", "input#accept_selected_cookies", "button.sp_choice_type_13", + "a.cookie__close_text", "#didomi-notice-agree-button:not(.x) ~ .didomi-continue-without-agreeing" + ].map { "a.example##+js(trusted-click-element, \($0))" } + [ + "set-cookie, cookiesAccepted, false", "set-cookie, cookie-consent, declined", "trusted-set-cookie, prefs, '{\"necessary\":true,\"ads\":false}'", + "trusted-set-cookie, cookie_preferences, '{\"allow\":[]}'", "set-cookie, cookieconsent_dismissed, yes", "set-cookie, notice_seen, 1", + "trusted-set-cookie, session_token, eyJhY2NlcHQiOnRydWV9OiJhY2NlcHRBbGwiLCJ2ZXJzaW9uIjoxfQ", + "trusted-set-cookie, cookie_consent, cookie-consent_rejected-2024" + ].map { "a.example##+js(\($0))" } + for rule in accepting { + XCTAssertTrue(FilterListConverter.convert("!#trusted on\n" + rule).scriptlets.isEmpty, rule) + } + for rule in kept { + XCTAssertEqual(FilterListConverter.convert("!#trusted on\n" + rule).scriptlets.count, 1, rule) + } + } + func testConvertPopulatesScriptlets() { let out = FilterListConverter.convert(""" youtube.com##+js(json-prune, playerResponse.adPlacements adPlacements) diff --git a/LiloTests/Engine/PageWorldTests.swift b/LiloTests/Engine/PageWorldTests.swift index 7a308d3b5..8db560c7d 100644 --- a/LiloTests/Engine/PageWorldTests.swift +++ b/LiloTests/Engine/PageWorldTests.swift @@ -129,7 +129,8 @@ final class PageWorldTests: XCTestCase { let factory = WKEngineFactory() // A site's cosmetics apply at once: their sheet must leave the page's DOM and stylesheets as they are. Its // scriptlets leave the functions they wrap printing as native code. - let scriptlets = #"{"example.com":[["set-constant",["history.adsBlocked","true"]],["no-setTimeout-if",["adProbe"]]]}"# + // The consent refusal every site runs leaves nothing either on a page without a consent platform. + let scriptlets = #"{"example.com":[["set-constant",["history.adsBlocked","true"]],["no-setTimeout-if",["adProbe"]]],"*":[["refuse-consent",[]]]}"# factory.setFilterTables(rulesTag: nil, rules: nil, scripts: WKEngineFactory.makeFilterScripts( cosmetic: nil, sites: #"{"example.com":[".ad"]}"#, scriptlets: scriptlets, procedural: nil ))