From 35df9b3b4073e9629125a9db78f7c5872d6cfb39 Mon Sep 17 00:00:00 2001 From: Zhongyue Lin Date: Tue, 14 Jul 2026 14:04:13 +0800 Subject: [PATCH] fix(facebook): extract search results from role=feed entity links (#2090) facebook search returned notifications/stories/live rows instead of results: the extractor keyed on [role=article]/[role=listitem], which on modern facebook.com wrap left-nav and story chrome, not results. Results now render inside [role=feed] as entity/content links, and FB injects /search/ decoy anchors plus hidden-character noise. Convert the pipeline adapter to a func (testable) that scopes extraction to [role=feed], keeps only exact-host facebook.com entity/content links (rejecting notfacebook.com substring impostors and l.facebook.com redirect shims), drops /search/ decoys, preserves query-identity params for content URLs (photo.php?fbid=/story.php?/watch/?v=), and filters conservative obfuscation (long digit tokens, spaced single-char decoys). Preserves the #625 navigate-before-extract order. Adds jsdom-fixture tests. --- clis/facebook/search.js | 109 +++++++++++++++++++------- clis/facebook/search.test.js | 146 +++++++++++++++++++++++------------ 2 files changed, 178 insertions(+), 77 deletions(-) diff --git a/clis/facebook/search.js b/clis/facebook/search.js index 0f6878939..fc3744963 100644 --- a/clis/facebook/search.js +++ b/clis/facebook/search.js @@ -1,5 +1,74 @@ import { cli } from '@jackwener/opencli/registry'; -cli({ +import { CommandExecutionError } from '@jackwener/opencli/errors'; + +function unwrapEvaluateResult(payload) { + if (payload && !Array.isArray(payload) && typeof payload === 'object' && 'session' in payload && 'data' in payload) { + return payload.data; + } + return payload; +} + +// Modern facebook.com /search/top renders results inside [role="feed"] as +// entity/content links (people, pages, groups, posts). The old extractor keyed +// on [role="article"]/[role="listitem"], which now wrap left-nav and story +// chrome instead — so it returned notifications/stories/live rows. FB also +// injects scrambled decoy anchors back to /search/ plus hidden-character noise. +// (#2090) +// +// Scoping to [role="feed"] and dropping /search/ decoys is the structural core; +// the obfuscation filters below are conservative and best-effort — verify on a +// live logged-in FB session. +function buildFacebookSearchJs(limit) { + return `(async () => { + const limit = ${JSON.stringify(limit)}; + const clean = (t) => (t || '').replace(/\\s+/g, ' ').trim(); + const isObfuscated = (t) => { + const s = (t || '').trim(); + if (!s) return true; + if (/\\d{12,}/.test(s.replace(/\\s+/g, ''))) return true; // long spaceless digit token + if (/^(?:\\S\\s+){3,}\\S$/.test(s)) return true; // spaced single-character decoy + return false; + }; + const out = []; + const seen = new Set(); + for (const feed of document.querySelectorAll('[role="feed"]')) { + for (const a of feed.querySelectorAll('a[href]')) { + if (out.length >= limit) break; + const raw = a.href || a.getAttribute('href') || ''; + let u; + try { u = new URL(raw, location.href); } catch { continue; } + const host = u.hostname.toLowerCase(); + // Real facebook.com entity/content links only — exact hostname match, not + // a substring (which would admit notfacebook.com) and not subdomains + // (l.facebook.com / lm.facebook.com are outbound-redirect shims). + if (host !== 'facebook.com' && host !== 'www.facebook.com') continue; + if (/^\\/search\\//.test(u.pathname)) continue; // drop /search/ decoys + // Keep the query for query-identity content URLs (photo.php?fbid=, + // story.php?, watch/?v=); strip it for vanity/profile paths where it is + // just tracking noise. + const needsQuery = /\\.php$/.test(u.pathname) || /^\\/watch\\/?$/.test(u.pathname); + const url = needsQuery ? (u.origin + u.pathname + u.search) : (u.origin + u.pathname); + if (seen.has(url)) continue; + const title = clean(a.textContent).substring(0, 80); + if (!title || isObfuscated(title)) continue; + seen.add(url); + // Use the post card when present; otherwise fall back to the anchor's own + // text, not a.parentElement — for feed-level links that can be the whole + // feed, pulling in unrelated results/decoys. + const container = a.closest('[role="article"]') || a; + out.push({ + index: out.length + 1, + title, + text: clean(container.textContent).substring(0, 150), + url, + }); + } + } + return out; + })()`; +} + +export const facebookSearchCommand = cli({ site: 'facebook', name: 'search', access: 'read', @@ -10,30 +79,16 @@ cli({ { name: 'limit', type: 'int', default: 10, help: 'Number of results' }, ], columns: ['index', 'title', 'text', 'url'], - pipeline: [ - { navigate: 'https://www.facebook.com' }, - { navigate: { url: 'https://www.facebook.com/search/top?q=${{ args.query | urlencode }}', settleMs: 4000 } }, - { evaluate: `(async () => { - const limit = \${{ args.limit }}; - // Search results are typically in role="article" or role="listitem" - let items = document.querySelectorAll('[role="article"]'); - if (items.length === 0) { - items = document.querySelectorAll('[role="listitem"]'); - } - return Array.from(items) - .filter(el => el.textContent.trim().length > 20) - .slice(0, limit) - .map((el, i) => { - const link = el.querySelector('a[href*="facebook.com/"]'); - const heading = el.querySelector('h2, h3, h4, strong'); - return { - index: i + 1, - title: heading ? heading.textContent.trim().substring(0, 80) : '', - text: el.textContent.trim().replace(/\\s+/g, ' ').substring(0, 150), - url: link ? link.href.split('?')[0] : '', - }; - }); -})() -` }, - ], + func: async (page, kwargs) => { + const limit = kwargs.limit || 10; + await page.goto('https://www.facebook.com'); + await page.goto(`https://www.facebook.com/search/top?q=${encodeURIComponent(kwargs.query)}`, { settleMs: 4000 }); + const rows = unwrapEvaluateResult(await page.evaluate(buildFacebookSearchJs(limit))); + if (!Array.isArray(rows)) { + throw new CommandExecutionError('facebook search: unexpected extraction payload (expected an array of rows)'); + } + return rows; + }, }); + +export const __test__ = { buildFacebookSearchJs, unwrapEvaluateResult }; diff --git a/clis/facebook/search.test.js b/clis/facebook/search.test.js index b3c8fd2fa..5680920a2 100644 --- a/clis/facebook/search.test.js +++ b/clis/facebook/search.test.js @@ -1,55 +1,101 @@ -/** - * Regression test for issue #625. - * Facebook search must navigate in the pipeline before DOM extraction. - */ +import { JSDOM } from 'jsdom'; import { describe, expect, it, vi } from 'vitest'; -import { getRegistry } from '@jackwener/opencli/registry'; -import { executePipeline } from '@jackwener/opencli/pipeline'; -// Import the adapter to register it -import './search.js'; -/** - * Minimal browser mock for pipeline execution tests. - * Only methods touched by this adapter path are implemented. - */ -function createMockPage() { - return { - goto: vi.fn(), - evaluate: vi.fn().mockResolvedValue([]), - getCookies: vi.fn().mockResolvedValue([]), - snapshot: vi.fn().mockResolvedValue(''), - click: vi.fn(), - typeText: vi.fn(), - pressKey: vi.fn(), - scrollTo: vi.fn(), - getFormState: vi.fn().mockResolvedValue({}), - wait: vi.fn(), - tabs: vi.fn().mockResolvedValue([]), - selectTab: vi.fn(), - networkRequests: vi.fn().mockResolvedValue([]), - consoleMessages: vi.fn().mockResolvedValue(''), - scroll: vi.fn(), - autoScroll: vi.fn(), - installInterceptor: vi.fn(), - getInterceptedRequests: vi.fn().mockResolvedValue([]), - waitForCapture: vi.fn().mockResolvedValue(undefined), - screenshot: vi.fn().mockResolvedValue(''), - }; +import { CommandExecutionError } from '@jackwener/opencli/errors'; +import { facebookSearchCommand, __test__ } from './search.js'; + +function runBrowserScript(html, script, url = 'https://www.facebook.com/search/top?q=test') { + const dom = new JSDOM(html, { url, runScripts: 'outside-only' }); + return dom.window.eval(script); } -describe('facebook search pipeline', () => { - it('navigates to search results before extracting DOM data', async () => { - const cmd = getRegistry().get('facebook/search'); - expect(cmd).toBeDefined(); - const pipeline = cmd.pipeline ?? []; - const page = createMockPage(); - await executePipeline(page, pipeline, { - args: { query: 'AI agent', limit: 3 }, - }); + +const FIXTURE = ` + Home +
+
+ John Doe +
John Doe · 1.2k followers
+
+ Some Page + search decoy + 1234567890123456 + a b c d e + External Site + Not Facebook + Redirect +
`; + +describe('facebook search extraction (#2090)', () => { + it('keeps real feed entity links and drops nav / decoys / obfuscation', async () => { + const rows = await runBrowserScript(FIXTURE, __test__.buildFacebookSearchJs(10)); + expect(rows.map(r => r.url)).toEqual([ + 'https://www.facebook.com/john.doe', // real person entity + 'https://www.facebook.com/somepage/', // real page entity + ]); + // Dropped: /search/ decoy, 16-digit token, "a b c d e" single-char decoy, + // external link, and the out-of-feed nav link. + expect(rows.map(r => r.index)).toEqual([1, 2]); + expect(rows[0].title).toBe('John Doe'); + }); + + it('respects the limit', async () => { + const rows = await runBrowserScript(FIXTURE, __test__.buildFacebookSearchJs(1)); + expect(rows).toHaveLength(1); + expect(rows[0].url).toBe('https://www.facebook.com/john.doe'); + }); + + it('returns [] when there is no role=feed container', async () => { + const rows = await runBrowserScript('
x
', __test__.buildFacebookSearchJs(10)); + expect(rows).toEqual([]); + }); + + it('preserves query-identity params for content URLs (photo.php / watch)', async () => { + const html = `
+ Cool Photo + A Video +
`; + const rows = await runBrowserScript(html, __test__.buildFacebookSearchJs(10)); + expect(rows.map(r => r.url)).toEqual([ + 'https://www.facebook.com/photo.php?fbid=999&__tn__=trk', + 'https://www.facebook.com/watch/?v=12345', + ]); + }); + + it('uses the anchor text (not the whole feed) for a bare feed-level link', async () => { + const html = `
+ Alice Wonderland + Bob Builder +
`; + const rows = await runBrowserScript(html, __test__.buildFacebookSearchJs(10)); + expect(rows[0].text).toBe('Alice Wonderland'); + }); +}); + +describe('facebook search command (func)', () => { + function makePage(evalResult) { + return { + goto: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn().mockResolvedValue(evalResult), + }; + } + + it('navigates to /search/top before extracting the DOM (regression #625)', async () => { + const rows = [{ index: 1, title: 'x', text: 'y', url: 'https://www.facebook.com/x' }]; + const page = makePage(rows); + const out = await facebookSearchCommand.func(page, { query: 'AI agent', limit: 3 }); + expect(out).toBe(rows); expect(page.goto).toHaveBeenNthCalledWith(1, 'https://www.facebook.com'); - expect(page.goto).toHaveBeenNthCalledWith(2, 'https://www.facebook.com/search/top?q=AI%20agent', { - waitUntil: undefined, - settleMs: 4000, - }); - expect(page.evaluate).toHaveBeenCalledTimes(1); - expect(String(page.evaluate.mock.calls[0]?.[0] ?? '')).not.toContain('window.location.href'); + expect(page.goto).toHaveBeenNthCalledWith(2, 'https://www.facebook.com/search/top?q=AI%20agent', { settleMs: 4000 }); + // #625: extraction must run only after navigation completed. + expect(page.evaluate.mock.invocationCallOrder[0]).toBeGreaterThan(page.goto.mock.invocationCallOrder[1]); + }); + + it('unwraps the Browser Bridge {session,data} envelope', async () => { + const rows = [{ index: 1, title: 'x', text: 'y', url: 'u' }]; + const out = await facebookSearchCommand.func(makePage({ session: 'site:facebook', data: rows }), { query: 'x' }); + expect(out).toBe(rows); + }); + + it('throws CommandExecutionError on a non-array payload', async () => { + await expect(facebookSearchCommand.func(makePage({ oops: true }), { query: 'x' })).rejects.toBeInstanceOf(CommandExecutionError); }); });