fix(facebook): extract search results from role=feed entity links (#2090)

clis/facebook/search.js

@@ -1,5 +1,74 @@
 import { cli } from '@jackwener/opencli/registry';
-cli({
+import { CommandExecutionError } from '@jackwener/opencli/errors';
+
+function unwrapEvaluateResult(payload) {
+    if (payload && !Array.isArray(payload) && typeof payload === 'object' && 'session' in payload && 'data' in payload) {
+        return payload.data;
+    }
+    return payload;
+}
+
+// Modern facebook.com /search/top renders results inside [role="feed"] as
+// entity/content links (people, pages, groups, posts). The old extractor keyed
+// on [role="article"]/[role="listitem"], which now wrap left-nav and story
+// chrome instead — so it returned notifications/stories/live rows. FB also
+// injects scrambled decoy anchors back to /search/ plus hidden-character noise.
+// (#2090)
+//
+// Scoping to [role="feed"] and dropping /search/ decoys is the structural core;
+// the obfuscation filters below are conservative and best-effort — verify on a
+// live logged-in FB session.
+function buildFacebookSearchJs(limit) {
+    return `(async () => {
+    const limit = ${JSON.stringify(limit)};
+    const clean = (t) => (t || '').replace(/\\s+/g, ' ').trim();
+    const isObfuscated = (t) => {
+      const s = (t || '').trim();
+      if (!s) return true;
+      if (/\\d{12,}/.test(s.replace(/\\s+/g, ''))) return true; // long spaceless digit token
+      if (/^(?:\\S\\s+){3,}\\S$/.test(s)) return true;          // spaced single-character decoy
+      return false;
+    };
+    const out = [];
+    const seen = new Set();
+    for (const feed of document.querySelectorAll('[role="feed"]')) {
+      for (const a of feed.querySelectorAll('a[href]')) {
+        if (out.length >= limit) break;
+        const raw = a.href || a.getAttribute('href') || '';
+        let u;
+        try { u = new URL(raw, location.href); } catch { continue; }
+        const host = u.hostname.toLowerCase();
+        // Real facebook.com entity/content links only — exact hostname match, not
+        // a substring (which would admit notfacebook.com) and not subdomains
+        // (l.facebook.com / lm.facebook.com are outbound-redirect shims).
+        if (host !== 'facebook.com' && host !== 'www.facebook.com') continue;
+        if (/^\\/search\\//.test(u.pathname)) continue; // drop /search/ decoys
+        // Keep the query for query-identity content URLs (photo.php?fbid=,
+        // story.php?, watch/?v=); strip it for vanity/profile paths where it is
+        // just tracking noise.
+        const needsQuery = /\\.php$/.test(u.pathname) || /^\\/watch\\/?$/.test(u.pathname);
+        const url = needsQuery ? (u.origin + u.pathname + u.search) : (u.origin + u.pathname);
+        if (seen.has(url)) continue;
+        const title = clean(a.textContent).substring(0, 80);
+        if (!title || isObfuscated(title)) continue;
+        seen.add(url);
+        // Use the post card when present; otherwise fall back to the anchor's own
+        // text, not a.parentElement — for feed-level links that can be the whole
+        // feed, pulling in unrelated results/decoys.
+        const container = a.closest('[role="article"]') || a;
+        out.push({
+          index: out.length + 1,
+          title,
+          text: clean(container.textContent).substring(0, 150),
+          url,
+        });
+      }
+    }
+    return out;
+  })()`;
+}
+
+export const facebookSearchCommand = cli({
     site: 'facebook',
     name: 'search',
     access: 'read',
@@ -10,30 +79,16 @@ cli({
         { name: 'limit', type: 'int', default: 10, help: 'Number of results' },
     ],
     columns: ['index', 'title', 'text', 'url'],
-    pipeline: [
-        { navigate: 'https://www.facebook.com' },
-        { navigate: { url: 'https://www.facebook.com/search/top?q=${{ args.query | urlencode }}', settleMs: 4000 } },
-        { evaluate: `(async () => {
-  const limit = \${{ args.limit }};
-  // Search results are typically in role="article" or role="listitem"
-  let items = document.querySelectorAll('[role="article"]');
-  if (items.length === 0) {
-    items = document.querySelectorAll('[role="listitem"]');
-  }
-  return Array.from(items)
-    .filter(el => el.textContent.trim().length > 20)
-    .slice(0, limit)
-    .map((el, i) => {
-      const link = el.querySelector('a[href*="facebook.com/"]');
-      const heading = el.querySelector('h2, h3, h4, strong');
-      return {
-        index: i + 1,
-        title: heading ? heading.textContent.trim().substring(0, 80) : '',
-        text: el.textContent.trim().replace(/\\s+/g, ' ').substring(0, 150),
-        url: link ? link.href.split('?')[0] : '',
-      };
-    });
-})()
-` },
-    ],
+    func: async (page, kwargs) => {
+        const limit = kwargs.limit || 10;
+        await page.goto('https://www.facebook.com');
+        await page.goto(`https://www.facebook.com/search/top?q=${encodeURIComponent(kwargs.query)}`, { settleMs: 4000 });
+        const rows = unwrapEvaluateResult(await page.evaluate(buildFacebookSearchJs(limit)));
+        if (!Array.isArray(rows)) {
+            throw new CommandExecutionError('facebook search: unexpected extraction payload (expected an array of rows)');
+        }
+        return rows;
+    },
 });
+
+export const __test__ = { buildFacebookSearchJs, unwrapEvaluateResult };

clis/facebook/search.test.js

@@ -1,55 +1,101 @@
-/**
- * Regression test for issue #625.
- * Facebook search must navigate in the pipeline before DOM extraction.
- */
+import { JSDOM } from 'jsdom';
 import { describe, expect, it, vi } from 'vitest';
-import { getRegistry } from '@jackwener/opencli/registry';
-import { executePipeline } from '@jackwener/opencli/pipeline';
-// Import the adapter to register it
-import './search.js';
-/**
- * Minimal browser mock for pipeline execution tests.
- * Only methods touched by this adapter path are implemented.
- */
-function createMockPage() {
-    return {
-        goto: vi.fn(),
-        evaluate: vi.fn().mockResolvedValue([]),
-        getCookies: vi.fn().mockResolvedValue([]),
-        snapshot: vi.fn().mockResolvedValue(''),
-        click: vi.fn(),
-        typeText: vi.fn(),
-        pressKey: vi.fn(),
-        scrollTo: vi.fn(),
-        getFormState: vi.fn().mockResolvedValue({}),
-        wait: vi.fn(),
-        tabs: vi.fn().mockResolvedValue([]),
-        selectTab: vi.fn(),
-        networkRequests: vi.fn().mockResolvedValue([]),
-        consoleMessages: vi.fn().mockResolvedValue(''),
-        scroll: vi.fn(),
-        autoScroll: vi.fn(),
-        installInterceptor: vi.fn(),
-        getInterceptedRequests: vi.fn().mockResolvedValue([]),
-        waitForCapture: vi.fn().mockResolvedValue(undefined),
-        screenshot: vi.fn().mockResolvedValue(''),
-    };
+import { CommandExecutionError } from '@jackwener/opencli/errors';
+import { facebookSearchCommand, __test__ } from './search.js';
+
+function runBrowserScript(html, script, url = 'https://www.facebook.com/search/top?q=test') {
+    const dom = new JSDOM(html, { url, runScripts: 'outside-only' });
+    return dom.window.eval(script);
 }
-describe('facebook search pipeline', () => {
-    it('navigates to search results before extracting DOM data', async () => {
-        const cmd = getRegistry().get('facebook/search');
-        expect(cmd).toBeDefined();
-        const pipeline = cmd.pipeline ?? [];
-        const page = createMockPage();
-        await executePipeline(page, pipeline, {
-            args: { query: 'AI agent', limit: 3 },
-        });
+
+const FIXTURE = `
+  <a href="https://www.facebook.com/leftnav">Home</a>
+  <div role="feed">
+    <div role="article">
+      <a href="https://www.facebook.com/john.doe">John Doe</a>
+      <div>John Doe · 1.2k followers</div>
+    </div>
+    <a href="https://www.facebook.com/somepage/">Some Page</a>
+    <a href="https://www.facebook.com/search/top?q=x">search decoy</a>
+    <a href="https://www.facebook.com/1234567890123456">1234567890123456</a>
+    <a href="https://www.facebook.com/zz">a b c d e</a>
+    <a href="https://example.com/notfb">External Site</a>
+    <a href="https://notfacebook.com/evil">Not Facebook</a>
+    <a href="https://l.facebook.com/l.php?u=https%3A%2F%2Fspam.com">Redirect</a>
+  </div>`;
+
+describe('facebook search extraction (#2090)', () => {
+    it('keeps real feed entity links and drops nav / decoys / obfuscation', async () => {
+        const rows = await runBrowserScript(FIXTURE, __test__.buildFacebookSearchJs(10));
+        expect(rows.map(r => r.url)).toEqual([
+            'https://www.facebook.com/john.doe', // real person entity
+            'https://www.facebook.com/somepage/', // real page entity
+        ]);
+        // Dropped: /search/ decoy, 16-digit token, "a b c d e" single-char decoy,
+        // external link, and the out-of-feed nav link.
+        expect(rows.map(r => r.index)).toEqual([1, 2]);
+        expect(rows[0].title).toBe('John Doe');
+    });
+
+    it('respects the limit', async () => {
+        const rows = await runBrowserScript(FIXTURE, __test__.buildFacebookSearchJs(1));
+        expect(rows).toHaveLength(1);
+        expect(rows[0].url).toBe('https://www.facebook.com/john.doe');
+    });
+
+    it('returns [] when there is no role=feed container', async () => {
+        const rows = await runBrowserScript('<div><a href="https://www.facebook.com/x">x</a></div>', __test__.buildFacebookSearchJs(10));
+        expect(rows).toEqual([]);
+    });
+
+    it('preserves query-identity params for content URLs (photo.php / watch)', async () => {
+        const html = `<div role="feed">
+          <a href="https://www.facebook.com/photo.php?fbid=999&__tn__=trk">Cool Photo</a>
+          <a href="https://www.facebook.com/watch/?v=12345">A Video</a>
+        </div>`;
+        const rows = await runBrowserScript(html, __test__.buildFacebookSearchJs(10));
+        expect(rows.map(r => r.url)).toEqual([
+            'https://www.facebook.com/photo.php?fbid=999&__tn__=trk',
+            'https://www.facebook.com/watch/?v=12345',
+        ]);
+    });
+
+    it('uses the anchor text (not the whole feed) for a bare feed-level link', async () => {
+        const html = `<div role="feed">
+          <a href="https://www.facebook.com/alice">Alice Wonderland</a>
+          <a href="https://www.facebook.com/bob">Bob Builder</a>
+        </div>`;
+        const rows = await runBrowserScript(html, __test__.buildFacebookSearchJs(10));
+        expect(rows[0].text).toBe('Alice Wonderland');
+    });
+});
+
+describe('facebook search command (func)', () => {
+    function makePage(evalResult) {
+        return {
+            goto: vi.fn().mockResolvedValue(undefined),
+            evaluate: vi.fn().mockResolvedValue(evalResult),
+        };
+    }
+
+    it('navigates to /search/top before extracting the DOM (regression #625)', async () => {
+        const rows = [{ index: 1, title: 'x', text: 'y', url: 'https://www.facebook.com/x' }];
+        const page = makePage(rows);
+        const out = await facebookSearchCommand.func(page, { query: 'AI agent', limit: 3 });
+        expect(out).toBe(rows);
         expect(page.goto).toHaveBeenNthCalledWith(1, 'https://www.facebook.com');
-        expect(page.goto).toHaveBeenNthCalledWith(2, 'https://www.facebook.com/search/top?q=AI%20agent', {
-            waitUntil: undefined,
-            settleMs: 4000,
-        });
-        expect(page.evaluate).toHaveBeenCalledTimes(1);
-        expect(String(page.evaluate.mock.calls[0]?.[0] ?? '')).not.toContain('window.location.href');
+        expect(page.goto).toHaveBeenNthCalledWith(2, 'https://www.facebook.com/search/top?q=AI%20agent', { settleMs: 4000 });
+        // #625: extraction must run only after navigation completed.
+        expect(page.evaluate.mock.invocationCallOrder[0]).toBeGreaterThan(page.goto.mock.invocationCallOrder[1]);
+    });
+
+    it('unwraps the Browser Bridge {session,data} envelope', async () => {
+        const rows = [{ index: 1, title: 'x', text: 'y', url: 'u' }];
+        const out = await facebookSearchCommand.func(makePage({ session: 'site:facebook', data: rows }), { query: 'x' });
+        expect(out).toBe(rows);
+    });
+
+    it('throws CommandExecutionError on a non-array payload', async () => {
+        await expect(facebookSearchCommand.func(makePage({ oops: true }), { query: 'x' })).rejects.toBeInstanceOf(CommandExecutionError);
     });
 });