fix(instagram): recursively collect explore media across nested layouts (#2091)

clis/instagram/explore.js

@@ -1,5 +1,76 @@
 import { cli } from '@jackwener/opencli/registry';
-cli({
+import { CommandExecutionError } from '@jackwener/opencli/errors';
+
+const EXPLORE_URL = 'https://www.instagram.com/api/v1/discover/web/explore_grid/';
+
+function unwrapEvaluateResult(payload) {
+    if (payload && !Array.isArray(payload) && typeof payload === 'object' && 'session' in payload && 'data' in payload) {
+        return payload.data;
+    }
+    return payload;
+}
+
+function buildExploreFetchScript() {
+    return `(async () => {
+    try {
+      const res = await fetch(${JSON.stringify(EXPLORE_URL)}, {
+        credentials: 'include',
+        headers: { 'X-IG-App-ID': '936619743392459' },
+      });
+      if (!res.ok) return { error: 'HTTP ' + res.status };
+      return { ok: true, data: await res.json() };
+    } catch (e) {
+      return { error: String(e && e.message || e) };
+    }
+  })()`;
+}
+
+function mapMedia(media) {
+    return {
+        user: media.user?.username || '',
+        caption: (media.caption?.text || '').replace(/\n/g, ' ').substring(0, 100),
+        // clips / reels expose engagement as play_count rather than like_count.
+        likes: media.like_count ?? media.play_count ?? 0,
+        comments: media.comment_count ?? 0,
+        type: media.media_type === 1 ? 'photo' : media.media_type === 2 ? 'video' : 'carousel',
+    };
+}
+
+/**
+ * IG moved explore media out of the flat
+ * `sectional_items[].layout_content.medias[]` path into mixed nested layout
+ * shapes (`one_by_two_item.clips.items[].media`, `fill_items[]`, …). Walk
+ * sectional_items recursively and collect every distinct `node.media`, deduped
+ * by pk/id/code. The generic walk also still matches the legacy flat path. (#2091)
+ */
+export function collectExploreMedia(data, limit) {
+    const seen = new Set();
+    const posts = [];
+    const visit = (node) => {
+        if (!node || typeof node !== 'object') return;
+        if (Array.isArray(node)) {
+            for (const child of node) visit(child);
+            return;
+        }
+        const media = node.media;
+        if (media && typeof media === 'object' && (media.pk || media.id || media.code)) {
+            const key = String(media.pk || media.id || media.code);
+            if (!seen.has(key)) {
+                seen.add(key);
+                posts.push(mapMedia(media));
+            }
+        }
+        for (const k in node) {
+            if (k === 'media') continue; // already captured; don't re-descend into it
+            visit(node[k]);
+        }
+    };
+    visit(data?.sectional_items || []);
+    const capped = Number.isInteger(limit) && limit > 0 ? posts.slice(0, limit) : posts;
+    return capped.map((p, i) => ({ rank: i + 1, ...p }));
+}
+
+export const exploreCommand = cli({
     site: 'instagram',
     name: 'explore',
     access: 'read',
@@ -9,34 +80,17 @@ cli({
         { name: 'limit', type: 'int', default: 20, help: 'Number of posts' },
     ],
     columns: ['rank', 'user', 'caption', 'likes', 'comments', 'type'],
-    pipeline: [
-        { navigate: 'https://www.instagram.com' },
-        { evaluate: `(async () => {
-  const limit = \${{ args.limit }};
-  const res = await fetch(
-    'https://www.instagram.com/api/v1/discover/web/explore_grid/',
-    {
-      credentials: 'include',
-      headers: { 'X-IG-App-ID': '936619743392459' }
-    }
-  );
-  if (!res.ok) throw new Error('HTTP ' + res.status + ' - make sure you are logged in to Instagram');
-  const data = await res.json();
-  const posts = [];
-  for (const sec of (data?.sectional_items || [])) {
-    for (const m of (sec?.layout_content?.medias || [])) {
-      const media = m?.media;
-      if (media) posts.push({
-        user: media.user?.username || '',
-        caption: (media.caption?.text || '').replace(/\\n/g, ' ').substring(0, 100),
-        likes: media.like_count ?? 0,
-        comments: media.comment_count ?? 0,
-        type: media.media_type === 1 ? 'photo' : media.media_type === 2 ? 'video' : 'carousel',
-      });
-    }
-  }
-  return posts.slice(0, limit).map((p, i) => ({ rank: i + 1, ...p }));
-})()
-` },
-    ],
+    func: async (page, kwargs) => {
+        const limit = kwargs.limit || 20;
+        await page.goto('https://www.instagram.com');
+        const result = unwrapEvaluateResult(await page.evaluate(buildExploreFetchScript()));
+        if (!result || result.error || !result.ok) {
+            throw new CommandExecutionError(
+                `instagram explore failed: ${result?.error || 'no response'} — make sure you are logged in to Instagram`,
+            );
+        }
+        return collectExploreMedia(result.data, limit);
+    },
 });
+
+export const __test__ = { collectExploreMedia, mapMedia, buildExploreFetchScript, unwrapEvaluateResult };

clis/instagram/explore.test.js

@@ -0,0 +1,96 @@
+import { describe, expect, it, vi } from 'vitest';
+import { CommandExecutionError } from '@jackwener/opencli/errors';
+import { exploreCommand, __test__ } from './explore.js';
+
+const { collectExploreMedia, buildExploreFetchScript } = __test__;
+
+function mk(pk, extra = {}) {
+    return { pk, user: { username: 'u' + pk }, caption: { text: 'c' + pk }, ...extra };
+}
+
+describe('instagram explore extraction (#2091)', () => {
+    it('collects media from the new nested layout shapes (clips/fill_items)', () => {
+        const data = {
+            sectional_items: [
+                { one_by_two_item: { clips: { items: [{ media: mk('1', { media_type: 2, play_count: 500 }) }] } } },
+                { fill_items: [{ media: mk('2', { media_type: 1, like_count: 10, comment_count: 3 }) }] },
+            ],
+        };
+        expect(collectExploreMedia(data, 20)).toEqual([
+            { rank: 1, user: 'u1', caption: 'c1', likes: 500, comments: 0, type: 'video' },
+            { rank: 2, user: 'u2', caption: 'c2', likes: 10, comments: 3, type: 'photo' },
+        ]);
+    });
+
+    it('still reads the legacy flat layout_content.medias[] path', () => {
+        const data = { sectional_items: [{ layout_content: { medias: [{ media: mk('9', { media_type: 8 }) }] } }] };
+        expect(collectExploreMedia(data, 20)).toEqual([
+            { rank: 1, user: 'u9', caption: 'c9', likes: 0, comments: 0, type: 'carousel' },
+        ]);
+    });
+
+    it('dedupes the same media by pk/id/code and respects limit', () => {
+        const shared = mk('5');
+        const data = { sectional_items: [{ a: { media: shared } }, { b: { media: shared } }, { c: { media: mk('6') } }] };
+        const rows = collectExploreMedia(data, 1);
+        expect(rows).toHaveLength(1);
+        expect(rows[0].rank).toBe(1);
+    });
+
+    it('falls back to play_count for likes when like_count is absent', () => {
+        const data = { sectional_items: [{ x: { media: mk('7', { play_count: 999 }) } }] };
+        expect(collectExploreMedia(data, 20)[0].likes).toBe(999);
+    });
+
+    it('returns [] for empty / missing / non-object input', () => {
+        expect(collectExploreMedia({}, 20)).toEqual([]);
+        expect(collectExploreMedia(null, 20)).toEqual([]);
+        expect(collectExploreMedia({ sectional_items: [] }, 20)).toEqual([]);
+    });
+
+    it('ignores nodes whose media has no stable id', () => {
+        const data = { sectional_items: [{ a: { media: { user: { username: 'x' } } } }] };
+        expect(collectExploreMedia(data, 20)).toEqual([]);
+    });
+
+    it('dedupes two distinct media objects that share a stable id', () => {
+        const data = {
+            sectional_items: [
+                { a: { media: mk('5', { like_count: 1 }) } },
+                { b: { media: mk('5', { like_count: 2 }) } }, // different object, same pk
+            ],
+        };
+        expect(collectExploreMedia(data, 20)).toHaveLength(1);
+    });
+
+    it('fetch script targets the explore_grid endpoint with the web app id', () => {
+        const js = buildExploreFetchScript();
+        expect(js).toContain('/api/v1/discover/web/explore_grid/');
+        expect(js).toContain('936619743392459');
+    });
+});
+
+describe('instagram explore command (func)', () => {
+    function makePage(evalResult) {
+        return {
+            goto: vi.fn().mockResolvedValue(undefined),
+            evaluate: vi.fn().mockResolvedValue(evalResult),
+        };
+    }
+
+    it('returns ranked rows from the fetched payload', async () => {
+        const data = { sectional_items: [{ x: { media: mk('1', { media_type: 1 }) } }] };
+        const rows = await exploreCommand.func(makePage({ ok: true, data }), { limit: 20 });
+        expect(rows).toEqual([{ rank: 1, user: 'u1', caption: 'c1', likes: 0, comments: 0, type: 'photo' }]);
+    });
+
+    it('unwraps the Browser Bridge {session,data} envelope', async () => {
+        const data = { sectional_items: [{ x: { media: mk('1', { media_type: 1 }) } }] };
+        const rows = await exploreCommand.func(makePage({ session: 'site:instagram', data: { ok: true, data } }), { limit: 20 });
+        expect(rows).toHaveLength(1);
+    });
+
+    it('throws CommandExecutionError when the in-page fetch fails', async () => {
+        await expect(exploreCommand.func(makePage({ error: 'HTTP 403' }), { limit: 20 })).rejects.toBeInstanceOf(CommandExecutionError);
+    });
+});