From e98194b451eaf8a5065f7df0023d14add9ab7321 Mon Sep 17 00:00:00 2001 From: Zhongyue Lin Date: Fri, 10 Jul 2026 17:09:38 +0800 Subject: [PATCH] fix(instagram): recursively collect explore media across nested layouts (#2091) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit instagram explore returned [] despite HTTP 200: IG moved explore_grid media out of the flat sectional_items[].layout_content.medias[] path into mixed nested layout shapes (one_by_two_item.clips.items[].media, fill_items[], …), which the adapter no longer read. Convert the pipeline adapter to a func (like clis/instagram/download.js) so the transform runs in Node and is unit-testable. buildExploreFetchScript() does the in-page fetch and returns { ok, data } | { error }; collectExploreMedia() recursively walks sectional_items collecting every distinct node.media (deduped by pk/id/code) and maps to rank/user/caption/likes/comments/type. likes falls back to play_count for clips/reels, and the generic walk still matches the legacy flat path. Adds clis/instagram/explore.test.js (11 JSON-fixture tests). --- clis/instagram/explore.js | 116 ++++++++++++++++++++++++--------- clis/instagram/explore.test.js | 96 +++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 31 deletions(-) create mode 100644 clis/instagram/explore.test.js diff --git a/clis/instagram/explore.js b/clis/instagram/explore.js index dc6e525cf..184536cde 100644 --- a/clis/instagram/explore.js +++ b/clis/instagram/explore.js @@ -1,5 +1,76 @@ import { cli } from '@jackwener/opencli/registry'; -cli({ +import { CommandExecutionError } from '@jackwener/opencli/errors'; + +const EXPLORE_URL = 'https://www.instagram.com/api/v1/discover/web/explore_grid/'; + +function unwrapEvaluateResult(payload) { + if (payload && !Array.isArray(payload) && typeof payload === 'object' && 'session' in payload && 'data' in payload) { + return payload.data; + } + return payload; +} + +function buildExploreFetchScript() { + return `(async () => { + try { + const res = await fetch(${JSON.stringify(EXPLORE_URL)}, { + credentials: 'include', + headers: { 'X-IG-App-ID': '936619743392459' }, + }); + if (!res.ok) return { error: 'HTTP ' + res.status }; + return { ok: true, data: await res.json() }; + } catch (e) { + return { error: String(e && e.message || e) }; + } + })()`; +} + +function mapMedia(media) { + return { + user: media.user?.username || '', + caption: (media.caption?.text || '').replace(/\n/g, ' ').substring(0, 100), + // clips / reels expose engagement as play_count rather than like_count. + likes: media.like_count ?? media.play_count ?? 0, + comments: media.comment_count ?? 0, + type: media.media_type === 1 ? 'photo' : media.media_type === 2 ? 'video' : 'carousel', + }; +} + +/** + * IG moved explore media out of the flat + * `sectional_items[].layout_content.medias[]` path into mixed nested layout + * shapes (`one_by_two_item.clips.items[].media`, `fill_items[]`, …). Walk + * sectional_items recursively and collect every distinct `node.media`, deduped + * by pk/id/code. The generic walk also still matches the legacy flat path. (#2091) + */ +export function collectExploreMedia(data, limit) { + const seen = new Set(); + const posts = []; + const visit = (node) => { + if (!node || typeof node !== 'object') return; + if (Array.isArray(node)) { + for (const child of node) visit(child); + return; + } + const media = node.media; + if (media && typeof media === 'object' && (media.pk || media.id || media.code)) { + const key = String(media.pk || media.id || media.code); + if (!seen.has(key)) { + seen.add(key); + posts.push(mapMedia(media)); + } + } + for (const k in node) { + if (k === 'media') continue; // already captured; don't re-descend into it + visit(node[k]); + } + }; + visit(data?.sectional_items || []); + const capped = Number.isInteger(limit) && limit > 0 ? posts.slice(0, limit) : posts; + return capped.map((p, i) => ({ rank: i + 1, ...p })); +} + +export const exploreCommand = cli({ site: 'instagram', name: 'explore', access: 'read', @@ -9,34 +80,17 @@ cli({ { name: 'limit', type: 'int', default: 20, help: 'Number of posts' }, ], columns: ['rank', 'user', 'caption', 'likes', 'comments', 'type'], - pipeline: [ - { navigate: 'https://www.instagram.com' }, - { evaluate: `(async () => { - const limit = \${{ args.limit }}; - const res = await fetch( - 'https://www.instagram.com/api/v1/discover/web/explore_grid/', - { - credentials: 'include', - headers: { 'X-IG-App-ID': '936619743392459' } - } - ); - if (!res.ok) throw new Error('HTTP ' + res.status + ' - make sure you are logged in to Instagram'); - const data = await res.json(); - const posts = []; - for (const sec of (data?.sectional_items || [])) { - for (const m of (sec?.layout_content?.medias || [])) { - const media = m?.media; - if (media) posts.push({ - user: media.user?.username || '', - caption: (media.caption?.text || '').replace(/\\n/g, ' ').substring(0, 100), - likes: media.like_count ?? 0, - comments: media.comment_count ?? 0, - type: media.media_type === 1 ? 'photo' : media.media_type === 2 ? 'video' : 'carousel', - }); - } - } - return posts.slice(0, limit).map((p, i) => ({ rank: i + 1, ...p })); -})() -` }, - ], + func: async (page, kwargs) => { + const limit = kwargs.limit || 20; + await page.goto('https://www.instagram.com'); + const result = unwrapEvaluateResult(await page.evaluate(buildExploreFetchScript())); + if (!result || result.error || !result.ok) { + throw new CommandExecutionError( + `instagram explore failed: ${result?.error || 'no response'} — make sure you are logged in to Instagram`, + ); + } + return collectExploreMedia(result.data, limit); + }, }); + +export const __test__ = { collectExploreMedia, mapMedia, buildExploreFetchScript, unwrapEvaluateResult }; diff --git a/clis/instagram/explore.test.js b/clis/instagram/explore.test.js new file mode 100644 index 000000000..a6ec790eb --- /dev/null +++ b/clis/instagram/explore.test.js @@ -0,0 +1,96 @@ +import { describe, expect, it, vi } from 'vitest'; +import { CommandExecutionError } from '@jackwener/opencli/errors'; +import { exploreCommand, __test__ } from './explore.js'; + +const { collectExploreMedia, buildExploreFetchScript } = __test__; + +function mk(pk, extra = {}) { + return { pk, user: { username: 'u' + pk }, caption: { text: 'c' + pk }, ...extra }; +} + +describe('instagram explore extraction (#2091)', () => { + it('collects media from the new nested layout shapes (clips/fill_items)', () => { + const data = { + sectional_items: [ + { one_by_two_item: { clips: { items: [{ media: mk('1', { media_type: 2, play_count: 500 }) }] } } }, + { fill_items: [{ media: mk('2', { media_type: 1, like_count: 10, comment_count: 3 }) }] }, + ], + }; + expect(collectExploreMedia(data, 20)).toEqual([ + { rank: 1, user: 'u1', caption: 'c1', likes: 500, comments: 0, type: 'video' }, + { rank: 2, user: 'u2', caption: 'c2', likes: 10, comments: 3, type: 'photo' }, + ]); + }); + + it('still reads the legacy flat layout_content.medias[] path', () => { + const data = { sectional_items: [{ layout_content: { medias: [{ media: mk('9', { media_type: 8 }) }] } }] }; + expect(collectExploreMedia(data, 20)).toEqual([ + { rank: 1, user: 'u9', caption: 'c9', likes: 0, comments: 0, type: 'carousel' }, + ]); + }); + + it('dedupes the same media by pk/id/code and respects limit', () => { + const shared = mk('5'); + const data = { sectional_items: [{ a: { media: shared } }, { b: { media: shared } }, { c: { media: mk('6') } }] }; + const rows = collectExploreMedia(data, 1); + expect(rows).toHaveLength(1); + expect(rows[0].rank).toBe(1); + }); + + it('falls back to play_count for likes when like_count is absent', () => { + const data = { sectional_items: [{ x: { media: mk('7', { play_count: 999 }) } }] }; + expect(collectExploreMedia(data, 20)[0].likes).toBe(999); + }); + + it('returns [] for empty / missing / non-object input', () => { + expect(collectExploreMedia({}, 20)).toEqual([]); + expect(collectExploreMedia(null, 20)).toEqual([]); + expect(collectExploreMedia({ sectional_items: [] }, 20)).toEqual([]); + }); + + it('ignores nodes whose media has no stable id', () => { + const data = { sectional_items: [{ a: { media: { user: { username: 'x' } } } }] }; + expect(collectExploreMedia(data, 20)).toEqual([]); + }); + + it('dedupes two distinct media objects that share a stable id', () => { + const data = { + sectional_items: [ + { a: { media: mk('5', { like_count: 1 }) } }, + { b: { media: mk('5', { like_count: 2 }) } }, // different object, same pk + ], + }; + expect(collectExploreMedia(data, 20)).toHaveLength(1); + }); + + it('fetch script targets the explore_grid endpoint with the web app id', () => { + const js = buildExploreFetchScript(); + expect(js).toContain('/api/v1/discover/web/explore_grid/'); + expect(js).toContain('936619743392459'); + }); +}); + +describe('instagram explore command (func)', () => { + function makePage(evalResult) { + return { + goto: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn().mockResolvedValue(evalResult), + }; + } + + it('returns ranked rows from the fetched payload', async () => { + const data = { sectional_items: [{ x: { media: mk('1', { media_type: 1 }) } }] }; + const rows = await exploreCommand.func(makePage({ ok: true, data }), { limit: 20 }); + expect(rows).toEqual([{ rank: 1, user: 'u1', caption: 'c1', likes: 0, comments: 0, type: 'photo' }]); + }); + + it('unwraps the Browser Bridge {session,data} envelope', async () => { + const data = { sectional_items: [{ x: { media: mk('1', { media_type: 1 }) } }] }; + const rows = await exploreCommand.func(makePage({ session: 'site:instagram', data: { ok: true, data } }), { limit: 20 }); + expect(rows).toHaveLength(1); + }); + + it('throws CommandExecutionError when the in-page fetch fails', async () => { + await expect(exploreCommand.func(makePage({ error: 'HTTP 403' }), { limit: 20 })).rejects.toBeInstanceOf(CommandExecutionError); + }); +});