Skip to content

Commit 56653ad

Browse files
author
stlc-bot
committed
feat(parse): mark PDF page numbers inline in Markdown with includeNumberedPages (#2110)
Stainless-Generated-From: 48fc8eb475735398832709e3d42eef72ef129e67
1 parent 29fdba0 commit 56653ad

3 files changed

Lines changed: 12 additions & 2 deletions

File tree

‎pkg/cmd/batch_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ func TestBatchSubmit(t *testing.T) {
8484
t,
8585
"--api-key", "string",
8686
"batch", "submit",
87-
"--input", "{data: {formats: {bytes: true, highlights: true, html: true, images: true, json: true, markdown: true, parse: true, product: true, screenshot: true}, urls: [{url: https://example.com/products/anvil, itemId: sku-1, meta: {category: bar}}, {url: https://example.com/products/hammer, itemId: sku-2, meta: {foo: bar}}], highlightsParams: {query: x, maxCharacters: 100}, imageParams: {dedupe: none, enrich: [dimensions]}, jsonParams: {schema: {type: bar, properties: bar, required: bar, additionalProperties: bar}, instructions: instructions}, markdownParams: {includeImages: true, includeLinks: true, inlineImages: placeholder, sourcePreference: [html-conversion]}, maxAgeMs: 0, parseParams: {rules: {title: h1, links: {selector: a, output: '@href', type: list}}, type: custom}, productParams: {dedupeImages: true}, screenshotParams: {area: viewport, format: png}, sharedParams: {actions: [{action: Click the product details tab, type: perform}], country: US, dismissCookies: true, dismissPopups: true, excludeSelectors: [P], headers: {Accept-Language: en-US}, includeFrames: true, includeSelectors: [P], mainContentOnly: true, parsers: {pdf: {endPage: 1, ocr: 'off', startPage: 1}}, profile: {profileId: prof_210b9798eb53baa4e69d31c1071cf03d, type: existing}, settleAnimations: true, theme: light, viewport: {height: 240, width: 240}, waitFor: 500}, tags: [production, team-alpha], timeoutOpts: {milliseconds: 1, behavior: fail}, zdr: disabled}, mode: scrape}",
87+
"--input", "{data: {formats: {bytes: true, highlights: true, html: true, images: true, json: true, markdown: true, parse: true, product: true, screenshot: true}, urls: [{url: https://example.com/products/anvil, itemId: sku-1, meta: {category: bar}}, {url: https://example.com/products/hammer, itemId: sku-2, meta: {foo: bar}}], highlightsParams: {query: x, maxCharacters: 100}, imageParams: {dedupe: none, enrich: [dimensions]}, jsonParams: {schema: {type: bar, properties: bar, required: bar, additionalProperties: bar}, instructions: instructions}, markdownParams: {includeImages: true, includeLinks: true, inlineImages: placeholder, sourcePreference: [html-conversion]}, maxAgeMs: 0, parseParams: {rules: {title: h1, links: {selector: a, output: '@href', type: list}}, type: custom}, productParams: {dedupeImages: true}, screenshotParams: {area: viewport, format: png}, sharedParams: {actions: [{action: Click the product details tab, type: perform}], country: US, dismissCookies: true, dismissPopups: true, excludeSelectors: [P], headers: {Accept-Language: en-US}, includeFrames: true, includeSelectors: [P], mainContentOnly: true, parsers: {pdf: {endPage: 1, includeNumberedPages: true, ocr: 'off', startPage: 1}}, profile: {profileId: prof_210b9798eb53baa4e69d31c1071cf03d, type: existing}, settleAnimations: true, theme: light, viewport: {height: 240, width: 240}, waitFor: 500}, tags: [production, team-alpha], timeoutOpts: {milliseconds: 1, behavior: fail}, zdr: disabled}, mode: scrape}",
8888
"--tag", "docs",
8989
"--tag", "competitor",
9090
"--webhook", "{url: https://example.com, retry: {delays_seconds: [10, 60, 300, 1800, 7200, 21600, 57600]}}",
@@ -102,7 +102,7 @@ func TestBatchSubmit(t *testing.T) {
102102
t,
103103
"--api-key", "string",
104104
"batch", "submit",
105-
"--input", "{data: {formats: {bytes: true, highlights: true, html: true, images: true, json: true, markdown: true, parse: true, product: true, screenshot: true}, urls: [{url: https://example.com/products/anvil, itemId: sku-1, meta: {category: bar}}, {url: https://example.com/products/hammer, itemId: sku-2, meta: {foo: bar}}], highlightsParams: {query: x, maxCharacters: 100}, imageParams: {dedupe: none, enrich: [dimensions]}, jsonParams: {schema: {type: bar, properties: bar, required: bar, additionalProperties: bar}, instructions: instructions}, markdownParams: {includeImages: true, includeLinks: true, inlineImages: placeholder, sourcePreference: [html-conversion]}, maxAgeMs: 0, parseParams: {rules: {title: h1, links: {selector: a, output: '@href', type: list}}, type: custom}, productParams: {dedupeImages: true}, screenshotParams: {area: viewport, format: png}, sharedParams: {actions: [{action: Click the product details tab, type: perform}], country: US, dismissCookies: true, dismissPopups: true, excludeSelectors: [P], headers: {Accept-Language: en-US}, includeFrames: true, includeSelectors: [P], mainContentOnly: true, parsers: {pdf: {endPage: 1, ocr: 'off', startPage: 1}}, profile: {profileId: prof_210b9798eb53baa4e69d31c1071cf03d, type: existing}, settleAnimations: true, theme: light, viewport: {height: 240, width: 240}, waitFor: 500}, tags: [production, team-alpha], timeoutOpts: {milliseconds: 1, behavior: fail}, zdr: disabled}, mode: scrape}",
105+
"--input", "{data: {formats: {bytes: true, highlights: true, html: true, images: true, json: true, markdown: true, parse: true, product: true, screenshot: true}, urls: [{url: https://example.com/products/anvil, itemId: sku-1, meta: {category: bar}}, {url: https://example.com/products/hammer, itemId: sku-2, meta: {foo: bar}}], highlightsParams: {query: x, maxCharacters: 100}, imageParams: {dedupe: none, enrich: [dimensions]}, jsonParams: {schema: {type: bar, properties: bar, required: bar, additionalProperties: bar}, instructions: instructions}, markdownParams: {includeImages: true, includeLinks: true, inlineImages: placeholder, sourcePreference: [html-conversion]}, maxAgeMs: 0, parseParams: {rules: {title: h1, links: {selector: a, output: '@href', type: list}}, type: custom}, productParams: {dedupeImages: true}, screenshotParams: {area: viewport, format: png}, sharedParams: {actions: [{action: Click the product details tab, type: perform}], country: US, dismissCookies: true, dismissPopups: true, excludeSelectors: [P], headers: {Accept-Language: en-US}, includeFrames: true, includeSelectors: [P], mainContentOnly: true, parsers: {pdf: {endPage: 1, includeNumberedPages: true, ocr: 'off', startPage: 1}}, profile: {profileId: prof_210b9798eb53baa4e69d31c1071cf03d, type: existing}, settleAnimations: true, theme: light, viewport: {height: 240, width: 240}, waitFor: 500}, tags: [production, team-alpha], timeoutOpts: {milliseconds: 1, behavior: fail}, zdr: disabled}, mode: scrape}",
106106
"--tag", "docs",
107107
"--tag", "competitor",
108108
"--webhook.url", "https://example.com",
@@ -188,6 +188,7 @@ func TestBatchSubmit(t *testing.T) {
188188
" parsers:\n" +
189189
" pdf:\n" +
190190
" endPage: 1\n" +
191+
" includeNumberedPages: true\n" +
191192
" ocr: 'off'\n" +
192193
" startPage: 1\n" +
193194
" profile:\n" +

‎pkg/cmd/parse.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,12 @@ var parseHandle = requestflag.WithInnerFlags(cli.Command{
4949
Default: true,
5050
QueryPath: "includeLinks",
5151
},
52+
&requestflag.Flag[bool]{
53+
Name: "include-numbered-pages",
54+
Usage: "Start each PDF page in the Markdown with a `<!-- page N -->` line, where N is the original 1-based page number.",
55+
Default: false,
56+
QueryPath: "includeNumberedPages",
57+
},
5258
&requestflag.Flag[bool]{
5359
Name: "ocr",
5460
Usage: "Read text from images and scanned PDF pages. PDF page ranges still apply.",

‎pkg/cmd/parse_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ func TestParseHandle(t *testing.T) {
2121
"--extension", "txt",
2222
"--include-images=true",
2323
"--include-links=true",
24+
"--include-numbered-pages=true",
2425
"--ocr=true",
2526
"--pdf", "{end: 1, start: 1}",
2627
"--shorten-base64-images=true",
@@ -45,6 +46,7 @@ func TestParseHandle(t *testing.T) {
4546
"--extension", "txt",
4647
"--include-images=true",
4748
"--include-links=true",
49+
"--include-numbered-pages=true",
4850
"--ocr=true",
4951
"--pdf.end", "1",
5052
"--pdf.start", "1",
@@ -67,6 +69,7 @@ func TestParseHandle(t *testing.T) {
6769
"--extension", "txt",
6870
"--include-images=true",
6971
"--include-links=true",
72+
"--include-numbered-pages=true",
7073
"--ocr=true",
7174
"--pdf", "{end: 1, start: 1}",
7275
"--shorten-base64-images=true",

0 commit comments

Comments
 (0)