Add typed JSON output to doc fetch

.changeset/doc-fetch-json-output.md

@@ -0,0 +1,5 @@
+---
+'@shopify/cli': minor
+---
+
+Add typed JSON output and schema discovery to `doc fetch`.

docs-shopify.dev/generated/generated_docs_data_v2.json

@@ -5154,7 +5154,7 @@
           "syntaxKind": "PropertySignature",
           "name": "--output <value>",
           "value": "string",
-          "description": "Write the document to this file path instead of printing it to stdout.",
+          "description": "Write the document to this file path instead of printing it to stdout. With --json, stdout contains the absolute path and Markdown format of the written file.",
           "isOptional": true,
           "environmentValue": "SHOPIFY_FLAG_OUTPUT"
         },
@@ -5174,9 +5174,18 @@
           "description": "Increase the verbosity of the output. May include sensitive data.",
           "isOptional": true,
           "environmentValue": "SHOPIFY_FLAG_VERBOSE"
+        },
+        {
+          "filePath": "docs-shopify.dev/commands/interfaces/doc-fetch.interface.ts",
+          "syntaxKind": "PropertySignature",
+          "name": "-j, --json",
+          "value": "''",
+          "description": "Output the result as JSON. Automatically disables color output.",
+          "isOptional": true,
+          "environmentValue": "SHOPIFY_FLAG_JSON"
         }
       ],
-      "value": "export interface docfetch {\n  /**\n   * Print the command's JSON schemas.\n   * @environment SHOPIFY_FLAG_JSON_SCHEMA\n   */\n  '--json-schema'?: ''\n\n  /**\n   * Filter code examples in the returned Markdown to this language. Supply the language of the app you are building so examples match your stack. Optional — if omitted, or if shopify.dev does not recognize the language for a given page, the document includes examples in every language.\n   * @environment SHOPIFY_FLAG_LANGUAGE\n   */\n  '--language <value>'?: string\n\n  /**\n   * Disable color output.\n   * @environment SHOPIFY_FLAG_NO_COLOR\n   */\n  '--no-color'?: ''\n\n  /**\n   * Disable interactive prompts and browser authentication.\n   * @environment SHOPIFY_FLAG_NO_INPUT\n   */\n  '--no-input'?: ''\n\n  /**\n   * Write the document to this file path instead of printing it to stdout.\n   * @environment SHOPIFY_FLAG_OUTPUT\n   */\n  '--output <value>'?: string\n\n  /**\n   * The shopify.dev URL to fetch.\n   * @environment SHOPIFY_FLAG_URL\n   */\n  '--url <value>': string\n\n  /**\n   * Increase the verbosity of the output. May include sensitive data.\n   * @environment SHOPIFY_FLAG_VERBOSE\n   */\n  '--verbose'?: ''\n}"
+      "value": "export interface docfetch {\n  /**\n   * Output the result as JSON. Automatically disables color output.\n   * @environment SHOPIFY_FLAG_JSON\n   */\n  '-j, --json'?: ''\n\n  /**\n   * Print the command's JSON schemas.\n   * @environment SHOPIFY_FLAG_JSON_SCHEMA\n   */\n  '--json-schema'?: ''\n\n  /**\n   * Filter code examples in the returned Markdown to this language. Supply the language of the app you are building so examples match your stack. Optional — if omitted, or if shopify.dev does not recognize the language for a given page, the document includes examples in every language.\n   * @environment SHOPIFY_FLAG_LANGUAGE\n   */\n  '--language <value>'?: string\n\n  /**\n   * Disable color output.\n   * @environment SHOPIFY_FLAG_NO_COLOR\n   */\n  '--no-color'?: ''\n\n  /**\n   * Disable interactive prompts and browser authentication.\n   * @environment SHOPIFY_FLAG_NO_INPUT\n   */\n  '--no-input'?: ''\n\n  /**\n   * Write the document to this file path instead of printing it to stdout. With --json, stdout contains the absolute path and Markdown format of the written file.\n   * @environment SHOPIFY_FLAG_OUTPUT\n   */\n  '--output <value>'?: string\n\n  /**\n   * The shopify.dev URL to fetch.\n   * @environment SHOPIFY_FLAG_URL\n   */\n  '--url <value>': string\n\n  /**\n   * Increase the verbosity of the output. May include sensitive data.\n   * @environment SHOPIFY_FLAG_VERBOSE\n   */\n  '--verbose'?: ''\n}"
     }
   },
   "docsearch": {

packages/cli/README.md

@@ -4475,11 +4475,15 @@ Download a complete document from shopify.dev. Every page on shopify.dev has a M
 
 ```
 USAGE
-  $ shopify doc fetch --url <value> [--json-schema] [--language
+  $ shopify doc fetch --url <value> [-j] [--json-schema] [--language
     javascript|typescript|python|ruby|php|rust|curl|liquid|graphql|html] [--no-color] [--no-input] [--output <value>]
     [--verbose]
 
 FLAGS
+  -j, --json
+      Output the result as JSON. Automatically disables color output.
+      [env: SHOPIFY_FLAG_JSON]
+
   --json-schema
       Print the command's JSON schemas.
       [env: SHOPIFY_FLAG_JSON_SCHEMA]
@@ -4500,7 +4504,8 @@ FLAGS
       [env: SHOPIFY_FLAG_NO_INPUT]
 
   --output=<value>
-      Write the document to this file path instead of printing it to stdout.
+      Write the document to this file path instead of printing it to stdout. With --json, stdout contains the absolute
+      path and Markdown format of the written file.
       [env: SHOPIFY_FLAG_OUTPUT]
 
   --url=<value>
@@ -4517,6 +4522,75 @@ DESCRIPTION
   a centrally-served skill. Pass `--language` for the language of the app you are building so code examples match your
   stack. For finding the relevant pieces of content across shopify.dev instead, use `doc search`.
 
+  Use `--json-schema` to print the result, error, and event schemas.
+
+  Output from `--json` conforms to the `DocFetchResult` schema.
+
+  ```json
+  {
+    "anyOf": [
+      {
+        "type": "object",
+        "properties": {
+          "document": {
+            "$ref": "#/definitions/Document"
+          }
+        },
+        "required": [
+          "document"
+        ],
+        "additionalProperties": false
+      },
+      {
+        "$ref": "#/definitions/DocumentFile"
+      }
+    ],
+    "title": "DocFetchResult",
+    "definitions": {
+      "Document": {
+        "type": "object",
+        "properties": {
+          "url": {
+            "type": "string",
+            "format": "uri",
+            "description": "The requested shopify.dev document URL."
+          },
+          "content": {
+            "type": "string",
+            "description": "The document in Markdown, with the requested language filter applied."
+          }
+        },
+        "required": [
+          "url",
+          "content"
+        ],
+        "additionalProperties": false
+      },
+      "DocumentFile": {
+        "type": "object",
+        "properties": {
+          "path": {
+            "type": "string",
+            "pattern": "^(?:\\/|[A-Za-z]:[\\\\/]|\\\\\\\\)",
+            "description": "The absolute native path of the written file."
+          },
+          "format": {
+            "type": "string",
+            "const": "markdown",
+            "description": "The file contains the original Markdown document, not a JSON wrapper."
+          }
+        },
+        "required": [
+          "path",
+          "format"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "$schema": "http://json-schema.org/draft-07/schema#"
+  }
+  ```
+
 EXAMPLES
   # fetch the Markdown version of a Shopify.dev page
 
@@ -4529,6 +4603,10 @@ EXAMPLES
   # save the document to a file instead of printing it
 
     $ shopify doc fetch --url https://shopify.dev/docs/api/shopify-cli --output docs/shopify-cli.md
+
+  # return a typed document as JSON
+
+    $ shopify doc fetch --url https://shopify.dev/docs/api/shopify-cli --json
 ```
 
 ## `shopify doc search`

packages/cli/oclif.manifest.json

@@ -6138,14 +6138,25 @@
       ],
       "args": {
       },
-      "description": "Download a complete document from shopify.dev. Every page on shopify.dev has a Markdown version, and that is what this tool returns. Use this to pull an entire document verbatim — for example, a set of instructions an agent follows like a centrally-served skill. Pass `--language` for the language of the app you are building so code examples match your stack. For finding the relevant pieces of content across shopify.dev instead, use `doc search`.",
+      "description": "Download a complete document from shopify.dev. Every page on shopify.dev has a Markdown version, and that is what this tool returns. Use this to pull an entire document verbatim — for example, a set of instructions an agent follows like a centrally-served skill. Pass `--language` for the language of the app you are building so code examples match your stack. For finding the relevant pieces of content across shopify.dev instead, use `doc search`.\n\nUse `--json-schema` to print the result, error, and event schemas.\n\nOutput from `--json` conforms to the `DocFetchResult` schema.\n\n```json\n{\n  \"anyOf\": [\n    {\n      \"type\": \"object\",\n      \"properties\": {\n        \"document\": {\n          \"$ref\": \"#/definitions/Document\"\n        }\n      },\n      \"required\": [\n        \"document\"\n      ],\n      \"additionalProperties\": false\n    },\n    {\n      \"$ref\": \"#/definitions/DocumentFile\"\n    }\n  ],\n  \"title\": \"DocFetchResult\",\n  \"definitions\": {\n    \"Document\": {\n      \"type\": \"object\",\n      \"properties\": {\n        \"url\": {\n          \"type\": \"string\",\n          \"format\": \"uri\",\n          \"description\": \"The requested shopify.dev document URL.\"\n        },\n        \"content\": {\n          \"type\": \"string\",\n          \"description\": \"The document in Markdown, with the requested language filter applied.\"\n        }\n      },\n      \"required\": [\n        \"url\",\n        \"content\"\n      ],\n      \"additionalProperties\": false\n    },\n    \"DocumentFile\": {\n      \"type\": \"object\",\n      \"properties\": {\n        \"path\": {\n          \"type\": \"string\",\n          \"pattern\": \"^(?:\\\\/|[A-Za-z]:[\\\\\\\\/]|\\\\\\\\\\\\\\\\)\",\n          \"description\": \"The absolute native path of the written file.\"\n        },\n        \"format\": {\n          \"type\": \"string\",\n          \"const\": \"markdown\",\n          \"description\": \"The file contains the original Markdown document, not a JSON wrapper.\"\n        }\n      },\n      \"required\": [\n        \"path\",\n        \"format\"\n      ],\n      \"additionalProperties\": false\n    }\n  },\n  \"$schema\": \"http://json-schema.org/draft-07/schema#\"\n}\n```",
+      "descriptionWithMarkdown": "Download a complete document from shopify.dev. Every page on shopify.dev has a Markdown version, and that is what this tool returns. Use this to pull an entire document verbatim — for example, a set of instructions an agent follows like a centrally-served skill. Pass `--language` for the language of the app you are building so code examples match your stack. For finding the relevant pieces of content across shopify.dev instead, use `doc search`.",
       "enableJsonFlag": false,
       "examples": [
         "# fetch the Markdown version of a Shopify.dev page\nshopify doc fetch --url https://shopify.dev/docs/api/shopify-cli",
         "# filter code examples to the language of the app you are building\nshopify doc fetch --url https://shopify.dev/docs/api/shopify-cli --language ruby",
-        "# save the document to a file instead of printing it\nshopify doc fetch --url https://shopify.dev/docs/api/shopify-cli --output docs/shopify-cli.md"
+        "# save the document to a file instead of printing it\nshopify doc fetch --url https://shopify.dev/docs/api/shopify-cli --output docs/shopify-cli.md",
+        "# return a typed document as JSON\nshopify doc fetch --url https://shopify.dev/docs/api/shopify-cli --json"
       ],
       "flags": {
+        "json": {
+          "allowNo": false,
+          "char": "j",
+          "description": "Output the result as JSON. Automatically disables color output.",
+          "env": "SHOPIFY_FLAG_JSON",
+          "hidden": false,
+          "name": "json",
+          "type": "boolean"
+        },
         "json-schema": {
           "allowNo": false,
           "description": "Print the command's JSON schemas.",
@@ -6189,7 +6200,7 @@
           "type": "boolean"
         },
         "output": {
-          "description": "Write the document to this file path instead of printing it to stdout.",
+          "description": "Write the document to this file path instead of printing it to stdout. With --json, stdout contains the absolute path and Markdown format of the written file.",
           "env": "SHOPIFY_FLAG_OUTPUT",
           "hasDynamicHelp": false,
           "multiple": false,

packages/cli/src/cli/commands/doc/fetch.test.ts

@@ -0,0 +1,165 @@
+import DocFetch from './fetch.js'
+import {docFetchJsonOutputSchema} from '../../services/commands/doc/types.js'
+import {fetch, Response} from '@shopify/cli-kit/node/http'
+import {inTemporaryDirectory, readFile, writeFile, fileExists} from '@shopify/cli-kit/node/fs'
+import {joinPath, relativePath, cwd} from '@shopify/cli-kit/node/path'
+import {launchCLI} from '@shopify/cli-kit/node/cli-launcher'
+import {ShopifyConfig} from '@shopify/cli-kit/node/custom-oclif-loader'
+import {mockAndCaptureOutput, withCapturedStandardStreams} from '@shopify/cli-kit/node/testing/output'
+import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest'
+
+vi.mock('@shopify/cli-kit/node/http', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@shopify/cli-kit/node/http')>()),
+  fetch: vi.fn(),
+}))
+
+const url = 'https://shopify.dev/docs/api/shopify-cli'
+const content = '# Shopify CLI\n\nA document.\n'
+
+beforeEach(() => {
+  vi.stubEnv('CI', '1')
+  vi.stubEnv('SHOPIFY_CLI_NO_ANALYTICS', '1')
+  vi.mocked(fetch).mockResolvedValue(new Response(content))
+  vi.spyOn(process, 'exit').mockReturnValue(undefined as never)
+})
+
+afterEach(() => {
+  vi.unstubAllEnvs()
+  mockAndCaptureOutput().clear()
+})
+
+describe('doc fetch command', () => {
+  test('preserves the Markdown output and stdout channel', async () => {
+    await withCapturedStandardStreams(async ({stdout, stderr}) => {
+      await DocFetch.run(['--url', url, '--no-input'], import.meta.url)
+      expect(stdout()).toBe(`${content}\n`)
+      expect(stderr()).toBe('')
+    })
+  })
+
+  test.each([{flags: []}, {flags: ['--no-input']}])(
+    'writes one document object with JSON and flags $flags',
+    async ({flags}) => {
+      await withCapturedStandardStreams(async ({stdout, stderr}) => {
+        await DocFetch.run(['--url', url, '--language', 'ruby', '--json', ...flags], import.meta.url)
+        expect(JSON.parse(stdout())).toEqual({document: {url, content}})
+        expect(stderr()).toBe('')
+        expect(fetch).toHaveBeenCalledWith(url, {
+          headers: {Accept: 'text/markdown', 'X-Shopify-Surface': 'cli', 'Accept-Language': 'ruby'},
+        })
+      })
+    },
+  )
+
+  test('supports the shared JSON environment flag', async () => {
+    vi.stubEnv('SHOPIFY_FLAG_JSON', '1')
+    await withCapturedStandardStreams(async ({stdout, stderr}) => {
+      await DocFetch.run(['--url', url], import.meta.url)
+      expect(JSON.parse(stdout())).toEqual({document: {url, content}})
+      expect(stderr()).toBe('')
+    })
+  })
+
+  test('reports missing required input before fetching', async () => {
+    vi.stubEnv('SHOPIFY_FLAG_JSON', '1')
+    await withCapturedStandardStreams(async ({stdout, stderr}) => {
+      await DocFetch.run(['--json', '--no-input'], import.meta.url)
+      expect(JSON.parse(stdout())).toHaveProperty('error')
+      expect(process.exit).toHaveBeenCalledWith(2)
+      expect(stderr()).toBe('')
+      expect(fetch).not.toHaveBeenCalled()
+    })
+  })
+
+  test.each(['text', 'json'] as const)('saves exact Markdown bytes with a relative path in %s mode', async (format) => {
+    await inTemporaryDirectory(async (directory) => {
+      const path = joinPath(directory, 'docs/shopify-cli.md')
+      await withCapturedStandardStreams(async ({stdout, stderr}) => {
+        await DocFetch.run(
+          ['--url', url, '--output', relativePath(cwd(), path), ...(format === 'json' ? ['--json'] : [])],
+          import.meta.url,
+        )
+
+        await expect(readFile(path)).resolves.toBe(content)
+        if (format === 'json') {
+          expect(JSON.parse(stdout())).toEqual({path, format: 'markdown'})
+          expect(JSON.parse(stderr())).toMatchObject({
+            type: 'diagnostic',
+            level: 'info',
+            message: `Saved ${url} to ${path}`,
+          })
+        } else {
+          expect(stdout()).toBe('')
+          expect(stderr()).toBe(`Saved ${url} to ${path}\n`)
+        }
+      })
+    })
+  })
+
+  test('does not emit a receipt when the file write fails', async () => {
+    vi.stubEnv('SHOPIFY_FLAG_JSON', '1')
+    await inTemporaryDirectory(async (directory) => {
+      const parent = joinPath(directory, 'file')
+      await writeFile(parent, 'existing content')
+      await withCapturedStandardStreams(async ({stdout, stderr}) => {
+        await expect(
+          DocFetch.run(['--url', url, '--output', joinPath(parent, 'doc.md'), '--json'], import.meta.url),
+        ).resolves.toBeUndefined()
+        expect(process.exit).toHaveBeenCalledWith(1)
+        expect(JSON.parse(stdout())).toHaveProperty('error')
+        expect(stdout()).not.toContain('"format": "markdown"')
+        expect(stderr()).toBe('')
+        await expect(readFile(parent)).resolves.toBe('existing content')
+      })
+    })
+  })
+
+  test('does not create a file when fetching fails', async () => {
+    vi.stubEnv('SHOPIFY_FLAG_JSON', '1')
+    vi.mocked(fetch).mockResolvedValue(new Response('', {status: 404, statusText: 'Not Found'}))
+    await inTemporaryDirectory(async (directory) => {
+      const path = joinPath(directory, 'doc.md')
+      await withCapturedStandardStreams(async ({stdout, stderr}) => {
+        await expect(DocFetch.run(['--url', url, '--output', path, '--json'], import.meta.url)).resolves.toBeUndefined()
+        expect(process.exit).toHaveBeenCalledWith(1)
+        expect(JSON.parse(stdout())).toEqual({error: {type: 'abort', message: `Failed to fetch ${url}: 404 Not Found`}})
+        expect(stderr()).toBe('')
+        await expect(fileExists(path)).resolves.toBe(false)
+      })
+    })
+  })
+
+  test.each(['not a url', 'https://example.com/docs'])(
+    'reports invalid input as one fatal document: %s',
+    async (input) => {
+      vi.stubEnv('SHOPIFY_FLAG_JSON', '1')
+      await withCapturedStandardStreams(async ({stdout, stderr}) => {
+        await expect(DocFetch.run(['--url', input, '--json'], import.meta.url)).resolves.toBeUndefined()
+        expect(process.exit).toHaveBeenCalledWith(1)
+        expect(JSON.parse(stdout()).error.type).toBe('abort')
+        expect(stderr()).toBe('')
+        expect(fetch).not.toHaveBeenCalled()
+      })
+    },
+  )
+
+  test('exposes the schema in help and through the launcher without fetching', async () => {
+    expect(DocFetch.jsonOutputSchema).toBe(docFetchJsonOutputSchema)
+    expect(DocFetch.flags.json).toBeDefined()
+    expect(DocFetch.description).toContain('Output from `--json` conforms to the `DocFetchResult` schema.')
+    vi.spyOn(ShopifyConfig.prototype, 'runHook').mockResolvedValue({successes: [], failures: []})
+    await withCapturedStandardStreams(async ({stdout, stderr}) => {
+      await launchCLI({
+        moduleURL: import.meta.url,
+        argv: ['doc', 'fetch', '--json-schema'],
+        lazyCommandLoader: async () => DocFetch,
+      })
+      const schema = JSON.parse(stdout())
+      expect(schema.definitions.Result.anyOf).toHaveLength(2)
+      expect(schema.definitions.Result.anyOf[0].properties.document.properties.content.type).toBe('string')
+      expect(schema.definitions.Result.anyOf[1].properties.format.const).toBe('markdown')
+      expect(stderr()).toBe('')
+      expect(fetch).not.toHaveBeenCalled()
+    })
+  })
+})