Add typed JSON output to theme check

Author: gonzaloriestra

docs-shopify.dev/generated/generated_docs_data_v2.json

@@ -6513,6 +6513,15 @@
           "isOptional": true,
           "environmentValue": "SHOPIFY_FLAG_ENVIRONMENT"
         },
+        {
+          "filePath": "docs-shopify.dev/commands/interfaces/theme-check.interface.ts",
+          "syntaxKind": "PropertySignature",
+          "name": "-j, --json",
+          "value": "''",
+          "description": "Output the result as JSON. Automatically disables color output.",
+          "isOptional": true,
+          "environmentValue": "SHOPIFY_FLAG_JSON"
+        },
         {
           "filePath": "docs-shopify.dev/commands/interfaces/theme-check.interface.ts",
           "syntaxKind": "PropertySignature",
@@ -6532,7 +6541,7 @@
           "environmentValue": "SHOPIFY_FLAG_VERSION"
         }
       ],
-      "value": "export interface themecheck {\n  /**\n   * Alias of the Shopify account to use for authentication.\n   * @environment SHOPIFY_FLAG_AUTH_ALIAS\n   */\n  '--auth-alias <value>'?: string\n\n  /**\n   * Automatically fix offenses\n   * @environment SHOPIFY_FLAG_AUTO_CORRECT\n   */\n  '-a, --auto-correct'?: ''\n\n  /**\n   * Use the config provided, overriding .theme-check.yml if present\n      Supports all theme-check: config values, e.g., theme-check:theme-app-extension,\n      theme-check:recommended, theme-check:all\n      For backwards compatibility, :theme_app_extension is also supported \n   * @environment SHOPIFY_FLAG_CONFIG\n   */\n  '-C, --config <value>'?: string\n\n  /**\n   * The environment to apply to the current command.\n   * @environment SHOPIFY_FLAG_ENVIRONMENT\n   */\n  '-e, --environment <value>'?: string\n\n  /**\n   * Minimum severity for exit with error code\n   * @environment SHOPIFY_FLAG_FAIL_LEVEL\n   */\n  '--fail-level <value>'?: string\n\n  /**\n   * Generate a .theme-check.yml file\n   * @environment SHOPIFY_FLAG_INIT\n   */\n  '--init'?: ''\n\n  /**\n   * Print the command's JSON schemas.\n   * @environment SHOPIFY_FLAG_JSON_SCHEMA\n   */\n  '--json-schema'?: ''\n\n  /**\n   * List enabled checks\n   * @environment SHOPIFY_FLAG_LIST\n   */\n  '--list'?: ''\n\n  /**\n   * Disable color output.\n   * @environment SHOPIFY_FLAG_NO_COLOR\n   */\n  '--no-color'?: ''\n\n  /**\n   * The output format to use\n   * @environment SHOPIFY_FLAG_OUTPUT\n   */\n  '-o, --output <value>'?: string\n\n  /**\n   * The path where you want to run the command. Defaults to the current working directory.\n   * @environment SHOPIFY_FLAG_PATH\n   */\n  '--path <value>'?: string\n\n  /**\n   * Output active config to STDOUT\n   * @environment SHOPIFY_FLAG_PRINT\n   */\n  '--print'?: ''\n\n  /**\n   * Increase the verbosity of the output. May include sensitive data.\n   * @environment SHOPIFY_FLAG_VERBOSE\n   */\n  '--verbose'?: ''\n\n  /**\n   * Print Theme Check version\n   * @environment SHOPIFY_FLAG_VERSION\n   */\n  '-v, --version'?: ''\n}"
+      "value": "export interface themecheck {\n  /**\n   * Alias of the Shopify account to use for authentication.\n   * @environment SHOPIFY_FLAG_AUTH_ALIAS\n   */\n  '--auth-alias <value>'?: string\n\n  /**\n   * Automatically fix offenses\n   * @environment SHOPIFY_FLAG_AUTO_CORRECT\n   */\n  '-a, --auto-correct'?: ''\n\n  /**\n   * Use the config provided, overriding .theme-check.yml if present\n      Supports all theme-check: config values, e.g., theme-check:theme-app-extension,\n      theme-check:recommended, theme-check:all\n      For backwards compatibility, :theme_app_extension is also supported \n   * @environment SHOPIFY_FLAG_CONFIG\n   */\n  '-C, --config <value>'?: string\n\n  /**\n   * The environment to apply to the current command.\n   * @environment SHOPIFY_FLAG_ENVIRONMENT\n   */\n  '-e, --environment <value>'?: string\n\n  /**\n   * Minimum severity for exit with error code\n   * @environment SHOPIFY_FLAG_FAIL_LEVEL\n   */\n  '--fail-level <value>'?: string\n\n  /**\n   * Generate a .theme-check.yml file\n   * @environment SHOPIFY_FLAG_INIT\n   */\n  '--init'?: ''\n\n  /**\n   * Output the result as JSON. Automatically disables color output.\n   * @environment SHOPIFY_FLAG_JSON\n   */\n  '-j, --json'?: ''\n\n  /**\n   * Print the command's JSON schemas.\n   * @environment SHOPIFY_FLAG_JSON_SCHEMA\n   */\n  '--json-schema'?: ''\n\n  /**\n   * List enabled checks\n   * @environment SHOPIFY_FLAG_LIST\n   */\n  '--list'?: ''\n\n  /**\n   * Disable color output.\n   * @environment SHOPIFY_FLAG_NO_COLOR\n   */\n  '--no-color'?: ''\n\n  /**\n   * The output format to use\n   * @environment SHOPIFY_FLAG_OUTPUT\n   */\n  '-o, --output <value>'?: string\n\n  /**\n   * The path where you want to run the command. Defaults to the current working directory.\n   * @environment SHOPIFY_FLAG_PATH\n   */\n  '--path <value>'?: string\n\n  /**\n   * Output active config to STDOUT\n   * @environment SHOPIFY_FLAG_PRINT\n   */\n  '--print'?: ''\n\n  /**\n   * Increase the verbosity of the output. May include sensitive data.\n   * @environment SHOPIFY_FLAG_VERBOSE\n   */\n  '--verbose'?: ''\n\n  /**\n   * Print Theme Check version\n   * @environment SHOPIFY_FLAG_VERSION\n   */\n  '-v, --version'?: ''\n}"
     }
   },
   "themeconsole": {

packages/cli/README.md

@@ -7070,8 +7070,8 @@ Validate the theme.
 ```
 USAGE
   $ shopify theme check [--auth-alias <value>] [-a] [-C <value>] [-e <value>...] [--fail-level
-    crash|error|suggestion|style|warning|info] [--init] [--json-schema] [--list] [--no-color] [-o text|json] [--path
-    <value>] [--print] [--verbose] [-v]
+    crash|error|suggestion|style|warning|info] [-j | --init | -v | --print | --list] [--json-schema] [--no-color] [-o
+    text|json] [--path <value>] [--verbose]
 
 FLAGS
   -C, --config=<value>
@@ -7089,6 +7089,10 @@ FLAGS
       The environment to apply to the current command.
       [env: SHOPIFY_FLAG_ENVIRONMENT]
 
+  -j, --json
+      Output the result as JSON. Automatically disables color output.
+      [env: SHOPIFY_FLAG_JSON]
+
   -o, --output=<option>
       [default: text] The output format to use
       [env: SHOPIFY_FLAG_OUTPUT]
@@ -7141,6 +7145,133 @@ DESCRIPTION
   Calls and runs "Theme Check" (https://shopify.dev/docs/themes/tools/theme-check) to analyze your theme code for errors
   and to ensure that it follows theme and Liquid best practices. "Learn more about the checks that Theme Check runs."
   (https://shopify.dev/docs/themes/tools/theme-check/checks)
+
+  Output from `--json` conforms to the `ThemeCheckResult` schema.
+
+  Use `--json-schema` to print the result, error, and event schemas.
+
+  ```json
+  {
+    "anyOf": [
+      {
+        "$ref": "#/definitions/ThemeCheckEnvironment/properties/result"
+      },
+      {
+        "type": "object",
+        "properties": {
+          "environments": {
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/ThemeCheckEnvironment"
+            }
+          }
+        },
+        "required": [
+          "environments"
+        ],
+        "additionalProperties": false
+      }
+    ],
+    "title": "ThemeCheckResult",
+    "definitions": {
+      "ThemeCheckOffense": {
+        "type": "object",
+        "properties": {
+          "check": {
+            "type": "string"
+          },
+          "severity": {
+            "type": "string",
+            "enum": [
+              "error",
+              "warning",
+              "info"
+            ]
+          },
+          "start_row": {
+            "type": "number"
+          },
+          "start_column": {
+            "type": "number"
+          },
+          "end_row": {
+            "type": "number"
+          },
+          "end_column": {
+            "type": "number"
+          },
+          "message": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "check",
+          "severity",
+          "start_row",
+          "start_column",
+          "end_row",
+          "end_column",
+          "message"
+        ],
+        "additionalProperties": false
+      },
+      "ThemeCheckFile": {
+        "type": "object",
+        "properties": {
+          "environment": {
+            "type": "string"
+          },
+          "path": {
+            "type": "string"
+          },
+          "offenses": {
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/ThemeCheckOffense"
+            }
+          },
+          "errorCount": {
+            "type": "number"
+          },
+          "warningCount": {
+            "type": "number"
+          },
+          "infoCount": {
+            "type": "number"
+          }
+        },
+        "required": [
+          "path",
+          "offenses",
+          "errorCount",
+          "warningCount",
+          "infoCount"
+        ],
+        "additionalProperties": false
+      },
+      "ThemeCheckEnvironment": {
+        "type": "object",
+        "properties": {
+          "environment": {
+            "type": "string"
+          },
+          "result": {
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/ThemeCheckFile"
+            }
+          }
+        },
+        "required": [
+          "environment",
+          "result"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "$schema": "http://json-schema.org/draft-07/schema#"
+  }
+  ```
 ```
 
 ## `shopify theme console`

packages/cli/oclif.manifest.json

@@ -9350,7 +9350,7 @@
       "args": {
       },
       "customPluginName": "@shopify/theme",
-      "description": "Calls and runs \"Theme Check\" (https://shopify.dev/docs/themes/tools/theme-check) to analyze your theme code for errors and to ensure that it follows theme and Liquid best practices. \"Learn more about the checks that Theme Check runs.\" (https://shopify.dev/docs/themes/tools/theme-check/checks)",
+      "description": "Calls and runs \"Theme Check\" (https://shopify.dev/docs/themes/tools/theme-check) to analyze your theme code for errors and to ensure that it follows theme and Liquid best practices. \"Learn more about the checks that Theme Check runs.\" (https://shopify.dev/docs/themes/tools/theme-check/checks)\n\nOutput from `--json` conforms to the `ThemeCheckResult` schema.\n\nUse `--json-schema` to print the result, error, and event schemas.\n\n```json\n{\n  \"anyOf\": [\n    {\n      \"$ref\": \"#/definitions/ThemeCheckEnvironment/properties/result\"\n    },\n    {\n      \"type\": \"object\",\n      \"properties\": {\n        \"environments\": {\n          \"type\": \"array\",\n          \"items\": {\n            \"$ref\": \"#/definitions/ThemeCheckEnvironment\"\n          }\n        }\n      },\n      \"required\": [\n        \"environments\"\n      ],\n      \"additionalProperties\": false\n    }\n  ],\n  \"title\": \"ThemeCheckResult\",\n  \"definitions\": {\n    \"ThemeCheckOffense\": {\n      \"type\": \"object\",\n      \"properties\": {\n        \"check\": {\n          \"type\": \"string\"\n        },\n        \"severity\": {\n          \"type\": \"string\",\n          \"enum\": [\n            \"error\",\n            \"warning\",\n            \"info\"\n          ]\n        },\n        \"start_row\": {\n          \"type\": \"number\"\n        },\n        \"start_column\": {\n          \"type\": \"number\"\n        },\n        \"end_row\": {\n          \"type\": \"number\"\n        },\n        \"end_column\": {\n          \"type\": \"number\"\n        },\n        \"message\": {\n          \"type\": \"string\"\n        }\n      },\n      \"required\": [\n        \"check\",\n        \"severity\",\n        \"start_row\",\n        \"start_column\",\n        \"end_row\",\n        \"end_column\",\n        \"message\"\n      ],\n      \"additionalProperties\": false\n    },\n    \"ThemeCheckFile\": {\n      \"type\": \"object\",\n      \"properties\": {\n        \"environment\": {\n          \"type\": \"string\"\n        },\n        \"path\": {\n          \"type\": \"string\"\n        },\n        \"offenses\": {\n          \"type\": \"array\",\n          \"items\": {\n            \"$ref\": \"#/definitions/ThemeCheckOffense\"\n          }\n        },\n        \"errorCount\": {\n          \"type\": \"number\"\n        },\n        \"warningCount\": {\n          \"type\": \"number\"\n        },\n        \"infoCount\": {\n          \"type\": \"number\"\n        }\n      },\n      \"required\": [\n        \"path\",\n        \"offenses\",\n        \"errorCount\",\n        \"warningCount\",\n        \"infoCount\"\n      ],\n      \"additionalProperties\": false\n    },\n    \"ThemeCheckEnvironment\": {\n      \"type\": \"object\",\n      \"properties\": {\n        \"environment\": {\n          \"type\": \"string\"\n        },\n        \"result\": {\n          \"type\": \"array\",\n          \"items\": {\n            \"$ref\": \"#/definitions/ThemeCheckFile\"\n          }\n        }\n      },\n      \"required\": [\n        \"environment\",\n        \"result\"\n      ],\n      \"additionalProperties\": false\n    }\n  },\n  \"$schema\": \"http://json-schema.org/draft-07/schema#\"\n}\n```",
       "descriptionWithMarkdown": "Calls and runs [Theme Check](https://shopify.dev/docs/themes/tools/theme-check) to analyze your theme code for errors and to ensure that it follows theme and Liquid best practices. [Learn more about the checks that Theme Check runs.](https://shopify.dev/docs/themes/tools/theme-check/checks)",
       "enableJsonFlag": false,
       "flags": {
@@ -9416,6 +9416,21 @@
           "required": false,
           "type": "boolean"
         },
+        "json": {
+          "allowNo": false,
+          "char": "j",
+          "description": "Output the result as JSON. Automatically disables color output.",
+          "env": "SHOPIFY_FLAG_JSON",
+          "exclusive": [
+            "init",
+            "version",
+            "print",
+            "list"
+          ],
+          "hidden": false,
+          "name": "json",
+          "type": "boolean"
+        },
         "json-schema": {
           "allowNo": false,
           "description": "Print the command's JSON schemas.",

packages/eslint-plugin-cli/rules/json-output-command-exceptions.js

@@ -60,7 +60,6 @@ const commandExceptions = [
   'packages/plugin-did-you-mean/src/commands/config/autocorrect/off.ts',
   'packages/plugin-did-you-mean/src/commands/config/autocorrect/on.ts',
   'packages/plugin-did-you-mean/src/commands/config/autocorrect/status.ts',
-  'packages/theme/src/cli/commands/theme/check.ts',
   'packages/theme/src/cli/commands/theme/init.ts',
   'packages/theme/src/cli/commands/theme/metafields/pull.ts',
   'packages/theme/src/cli/commands/theme/package.ts',

packages/theme/src/cli/commands/theme/check-json.test.ts

@@ -0,0 +1,128 @@
+import Check from './check.js'
+import {checkTheme, formatOffensesJson, sortOffenses} from '../../services/check.js'
+import {themeCheckJsonOutputSchema} from '../../services/check/types.js'
+import {encodeThemeCheckResult} from '../../services/check/result.js'
+import {captureStandardStreams} from '../../utilities/testing/streams.js'
+import {afterEach, expect, test, vi} from 'vitest'
+import {themeCheckRun, Severity, SourceCodeType, path as pathUtils} from '@shopify/theme-check-node'
+import {inTemporaryDirectory} from '@shopify/cli-kit/node/fs'
+
+vi.mock('@shopify/theme-check-node', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@shopify/theme-check-node')>()),
+  themeCheckRun: vi.fn(),
+}))
+
+vi.mock('@shopify/cli-kit/node/environments')
+
+const offense = {
+  type: SourceCodeType.LiquidHtml,
+  check: 'ExampleCheck',
+  severity: Severity.ERROR,
+  uri: 'file:///theme/templates/index.liquid',
+  start: {index: 0, line: 0, character: 0},
+  end: {index: 1, line: 0, character: 1},
+  message: 'Example offense',
+}
+
+afterEach(() => vi.unstubAllEnvs())
+
+test('exposes the contract and preserves compact JSON, environment omission and counts', async () => {
+  expect(Check.jsonOutputSchema).toBe(themeCheckJsonOutputSchema)
+  vi.mocked(themeCheckRun).mockResolvedValue({offenses: [offense], theme: [], config: {} as never})
+  const {result} = await checkTheme('/theme')
+  expect(result).toEqual(formatOffensesJson(sortOffenses([offense])))
+  expect(encodeThemeCheckResult(result)).toBe(
+    JSON.stringify([
+      {
+        path: pathUtils.fsPath(offense.uri),
+        offenses: [
+          {
+            check: 'ExampleCheck',
+            severity: 'error',
+            start_row: 0,
+            start_column: 0,
+            end_row: 0,
+            end_column: 1,
+            message: 'Example offense',
+          },
+        ],
+        errorCount: 1,
+        warningCount: 0,
+        infoCount: 0,
+      },
+    ]),
+  )
+  expect(encodeThemeCheckResult([])).toBe('[]')
+  expect(() => themeCheckJsonOutputSchema.validate([{...result[0], errorCount: '1'}])).toThrow()
+  expect(() =>
+    themeCheckJsonOutputSchema.validate([{...result[0], offenses: [{...result[0]!.offenses[0], severity: 'fatal'}]}]),
+  ).toThrow()
+})
+
+test.each(['--json', '--output=json'])('writes compact JSON and preserves unsuccessful exit with %s', async (flag) => {
+  await inTemporaryDirectory(async (directory) => {
+    vi.stubEnv('SHOPIFY_UNIT_TEST', 'false')
+    vi.resetModules()
+    const {default: StreamCheck} = await import('./check.js')
+    const {themeCheckRun: check} = await import('@shopify/theme-check-node')
+    const {Config} = await import('@oclif/core')
+    const config = new Config({root: __dirname})
+    await config.load()
+    vi.mocked(check).mockResolvedValue({offenses: [offense], theme: [], config: {} as never})
+    const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never)
+    const streams = captureStandardStreams()
+    try {
+      await new StreamCheck([`--path=${directory}`, flag], config).run()
+    } finally {
+      streams.restore()
+    }
+    expect(streams.stdout()).toBe(`${JSON.stringify(formatOffensesJson(sortOffenses([offense])))}\n`)
+    expect(streams.stderr()).toBe('')
+    expect(exit).toHaveBeenCalledWith(1)
+  })
+})
+
+test.each(['init', 'version', 'print', 'list'])('rejects --json with --%s before execution', async (mode) => {
+  const {Config} = await import('@oclif/core')
+  const config = new Config({root: __dirname})
+  await config.load()
+  await expect(new Check(['--json', `--${mode}`], config).run()).rejects.toThrow()
+  expect(themeCheckRun).not.toHaveBeenCalled()
+})
+
+test.each(['success', 'partial failure', 'total failure'])(
+  'collects multiple environments into one document on %s',
+  async (mode) => {
+    await inTemporaryDirectory(async (directory) => {
+      vi.stubEnv('SHOPIFY_UNIT_TEST', 'false')
+      vi.resetModules()
+      const {default: StreamCheck} = await import('./check.js')
+      const {themeCheckRun: check} = await import('@shopify/theme-check-node')
+      const {Config} = await import('@oclif/core')
+      const config = new Config({root: __dirname})
+      await config.load()
+      vi.mocked(check).mockResolvedValue({offenses: [], theme: [], config: {} as never})
+      if (mode === 'partial failure') vi.mocked(check).mockRejectedValueOnce(new Error('Check failed'))
+      if (mode === 'total failure') vi.mocked(check).mockRejectedValue(new Error('Check failed'))
+      const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never)
+      const {loadEnvironment} = await import('@shopify/cli-kit/node/environments')
+      vi.mocked(loadEnvironment).mockResolvedValue({path: directory})
+      const {runWithCommandEventsForCommand} = await import('@shopify/cli-kit/node/command-events')
+      const streams = captureStandardStreams()
+      try {
+        const argv = ['--environment=first', '--environment=second', '--json']
+        await runWithCommandEventsForCommand(argv, () => new StreamCheck(argv, config).run())
+      } finally {
+        streams.restore()
+      }
+      let names = ['first', 'second']
+      if (mode === 'partial failure') names = ['second']
+      if (mode === 'total failure') names = []
+      expect(JSON.parse(streams.stdout())).toEqual({
+        environments: names.map((environment) => ({environment, result: []})),
+      })
+      if (mode.includes('failure')) expect(streams.stderr()).toContain('Check failed')
+      expect(exit).not.toHaveBeenCalled()
+    })
+  },
+)