From fd9747aeb6bf13e9078be5f7ce9acf8e59923113 Mon Sep 17 00:00:00 2001 From: Devin McIntyre Date: Wed, 16 Oct 2024 13:48:38 -0600 Subject: [PATCH 1/5] fix(test-runner-core): allow advanced istanbul report configurations --- .../docs/test-runner/cli-and-configuration.md | 21 +- .../writing-tests/code-coverage.md | 253 +++++++++++++++++- .../src/cli/writeCoverageReport.ts | 12 +- .../src/config/TestRunnerCoreConfig.ts | 6 +- 4 files changed, 267 insertions(+), 25 deletions(-) diff --git a/docs/docs/test-runner/cli-and-configuration.md b/docs/docs/test-runner/cli-and-configuration.md index 7e718f000..0d96b80f0 100644 --- a/docs/docs/test-runner/cli-and-configuration.md +++ b/docs/docs/test-runner/cli-and-configuration.md @@ -83,13 +83,15 @@ A configuration file accepts most of the command line args camel-cased, with som ```ts import { Plugin, Middleware } from '@web/dev-server'; -import { ReportType } from 'istanbul-reports'; +import { ReportType, ReportOptions } from 'istanbul-reports'; +import { Watermarks, Summarizers } from 'istanbul-lib-report'; interface TestFramework { path: string; config?: unknown; } +// the threshold to reach for each code-coverage metric interface CoverageThresholdConfig { statements: number; branches: number; @@ -98,12 +100,25 @@ interface CoverageThresholdConfig { } interface CoverageConfig { + // globs of files to include in code coverage measurements include?: string[]; + // globs of files to exclude from code coverage measurements exclude?: string[]; + // set to false to turn off native instrumentation, such as when using the babel plugin + nativeInstrumentation?: boolean; + // the threshold to reach for each code-coverage metric threshold?: CoverageThresholdConfig; - report: boolean; - reportDir: string; + report?: boolean; + // the output directory for coverage reports + reportDir?: string; + // which reporters to use for coverage reports reporters?: ReportType[]; + // the display low-water mark and high-water mark for each code-coverage metric + watermarks?: Partial; + // additional options for particular reporters, if needed + reportOptions?: ReportOptions; + // output structure for particular reporters + defaultSummarizer?: Summarizers; } type MimeTypeMappings = Record; diff --git a/docs/docs/test-runner/writing-tests/code-coverage.md b/docs/docs/test-runner/writing-tests/code-coverage.md index 79eb779cf..f60f03ee3 100644 --- a/docs/docs/test-runner/writing-tests/code-coverage.md +++ b/docs/docs/test-runner/writing-tests/code-coverage.md @@ -6,17 +6,25 @@ You can run tests with code coverage using the `--coverage` flag: wtr test/**/*.test.js --coverage ``` -In the config you can define code coverage thresholds, the test run fails if you drop below this level. You can also configure where and if the detailed test report is written to disk. +Additionally, your config file has a `coverage` boolean to toggle coverage on and off. -**Example config:** +## Basic configuration + +Code coverage is measured using four metrics: statements, branches, functions, and lines. +The test runner measures these metrics for each file. + +In `coverageConfig`, you need to define your desired code coverage thresholds for each metric. +The test run fails if you drop below this level for any metric in any file. +The default setting is 0 for each metric. + +**Example minimum to require code coverage:** ```js // web-test-runner.config.mjs export default { + coverage: true, coverageConfig: { - report: true, - reportDir: 'test-coverage', threshold: { statements: 70, branches: 70, @@ -27,15 +35,46 @@ export default { }; ``` -## Ignoring uncovered lines +## Including and excluding code + +Use the `include` and `exclude` settings in `coverageConfig` to control which files' code coverage measurements count toward your total. +Each is an array of glob patterns. By default, all touched files are included unless they match an exclude pattern. + +The default exclude patterns are `['**/node_modules/**/*', '**/web_modules/**/*']`. Additionally, files following your +test-file patterns are excluded. + +**Example include and exclude:** + +```js +// web-test-runner.config.mjs + +export default { + files: ['tests/**'], // these are excluded also + coverage: true, + coverageConfig: { + include: ['ui/**', 'lib/**'], + exclude: ['lib/**.cjs', 'ui/untestable-file.js'], + threshold: { + statements: 70, + branches: 70, + functions: 70, + lines: 70, + }, + }, +}; +``` + +### Ignoring uncovered lines -Web Test Runner uses [`v8-to-istanbul`](https://github.com/istanbuljs/v8-to-istanbul) to covert V8 based code coverage to a form that can be reported by Istanbul. `v8-to-istanbul` allows for ignoring uncovered lines when calculating code coverage through the use of the following custom comment: +Web Test Runner uses [`v8-to-istanbul`](https://github.com/istanbuljs/v8-to-istanbul) to convert V8-based code coverage to a form that can be reported by Istanbul. +`v8-to-istanbul` allows for ignoring uncovered lines when calculating code coverage through the use of the following custom comment: ```js /* c8 ignore next [line count] */ ``` -This is somewhat different than other tools where you might have specifically targeted `if` / `else` branches of logic with an ignore statement. Particularly, V8 does not create phantom `else` statements when calculating coverage, so it is likely that you will be able to use less of these statements than in the past. +This is somewhat different from other tools in which you might have specifically targeted `if` / `else` branches of logic with an ignore statement. +Particularly, V8 does not create phantom `else` statements when calculating coverage, so it is likely that you will be able to use fewer of these statements than in the past. In this way, you can skip the rest of a line: @@ -63,13 +102,16 @@ if (normalCase) { ## Coverage browser support -The default coverage of the test runner uses the ability of Chromium to do native code coverage instrumentation. This gives us the best speed. When testing multiple browsers this should still be fine, you don't need to get code coverage from all browsers. One browser is usually enough. +The default coverage of the test runner uses the ability of Chromium to do native code coverage instrumentation. +This gives us the best speed. When testing multiple browsers this should still be fine, you don't need to get code coverage from all browsers. +One browser is usually enough. -If you need to collect coverage from all browsers, or if you're not testing for Chromium at all, you can use [babel-plugin-istanbul](https://github.com/istanbuljs/babel-plugin-istanbul) to do the code instrumentation. You can use the rollup babel plugin to set this up. This approach is slower and works differently because the instrumentation is done in javascript. +If you need to collect coverage from all browsers, or if you're not testing for Chromium at all, you can use [babel-plugin-istanbul](https://github.com/istanbuljs/babel-plugin-istanbul) to do the code instrumentation. +You can use the rollup babel plugin to set this up. This approach is slower and works differently because the instrumentation is done in javascript. If you choose to use the babel plugin, you can turn off native instrumentation by setting `nativeInstrumentation` to false. This avoids double instrumentation. -**Example config:** +**Example with babel:** ```js // web-test-runner.config.mjs @@ -97,24 +139,207 @@ export default { ## Coverage reporting -By default coverage reporting uses the lcov reporter. Should you want to use additional reporters, for example, cobertura, then the `reporter` config element should be modified. +By default, the test runner's coverage reporting uses the `lcov` reporter. Use the `reporters` configuration to select others. +You can choose from the reporters listed in [istanbul-reports/lib](https://github.com/istanbuljs/istanbuljs/tree/main/packages/istanbul-reports/lib). +Previews for some popular ones are at [Alternative Reporters](https://istanbul.js.org/docs/advanced/alternative-reporters/). + +Use the `reportDir` setting to choose where reports are written. +The default setting is `coverage`. + +**Example with cobertura and lcov:** + +```js +// web-test-runner.config.mjs + +export default { + coverage: true, + coverageConfig: { + reportDir: 'test-coverage', + reporters: ['cobertura', 'lcov'], + threshold: { + statements: 70, + branches: 70, + functions: 70, + lines: 70, + }, + }, +}; +``` + +### Code coverage without reports + +You can turn off reports but otherwise require code coverage using the `none` reporter. + +**Example without reports:** + +```js +// web-test-runner.config.mjs + +export default { + coverage: true, + coverageConfig: { + reporters: ['none'], + threshold: { + statements: 70, + branches: 70, + functions: 70, + lines: 70, + }, + }, +}; +``` + +### Options for each reporter + +You can provide additional options for many reporters, such as their output file name. +The available options for each are listed in [@types/istanbul-reports](https://www.npmjs.com/package/@types/istanbul-reports), +but there is little documentation for what each setting does. +Provide `reportOptions` items separately for each reporter. + +In this example, the reports directory will contain both 'cobertura.xml' and an 'html-output' directory containing the HTML report. + +**Example with cobertura and html-spa full options:** + +```js +// web-test-runner.config.mjs + +export default { + coverage: true, + coverageConfig: { + reportDir: 'reports', + reporters: ['html-spa', 'cobertura'], + reportOptions: { + cobertura: { + file: 'cobertura.xml', + }, + 'html-spa': { + verbose: true, + skipEmpty: true, + subdir: 'html-output', + linkMapper: { + getPath: node => myPathTransformFunction(node), + relativePath: node => myRelativePathTransformFunction(node), + assetPath: node => myAssetPathTransformFunction(node), + }, + metricsToShow: ['lines', 'branches', 'functions', 'statements'], + }, + }, + }, +}; +``` + +### Report color-coding + +Some reporters show red/yellow/green color-coding, which is adjustable using the `watermarks` configuration. +Coverage below the lower mark displays in red, coverage between the marks displays in yellow, +and coverage above the upper mark displays in green. The default marks are `[50, 80]` for each metric. + +Affected reporters: + +- html +- html-spa +- lcov +- text +- text-summary + +In this example, the displayed `lcov` HTML report will show yellow for files that almost reach each threshold, +red for files that are far below each threshold, and green for files that are above the threshold. + +**Example color-coding that matches thresholds:** + +```js +// web-test-runner.config.mjs + +export default { + coverage: true, + coverageConfig: { + reporters: ['lcov'], + threshold: { + statements: 90, + branches: 50, + functions: 80, + lines: 90, + }, + watermarks: { + statements: [85, 90], + branches: [45, 50], + functions: [75, 80], + lines: [85, 90], + }, + }, +}; +``` + +### Report structure + +For some reporters, you can change the summary and hierarchy of reported files using the option `defaultSummarizer`. + +- `pkg` + - The default setting + - Lists every subdirectory path on the main index page + - Shows the summary of the files in each subdirectory + - You must click into each subdirectory to see files +- `nested`: + - Displays nested file structure, like in your operating system or IDE + - Shows a deep summary of all the contents of each directory + - You must click through the directory structure to each file +- `flat`: + - All files are in one flat list + - No directory-based summaries + - There is only one page of results + +**Example HTML report structures:** + +_Actual file structure:_ + +- js + - a.js + - controller + - b.js + - manager + - factory + - c.js + +_pkg_: + +- js + - a.js +- js/controller + - b.js +- js/controller/manager/factory + - c.js + +_nested_: + +- a.js +- controller + - b.js + - manager + - factory + - c.js + +_flat_: + +- a.js +- controller/b.js +- controller/manager/factory/c.js -**Example config:** +**Example flat-organization setting:** ```js // web-test-runner.config.mjs export default { coverageConfig: { - report: true, reportDir: 'test-coverage', - reporters: ['cobertura', 'lcov'] + reporters: ['html'], threshold: { statements: 70, branches: 70, functions: 70, lines: 70, }, + defaultSummarizer: 'flat', }, }; ``` diff --git a/packages/test-runner-core/src/cli/writeCoverageReport.ts b/packages/test-runner-core/src/cli/writeCoverageReport.ts index 0a6881989..5ae1cb9af 100644 --- a/packages/test-runner-core/src/cli/writeCoverageReport.ts +++ b/packages/test-runner-core/src/cli/writeCoverageReport.ts @@ -8,21 +8,19 @@ export function writeCoverageReport(testCoverage: TestCoverage, config: Coverage // create a context for report generation const context = libReport.createContext({ dir: config.reportDir, - watermarks: { - statements: [50, 80], - functions: [50, 80], - branches: [50, 80], - lines: [50, 80], - }, + watermarks: config.watermarks, coverageMap: testCoverage.coverageMap, - }); + defaultSummarizer: config.defaultSummarizer, + } as libReport.ContextOptions); const reporters = config.reporters || []; for (const reporter of reporters) { + const options = config.reportOptions?.[reporter] || {}; const report = reports.create(reporter, { projectRoot: process.cwd(), maxCols: process.stdout.columns || 100, + ...options }); (report as any).execute(context); } diff --git a/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts b/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts index 6a17dacf8..71d4cc0f5 100644 --- a/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts +++ b/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts @@ -5,7 +5,8 @@ import { TestSession } from '../test-session/TestSession.js'; import { Reporter } from '../reporter/Reporter.js'; import { Logger } from '../logger/Logger.js'; import { TestRunnerPlugin } from '../server/TestRunnerPlugin.js'; -import { ReportType } from 'istanbul-reports'; +import { ReportType, ReportOptions } from 'istanbul-reports'; +import { Watermarks, Summarizers } from 'istanbul-lib-report'; export interface CoverageThresholdConfig { statements: number; @@ -22,6 +23,9 @@ export interface CoverageConfig { report?: boolean; reportDir?: string; reporters?: ReportType[]; + watermarks?: Partial; + reportOptions?: ReportOptions; + defaultSummarizer: Summarizers; } export interface TestRunnerCoreConfig { From 5941ed06978d4fe87399d021bda08e323da96932 Mon Sep 17 00:00:00 2001 From: Devin McIntyre Date: Wed, 16 Oct 2024 13:52:34 -0600 Subject: [PATCH 2/5] fix(test-runner-core): fix accidental deletion --- packages/test-runner-core/src/config/TestRunnerCoreConfig.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts b/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts index 71d4cc0f5..62559a7bb 100644 --- a/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts +++ b/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts @@ -25,7 +25,7 @@ export interface CoverageConfig { reporters?: ReportType[]; watermarks?: Partial; reportOptions?: ReportOptions; - defaultSummarizer: Summarizers; + defaultSummarizer?: Summarizers; } export interface TestRunnerCoreConfig { From 8abd32669cbb7ee97c0e20c4bb0ae737539e7b3f Mon Sep 17 00:00:00 2001 From: Devin McIntyre Date: Wed, 16 Oct 2024 16:51:07 -0600 Subject: [PATCH 3/5] feat(test-runner-core): #2806 advanced istanbul report configurations --- .changeset/silent-shoes-care.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/silent-shoes-care.md diff --git a/.changeset/silent-shoes-care.md b/.changeset/silent-shoes-care.md new file mode 100644 index 000000000..176a6061e --- /dev/null +++ b/.changeset/silent-shoes-care.md @@ -0,0 +1,5 @@ +--- +'@web/test-runner-core': patch +--- + +feat(test-runner-core): #2806 advanced istanbul report configurations From 1a96c92bb2f0bb31918e8b93c38df474d9de076d Mon Sep 17 00:00:00 2001 From: Devin McIntyre Date: Fri, 25 Oct 2024 12:31:45 -0600 Subject: [PATCH 4/5] fix(docs): Code coverage fails per summary, not per file --- docs/docs/test-runner/writing-tests/code-coverage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/test-runner/writing-tests/code-coverage.md b/docs/docs/test-runner/writing-tests/code-coverage.md index f60f03ee3..cfb6d7173 100644 --- a/docs/docs/test-runner/writing-tests/code-coverage.md +++ b/docs/docs/test-runner/writing-tests/code-coverage.md @@ -11,10 +11,10 @@ Additionally, your config file has a `coverage` boolean to toggle coverage on an ## Basic configuration Code coverage is measured using four metrics: statements, branches, functions, and lines. -The test runner measures these metrics for each file. +The test runner measures these metrics across all files combined. In `coverageConfig`, you need to define your desired code coverage thresholds for each metric. -The test run fails if you drop below this level for any metric in any file. +The test run fails if you drop below this level for any metric. The default setting is 0 for each metric. **Example minimum to require code coverage:** From 2298db4c840574700ca21c3101f05df29b6cb903 Mon Sep 17 00:00:00 2001 From: Devin McIntyre Date: Fri, 25 Oct 2024 13:54:24 -0600 Subject: [PATCH 5/5] fix(docs): Small typo chose -- choose --- docs/guides/test-runner/code-coverage/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/test-runner/code-coverage/index.md b/docs/guides/test-runner/code-coverage/index.md index ef8bee66f..3ef4279b1 100644 --- a/docs/guides/test-runner/code-coverage/index.md +++ b/docs/guides/test-runner/code-coverage/index.md @@ -209,7 +209,7 @@ You should, therefore, see code coverage as a tool that only gives you guidance ## Ignoring uncovered lines -In more complex applications, it is likely that you will find yourself creating difficult, if not impossible, to test branches of functionality. While this can absolutely be a pointer to logic that is worth breaking down into more approachable parts, there will be cases where this is not feasible. If so, you may chose to ignore a line of code by using the `/* c8 ignore next */` custom comment. Using this, or more advanced forms of [ignoring uncovered lines](../../../docs/test-runner/writing-tests/code-coverage.md#ignoring-uncovered-lines) while computing code coverage can go a long way in preparing your project for long term maintenance. +In more complex applications, it is likely that you will find yourself creating difficult, if not impossible, to test branches of functionality. While this can absolutely be a pointer to logic that is worth breaking down into more approachable parts, there will be cases where this is not feasible. If so, you may choose to ignore a line of code by using the `/* c8 ignore next */` custom comment. Using this, or more advanced forms of [ignoring uncovered lines](../../../docs/test-runner/writing-tests/code-coverage.md#ignoring-uncovered-lines) while computing code coverage can go a long way in preparing your project for long term maintenance. ## Coverage browser support