Introducing AVA snapshots v3

The v3 format is encoded using CBOR. It contains all information necessary to regenerate the report file. This allows for snapshots to be updated even if tests or assertions are skipped.

This is not backwards compatible with the v2 format used in AVA 3.

Co-authored-by: Mark Wubben <[email protected]>

Author: ninevra

docs/01-writing-tests.md

@@ -120,7 +120,7 @@ You can use the `.only` modifier with all tests. It cannot be used with hooks or
 
 *Note:* The `.only` modifier applies to the test file it's defined in, so if you run multiple test files, tests in other files will still run. If you want to only run the `test.only` test, provide just that test file to AVA.
 
-You cannot update snapshots when using `.only()`.
+In AVA 3, you cannot update snapshots when using `.only()`.
 
 ## Skipping tests
 
@@ -134,7 +134,9 @@ test.skip('will not be run', t => {
 
 You must specify the implementation function. You can use the `.skip` modifier with all tests and hooks, but not with `.todo()`. You can not apply further modifiers to `.skip`.
 
-You cannot update snapshots when using `.skip()`. If the test is likely to be failing for a while, use `.failing()` instead.
+If the test is likely to be failing for a while, use `.failing()` instead.
+
+In AVA 3, you cannot update snapshots when using `.skip()`.
 
 ## Test placeholders ("todo")
 

docs/03-assertions.md

@@ -337,7 +337,7 @@ Compares the `expected` value with a previously recorded snapshot. Snapshots are
 
 AVA 3 supports an  `options` object that lets you select a specific snapshot, for instance `{id: 'my snapshot'}`. This is buggy and will be removed in AVA 4.
 
-Snapshot assertions cannot be skipped when snapshots are being updated.
+In AVA 3, you cannot update snapshots while using `t.snapshot.skip()`.
 
 ### `.try(title?, implementation | macro | macro[], ...args?)`
 

docs/04-snapshot-testing.md

@@ -44,6 +44,8 @@ You can then check your code. If the change was intentional you can use the `--u
 $ ava --update-snapshots
 ```
 
+In AVA 4, if you need to update snapshots for only a particular test, you can use `--update-snapshots` together with e.g. `--match` or `.only()` to select the test.
+
 You can specify a fixed location for storing the snapshot files in AVA's [`package.json` configuration](./06-configuration.md):
 
 **`package.json`:**

lib/cli.js

@@ -210,10 +210,6 @@ exports.run = async () => { // eslint-disable-line complexity
 	const chalkOptions = {level: combined.color === false ? 0 : require('chalk').level};
 	const chalk = require('./chalk').set(chalkOptions);
 
-	if (combined.updateSnapshots && combined.match) {
-		exit('Snapshots cannot be updated when matching specific tests.');
-	}
-
 	if (confError) {
 		if (confError.parent) {
 			exit(`${confError.message}\n\n${chalk.gray((confError.parent && confError.parent.stack) || confError.parent)}`);
@@ -390,9 +386,6 @@ exports.run = async () => { // eslint-disable-line complexity
 			pattern: normalizePattern(path.relative(projectDir, path.resolve(process.cwd(), pattern))),
 			...rest
 		}));
-	if (combined.updateSnapshots && filter.some(condition => condition.lineNumbers !== null)) {
-		exit('Snapshots cannot be updated when selecting specific tests by their line number.');
-	}
 
 	const api = new Api({
 		cacheEnabled: combined.cache !== false,

lib/reporters/default.js

@@ -198,7 +198,6 @@ class Reporter {
 		this.sharedWorkerErrors = [];
 		this.uncaughtExceptions = [];
 		this.unhandledRejections = [];
-		this.unsavedSnapshots = [];
 
 		this.previousFailures = 0;
 
@@ -354,10 +353,6 @@ class Reporter {
 				break;
 			}
 
-			case 'snapshot-error':
-				this.unsavedSnapshots.push(event);
-				break;
-
 			case 'uncaught-exception': {
 				this.uncaughtExceptions.push(event);
 
@@ -825,16 +820,6 @@ class Reporter {
 			}
 		}
 
-		if (this.unsavedSnapshots.length > 0) {
-			this.lineWriter.writeLine(colors.title('Could not update snapshots for the following test files:'));
-			this.lineWriter.writeLine();
-			for (const event of this.unsavedSnapshots) {
-				this.lineWriter.writeLine(`${figures.warning} ${this.relativeFile(event.testFile)}`);
-			}
-
-			this.lineWriter.writeLine();
-		}
-
 		if (this.failFastEnabled && (this.stats.remainingTests > 0 || this.stats.files > this.stats.finishedWorkers)) {
 			let remaining = '';
 			if (this.stats.remainingTests > 0) {

lib/reporters/tap.js

@@ -165,9 +165,6 @@ class TapReporter {
 					this.writeTest(evt, {passed: false, todo: true, skip: false});
 				}
 
-				break;
-			case 'snapshot-error':
-				this.writeComment(evt, {title: 'Could not update snapshots'});
 				break;
 			case 'stats':
 				this.stats = evt.stats;

lib/runner.js

@@ -23,16 +23,24 @@ class Runner extends Emittery {
 		this.recordNewSnapshots = options.recordNewSnapshots === true;
 		this.runOnlyExclusive = options.runOnlyExclusive === true;
 		this.serial = options.serial === true;
-		this.skippingTests = false;
 		this.snapshotDir = options.snapshotDir;
 		this.updateSnapshots = options.updateSnapshots;
 
 		this.activeRunnables = new Set();
 		this.boundCompareTestSnapshot = this.compareTestSnapshot.bind(this);
-		this.skippedSnapshots = false;
 		this.boundSkipSnapshot = this.skipSnapshot.bind(this);
 		this.interrupted = false;
-		this.snapshots = null;
+		this.snapshots = snapshotManager.load({
+			file: this.file,
+			fixedLocation: this.snapshotDir,
+			projectDir: this.projectDir,
+			recordNewSnapshots: this.recordNewSnapshots,
+			updating: this.updateSnapshots
+		});
+		if (this.snapshots.snapPath !== undefined) {
+			this.emit('dependency', this.snapshots.snapPath);
+		}
+
 		this.nextTaskIndex = 0;
 		this.tasks = {
 			after: [],
@@ -152,15 +160,14 @@ class Runner extends Emittery {
 							task.metadata.exclusive = matcher([title], this.match).length === 1;
 						}
 
-						if (task.metadata.skipped) {
-							this.skippingTests = true;
-						}
-
 						if (task.metadata.exclusive) {
 							this.runOnlyExclusive = true;
 						}
 
 						this.tasks[metadata.serial ? 'serial' : 'concurrent'].push(task);
+
+						this.snapshots.touch(title, metadata.taskIndex);
+
 						this.emit('stateChange', {
 							type: 'declared-test',
 							title,
@@ -185,49 +192,15 @@ class Runner extends Emittery {
 	}
 
 	compareTestSnapshot(options) {
-		if (!this.snapshots) {
-			this.snapshots = snapshotManager.load({
-				file: this.file,
-				fixedLocation: this.snapshotDir,
-				projectDir: this.projectDir,
-				recordNewSnapshots: this.recordNewSnapshots,
-				updating: this.updateSnapshots && !this.runOnlyExclusive && !this.skippingTests
-			});
-			this.emit('dependency', this.snapshots.snapPath);
-		}
-
 		return this.snapshots.compare(options);
 	}
 
-	skipSnapshot() {
-		this.skippedSnapshots = true;
+	skipSnapshot(options) {
+		return this.snapshots.skipSnapshot(options);
 	}
 
 	saveSnapshotState() {
-		if (
-			this.updateSnapshots &&
-			(
-				this.runOnlyExclusive ||
-				this.skippingTests ||
-				this.skippedSnapshots
-			)
-		) {
-			return {cannotSave: true};
-		}
-
-		if (this.snapshots) {
-			return {touchedFiles: this.snapshots.save()};
-		}
-
-		if (this.updateSnapshots) {
-			return {touchedFiles: snapshotManager.cleanSnapshots({
-				file: this.file,
-				fixedLocation: this.snapshotDir,
-				projectDir: this.projectDir
-			})};
-		}
-
-		return {};
+		return {touchedFiles: this.snapshots.save()};
 	}
 
 	onRun(runnable) {
@@ -301,7 +274,7 @@ class Runner extends Emittery {
 		return result;
 	}
 
-	async runHooks(tasks, contextRef, {titleSuffix, testPassed, associatedTaskIndex} = {}) {
+	async runHooks(tasks, contextRef, {titleSuffix, testPassed} = {}) {
 		const hooks = tasks.map(task => new Runnable({
 			contextRef,
 			experiments: this.experiments,
@@ -312,7 +285,7 @@ class Runner extends Emittery {
 			compareTestSnapshot: this.boundCompareTestSnapshot,
 			skipSnapshot: this.boundSkipSnapshot,
 			updateSnapshots: this.updateSnapshots,
-			metadata: {...task.metadata, associatedTaskIndex},
+			metadata: task.metadata,
 			powerAssert: this.powerAssert,
 			title: `${task.title}${titleSuffix || ''}`,
 			isHook: true,
@@ -347,8 +320,7 @@ class Runner extends Emittery {
 			this.tasks.beforeEach,
 			contextRef,
 			{
-				titleSuffix: hookSuffix,
-				associatedTaskIndex: task.metadata.taskIndex
+				titleSuffix: hookSuffix
 			}
 		);
 
@@ -388,8 +360,7 @@ class Runner extends Emittery {
 					contextRef,
 					{
 						titleSuffix: hookSuffix,
-						testPassed: testOk,
-						associatedTaskIndex: task.metadata.taskIndex
+						testPassed: testOk
 					});
 			} else {
 				this.emit('stateChange', {
@@ -409,8 +380,7 @@ class Runner extends Emittery {
 			contextRef,
 			{
 				titleSuffix: hookSuffix,
-				testPassed: testOk,
-				associatedTaskIndex: task.metadata.taskIndex
+				testPassed: testOk
 			});
 		return alwaysOk && hooksOk && testOk;
 	}
@@ -420,10 +390,12 @@ class Runner extends Emittery {
 		const serialTests = [];
 		for (const task of this.tasks.serial) {
 			if (this.runOnlyExclusive && !task.metadata.exclusive) {
+				this.snapshots.skipBlock(task.title, task.metadata.taskIndex);
 				continue;
 			}
 
 			if (this.checkSelectedByLineNumbers && !task.metadata.selected) {
+				this.snapshots.skipBlock(task.title, task.metadata.taskIndex);
 				continue;
 			}
 
@@ -435,17 +407,21 @@ class Runner extends Emittery {
 				todo: false
 			});
 
-			if (!task.metadata.skipped) {
+			if (task.metadata.skipped) {
+				this.snapshots.skipBlock(task.title, task.metadata.taskIndex);
+			} else {
 				serialTests.push(task);
 			}
 		}
 
 		for (const task of this.tasks.concurrent) {
 			if (this.runOnlyExclusive && !task.metadata.exclusive) {
+				this.snapshots.skipBlock(task.title, task.metadata.taskIndex);
 				continue;
 			}
 
 			if (this.checkSelectedByLineNumbers && !task.metadata.selected) {
+				this.snapshots.skipBlock(task.title, task.metadata.taskIndex);
 				continue;
 			}
 
@@ -457,12 +433,12 @@ class Runner extends Emittery {
 				todo: false
 			});
 
-			if (!task.metadata.skipped) {
-				if (this.serial) {
-					serialTests.push(task);
-				} else {
-					concurrentTests.push(task);
-				}
+			if (task.metadata.skipped) {
+				this.snapshots.skipBlock(task.title, task.metadata.taskIndex);
+			} else if (this.serial) {
+				serialTests.push(task);
+			} else {
+				concurrentTests.push(task);
 			}
 		}