Skip to content

Add script to update list of failing generated fourslash tests #1361

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 7, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 17 additions & 12 deletions internal/fourslash/_scripts/convertFourslash.mts
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,26 @@ import * as cp from "child_process";
import * as fs from "fs";
import * as path from "path";
import * as ts from "typescript";
import * as url from "url";
import which from "which";

const stradaFourslashPath = path.resolve(import.meta.dirname, "../", "../", "../", "_submodules", "TypeScript", "tests", "cases", "fourslash");

let inputFileSet: Set<string> | undefined;

const failingTestsPath = path.join(import.meta.dirname, "failingTests.txt");
const failingTestsList = fs.readFileSync(failingTestsPath, "utf-8").split("\n").map(line => line.trim().substring(4)).filter(line => line.length > 0);
const failingTests = new Set(failingTestsList);
const helperFilePath = path.join(import.meta.dirname, "../", "tests", "util_test.go");

const outputDir = path.join(import.meta.dirname, "../", "tests", "gen");

const unparsedFiles: string[] = [];

function main() {
function getFailingTests(): Set<string> {
const failingTestsList = fs.readFileSync(failingTestsPath, "utf-8").split("\n").map(line => line.trim().substring(4)).filter(line => line.length > 0);
return new Set(failingTestsList);
}

export function main() {
const args = process.argv.slice(2);
const inputFilesPath = args[0];
if (inputFilesPath) {
Expand All @@ -28,18 +32,17 @@ function main() {
inputFileSet = new Set(inputFiles);
}

if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
fs.rmSync(outputDir, { recursive: true, force: true });
fs.mkdirSync(outputDir, { recursive: true });

generateHelperFile();
parseTypeScriptFiles(stradaFourslashPath);
parseTypeScriptFiles(getFailingTests(), stradaFourslashPath);
console.log(unparsedFiles.join("\n"));
const gofmt = which.sync("go");
cp.execFileSync(gofmt, ["tool", "mvdan.cc/gofumpt", "-lang=go1.24", "-w", outputDir]);
}

function parseTypeScriptFiles(folder: string): void {
function parseTypeScriptFiles(failingTests: Set<string>, folder: string): void {
const files = fs.readdirSync(folder);

files.forEach(file => {
Expand All @@ -50,13 +53,13 @@ function parseTypeScriptFiles(folder: string): void {
}

if (stat.isDirectory()) {
parseTypeScriptFiles(filePath);
parseTypeScriptFiles(failingTests, filePath);
}
else if (file.endsWith(".ts")) {
const content = fs.readFileSync(filePath, "utf-8");
const test = parseFileContent(file, content);
if (test) {
const testContent = generateGoTest(test);
const testContent = generateGoTest(failingTests, test);
const testPath = path.join(outputDir, `${test.name}_test.go`);
fs.writeFileSync(testPath, testContent, "utf-8");
}
Expand Down Expand Up @@ -791,7 +794,7 @@ interface GoTest {
commands: Cmd[];
}

function generateGoTest(test: GoTest): string {
function generateGoTest(failingTests: Set<string>, test: GoTest): string {
const testName = test.name[0].toUpperCase() + test.name.substring(1);
const content = test.content;
const commands = test.commands.map(cmd => generateCmd(cmd)).join("\n");
Expand Down Expand Up @@ -827,4 +830,6 @@ function generateHelperFile() {
fs.copyFileSync(helperFilePath, path.join(outputDir, "util_test.go"));
}

main();
if (url.fileURLToPath(import.meta.url) == process.argv[1]) {
main();
}
42 changes: 11 additions & 31 deletions internal/fourslash/_scripts/failingTests.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,6 @@ TestCompletionEntryForClassMembers_StaticWhenBaseTypeIsNotResolved
TestCompletionExportFrom
TestCompletionForComputedStringProperties
TestCompletionForStringLiteral
TestCompletionForStringLiteral_quotePreference
TestCompletionForStringLiteral_quotePreference1
TestCompletionForStringLiteral_quotePreference2
TestCompletionForStringLiteral_quotePreference3
TestCompletionForStringLiteral_quotePreference4
TestCompletionForStringLiteral_quotePreference5
TestCompletionForStringLiteral_quotePreference6
TestCompletionForStringLiteral12
TestCompletionForStringLiteral15
TestCompletionForStringLiteral2
Expand Down Expand Up @@ -60,9 +53,9 @@ TestCompletionImportModuleSpecifierEndingTs
TestCompletionImportModuleSpecifierEndingTsxPreserve
TestCompletionImportModuleSpecifierEndingTsxReact
TestCompletionImportModuleSpecifierEndingUnsupportedExtension
TestCompletionInfoWithExplicitTypeArguments
TestCompletionInFunctionLikeBody_includesPrimitiveTypes
TestCompletionInJsDoc
TestCompletionInfoWithExplicitTypeArguments
TestCompletionListAndMemberListOnCommentedDot
TestCompletionListAndMemberListOnCommentedLine
TestCompletionListAndMemberListOnCommentedWhiteSpace
Expand Down Expand Up @@ -95,23 +88,24 @@ TestCompletionListInUnclosedTaggedTemplate01
TestCompletionListInUnclosedTaggedTemplate02
TestCompletionListInUnclosedTemplate01
TestCompletionListInUnclosedTemplate02
TestCompletionListInvalidMemberNames_withExistingIdentifier
TestCompletionListInvalidMemberNames2
TestCompletionListInvalidMemberNames_withExistingIdentifier
TestCompletionListOnAliases2
TestCompletionListPrivateNames
TestCompletionListPrivateNamesAccessors
TestCompletionListPrivateNamesMethods
TestCompletionListsStringLiteralTypeAsIndexedAccessTypeObject
TestCompletionListStaticMembers
TestCompletionListStaticProtectedMembers
TestCompletionListStaticProtectedMembers2
TestCompletionListStaticProtectedMembers3
TestCompletionListStringParenthesizedExpression
TestCompletionListStringParenthesizedType
TestCompletionListWithoutVariableinitializer
TestCompletionListsStringLiteralTypeAsIndexedAccessTypeObject
TestCompletionNoAutoInsertQuestionDotWithUserPreferencesOff
TestCompletionOfAwaitPromise6
TestCompletionPreferredSuggestions1
TestCompletionWithConditionalOperatorMissingColon
TestCompletionsAfterJSDoc
TestCompletionsBeforeRestArg1
TestCompletionsECMAPrivateMemberTriggerCharacter
Expand All @@ -123,9 +117,9 @@ TestCompletionsJSDocImportTagAttributesErrorModuleSpecifier1
TestCompletionsJSDocImportTagEmptyModuleSpecifier1
TestCompletionsJSDocNoCrash1
TestCompletionsJSDocNoCrash2
TestCompletionsJsPropertyAssignment
TestCompletionsJsdocParamTypeBeforeName
TestCompletionsJsdocTypeTagCast
TestCompletionsJsPropertyAssignment
TestCompletionsJsxAttribute2
TestCompletionsKeyof
TestCompletionsLiteralFromInferenceWithinInferredType3
Expand All @@ -143,13 +137,13 @@ TestCompletionsOverridingMethod4
TestCompletionsOverridingMethod9
TestCompletionsOverridingMethodCrash1
TestCompletionsOverridingProperties1
TestCompletionsPathsJsonModule
TestCompletionsPathsRelativeJsonModule
TestCompletionsPaths_importType
TestCompletionsPaths_kinds
TestCompletionsPaths_pathMapping
TestCompletionsPaths_pathMapping_nonTrailingWildcard1
TestCompletionsPaths_pathMapping_parentDirectory
TestCompletionsPathsJsonModule
TestCompletionsPathsRelativeJsonModule
TestCompletionsPropertiesPriorities
TestCompletionsRecommended_union
TestCompletionsRedeclareModuleAsGlobal
Expand All @@ -159,7 +153,6 @@ TestCompletionsSymbolMembers
TestCompletionsTriggerCharacter
TestCompletionsUniqueSymbol1
TestCompletionsWithStringReplacementMode1
TestCompletionWithConditionalOperatorMissingColon
TestExportEqualCallableInterface
TestGetJavaScriptCompletions1
TestGetJavaScriptCompletions10
Expand Down Expand Up @@ -200,22 +193,9 @@ TestImportCompletions_importsMap2
TestImportCompletions_importsMap3
TestImportCompletions_importsMap4
TestImportCompletions_importsMap5
TestImportCompletionsPackageJsonExportsSpecifierEndsInTs
TestImportCompletionsPackageJsonExportsTrailingSlash1
TestImportCompletionsPackageJsonImports_ts
TestImportCompletionsPackageJsonImportsConditions1
TestImportCompletionsPackageJsonImportsLength1
TestImportCompletionsPackageJsonImportsLength2
TestImportCompletionsPackageJsonImportsPattern
TestImportCompletionsPackageJsonImportsPattern_capsInPath1
TestImportCompletionsPackageJsonImportsPattern_capsInPath2
TestImportCompletionsPackageJsonImportsPattern_js_ts
TestImportCompletionsPackageJsonImportsPattern_ts
TestImportCompletionsPackageJsonImportsPattern_ts_ts
TestImportCompletionsPackageJsonImportsPattern2
TestImportStatementCompletions4
TestImportStatementCompletions_noPatternAmbient
TestImportStatementCompletions_pnpmTransitive
TestImportStatementCompletions4
TestImportTypeMemberCompletions
TestJavaScriptClass4
TestJavaScriptModules12
Expand All @@ -225,15 +205,15 @@ TestJavaScriptModules19
TestJavascriptModules20
TestJavascriptModules21
TestJavascriptModulesTypeImport
TestJsdocExtendsTagCompletion
TestJsDocFunctionSignatures3
TestJsDocGenerics1
TestJsdocExtendsTagCompletion
TestJsdocImplementsTagCompletion
TestJsdocOverloadTagCompletion
TestJsdocParameterNameCompletion
TestJsdocParamTagSpecialKeywords
TestJsdocPropertyTagCompletion
TestJsdocParameterNameCompletion
TestJsdocPropTagCompletion
TestJsdocPropertyTagCompletion
TestJsdocSatisfiesTagCompletion1
TestJsdocTemplatePrototypeCompletions
TestJsdocTemplateTagCompletion
Expand Down
3 changes: 2 additions & 1 deletion internal/fourslash/_scripts/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"compilerOptions": {
"strict": true,
"noEmit": true,
"module": "nodenext"
"module": "nodenext",
"allowImportingTsExtensions": true
}
}
32 changes: 32 additions & 0 deletions internal/fourslash/_scripts/updateFailing.mts
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is great 👍 it's a similar flow I've used manually myself

Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import * as cp from "child_process";
import * as fs from "fs";
import path from "path";
import which from "which";
import { main as convertFourslash } from "./convertFourslash.mts";

const failingTestsPath = path.join(import.meta.dirname, "failingTests.txt");

function main() {
fs.writeFileSync(failingTestsPath, "", "utf-8");
convertFourslash();
const go = which.sync("go");
let testOutput: string;
try {
testOutput = cp.execFileSync(go, ["test", "./internal/fourslash/tests/gen"], { encoding: "utf-8" });
}
catch (error) {
testOutput = (error as { stdout: string; }).stdout as string;
}
const regex = /--- FAIL: ([\S]+)/gm;
const failingTests: string[] = [];
let match;

while ((match = regex.exec(testOutput)) !== null) {
failingTests.push(match[1]);
}

fs.writeFileSync(failingTestsPath, failingTests.sort().join("\n") + "\n", "utf-8");
convertFourslash();
}

main();
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@
"extension:build": "npm run -w _extension build",
"extension:watch": "npm run -w _extension watch",
"node": "node --no-warnings --conditions @typescript/source",
"convertfourslash": "node --experimental-strip-types --no-warnings internal/fourslash/_scripts/convertFourslash.mts"
"convertfourslash": "node --experimental-strip-types --no-warnings internal/fourslash/_scripts/convertFourslash.mts",
"updatefailing": "node --experimental-strip-types --no-warnings internal/fourslash/_scripts/updateFailing.mts"
},
"workspaces": [
"./_extension",
Expand Down