Try to fix some intermittent nightly failures

- Clear task cache each activation
- Capture terminate promise before start debug test session
- gatherTests does not need to be async
- Make sure CodeLLDB is setup
- Try external terminal for LLDB
- Better reseting of diagnostics collection

Author: award999

.vscode-test.js

@@ -78,6 +78,7 @@ module.exports = defineConfig({
                 grep: isFastTestRun ? "@slow" : undefined,
                 invert: isFastTestRun,
                 slow: 10000,
+                retries: 1,
                 reporter: path.join(__dirname, ".mocha-reporter.js"),
                 reporterOptions: {
                     jsonReporterOptions: {

assets/test/.vscode/settings.json

@@ -8,5 +8,6 @@
         "-DTEST_ARGUMENT_SET_VIA_TEST_BUILD_ARGUMENTS_SETTING"
     ],
     "lldb.verboseLogging": true,
+    "lldb.launch.terminal": "external", // May prevent CodeLLDB hanging https://github.com/vadimcn/codelldb/issues/1040
     "swift.sourcekit-lsp.backgroundIndexing": "off"
 }

scripts/test.sh

@@ -17,7 +17,7 @@ set -ex
 
 current_directory=$(pwd)
 
-mkdir /tmp/code
+mkdir -p /tmp/code
 # Add the -v flag to see what is getting copied in to the working folder
 rsync -a --exclude "node_modules" \
     --exclude "out" \

src/TestExplorer/TestRunner.ts

@@ -1027,6 +1027,23 @@ export class TestRunner {
                         });
                         subscriptions.push(startSession);
 
+                        const terminateSession = vscode.debug.onDidTerminateDebugSession(e => {
+                            if (e.name !== config.name) {
+                                return;
+                            }
+                            this.workspaceContext.outputChannel.logDiagnostic(
+                                "Stop Test Debugging",
+                                this.folderContext.name
+                            );
+                            // dispose terminate debug handler
+                            subscriptions.forEach(sub => sub.dispose());
+
+                            void vscode.commands
+                                .executeCommand("workbench.view.extension.test")
+                                .then(() => resolve());
+                        });
+                        subscriptions.push(terminateSession);
+
                         vscode.debug
                             .startDebugging(this.folderContext.workspaceFolder, config)
                             .then(
@@ -1040,21 +1057,6 @@ export class TestRunner {
                                                 runState
                                             );
                                         }
-
-                                        const terminateSession =
-                                            vscode.debug.onDidTerminateDebugSession(() => {
-                                                this.workspaceContext.outputChannel.logDiagnostic(
-                                                    "Stop Test Debugging",
-                                                    this.folderContext.name
-                                                );
-                                                // dispose terminate debug handler
-                                                subscriptions.forEach(sub => sub.dispose());
-
-                                                void vscode.commands
-                                                    .executeCommand("workbench.view.extension.test")
-                                                    .then(() => resolve());
-                                            });
-                                        subscriptions.push(terminateSession);
                                     } else {
                                         subscriptions.forEach(sub => sub.dispose());
                                         reject("Debugger not started");

src/WorkspaceContext.ts

@@ -36,6 +36,7 @@ import { TestKind } from "./TestExplorer/TestKind";
 import { isValidWorkspaceFolder, searchForPackages } from "./utilities/workspace";
 import { SwiftPluginTaskProvider } from "./tasks/SwiftPluginTaskProvider";
 import { SwiftTaskProvider } from "./tasks/SwiftTaskProvider";
+import { LLDBDebugConfigurationProvider } from "./debugger/debugAdapterFactory";
 
 /**
  * Context for whole workspace. Holds array of contexts for each workspace folder
@@ -52,6 +53,7 @@ export class WorkspaceContext implements vscode.Disposable {
     public diagnostics: DiagnosticsManager;
     public taskProvider: SwiftTaskProvider;
     public pluginProvider: SwiftPluginTaskProvider;
+    public launchProvider: LLDBDebugConfigurationProvider;
     public subscriptions: vscode.Disposable[];
     public commentCompletionProvider: CommentCompletionProviders;
     public documentation: DocumentationManager;
@@ -82,6 +84,11 @@ export class WorkspaceContext implements vscode.Disposable {
         this.diagnostics = new DiagnosticsManager(this);
         this.taskProvider = new SwiftTaskProvider(this);
         this.pluginProvider = new SwiftPluginTaskProvider(this);
+        this.launchProvider = new LLDBDebugConfigurationProvider(
+            process.platform,
+            this,
+            outputChannel
+        );
         this.documentation = new DocumentationManager(extensionContext, this);
         this.currentDocument = null;
         this.commentCompletionProvider = new CommentCompletionProviders();

src/debugger/debugAdapterFactory.ts

@@ -47,9 +47,7 @@ export function registerDebugger(workspaceContext: WorkspaceContext): vscode.Dis
 
     function register() {
         subscriptions.push(registerLoggingDebugAdapterTracker());
-        subscriptions.push(
-            registerLLDBDebugAdapter(workspaceContext, workspaceContext.outputChannel)
-        );
+        subscriptions.push(registerLLDBDebugAdapter(workspaceContext));
     }
 
     if (!configuration.debugger.disable) {
@@ -69,13 +67,10 @@ export function registerDebugger(workspaceContext: WorkspaceContext): vscode.Dis
  * @param workspaceContext The workspace context
  * @returns A disposable to be disposed when the extension is deactivated
  */
-function registerLLDBDebugAdapter(
-    workspaceContext: WorkspaceContext,
-    outputChannel: SwiftOutputChannel
-): vscode.Disposable {
+function registerLLDBDebugAdapter(workspaceContext: WorkspaceContext): vscode.Disposable {
     return vscode.debug.registerDebugConfigurationProvider(
         SWIFT_LAUNCH_CONFIG_TYPE,
-        new LLDBDebugConfigurationProvider(process.platform, workspaceContext, outputChannel)
+        workspaceContext.launchProvider
     );
 }
 
@@ -203,7 +198,7 @@ export class LLDBDebugConfigurationProvider implements vscode.DebugConfiguration
         }
     }
 
-    private async promptForCodeLldbSettings(toolchain: SwiftToolchain): Promise<boolean> {
+    async promptForCodeLldbSettings(toolchain: SwiftToolchain): Promise<boolean> {
         const libLldbPathResult = await getLLDBLibPath(toolchain);
         if (!libLldbPathResult.success) {
             const errorMessage = `Error: ${getErrorDescription(libLldbPathResult.failure)}`;

test/integration-tests/DiagnosticsManager.test.ts

@@ -18,7 +18,7 @@ import { SwiftToolchain } from "../../src/toolchain/toolchain";
 import { executeTaskAndWaitForResult, waitForNoRunningTasks } from "../utilities/tasks";
 import { WorkspaceContext } from "../../src/WorkspaceContext";
 import { testAssetUri, testSwiftTask } from "../fixtures";
-import { createBuildAllTask } from "../../src/tasks/SwiftTaskProvider";
+import { createBuildAllTask, resetBuildAllTaskCache } from "../../src/tasks/SwiftTaskProvider";
 import { DiagnosticsManager } from "../../src/DiagnosticsManager";
 import { FolderContext } from "../../src/FolderContext";
 import { Version } from "../../src/utilities/version";
@@ -63,8 +63,7 @@ function assertWithoutDiagnostic(uri: vscode.Uri, expected: vscode.Diagnostic) {
 }
 
 suite("DiagnosticsManager Test Suite", function () {
-    // Was hitting a timeout in suiteSetup during CI build once in a while
-    this.timeout(15000);
+    this.timeout(60 * 1000 * 5); // Allow up to 5 minutes for build
 
     let workspaceContext: WorkspaceContext;
     let folderContext: FolderContext;
@@ -78,11 +77,9 @@ suite("DiagnosticsManager Test Suite", function () {
     let cppUri: vscode.Uri;
     let cppHeaderUri: vscode.Uri;
     let diagnosticWaiterDisposable: vscode.Disposable | undefined;
-    let remainingExpectedDiagnostics:
-        | {
-              [uri: string]: vscode.Diagnostic[];
-          }
-        | undefined;
+    let remainingExpectedDiagnostics: {
+        [uri: string]: vscode.Diagnostic[];
+    };
 
     // Wait for all the expected diagnostics to be recieved. This may happen over several `onChangeDiagnostics` events.
     type ExpectedDiagnostics = { [uri: string]: vscode.Diagnostic[] };
@@ -96,22 +93,23 @@ suite("DiagnosticsManager Test Suite", function () {
             }
             // Keep a lookup of diagnostics we haven't encountered yet. When all array values in
             // this lookup are empty then we've seen all diagnostics and we can resolve successfully.
-            const expected = { ...expectedDiagnostics };
+            remainingExpectedDiagnostics = { ...expectedDiagnostics };
             diagnosticWaiterDisposable = vscode.languages.onDidChangeDiagnostics(e => {
                 const matchingPaths = Object.keys(expectedDiagnostics).filter(uri =>
                     e.uris.some(u => u.fsPath === uri)
                 );
                 for (const uri of matchingPaths) {
                     const actualDiagnostics = vscode.languages.getDiagnostics(vscode.Uri.file(uri));
-                    expected[uri] = expected[uri].filter(expectedDiagnostic => {
-                        return !actualDiagnostics.some(actualDiagnostic =>
-                            isEqual(actualDiagnostic, expectedDiagnostic)
-                        );
-                    });
-                    remainingExpectedDiagnostics = expected;
+                    for (const actualDiagnostic of actualDiagnostics) {
+                        remainingExpectedDiagnostics[uri] = remainingExpectedDiagnostics[
+                            uri
+                        ].filter(expectedDiagnostic => {
+                            return !isEqual(actualDiagnostic, expectedDiagnostic);
+                        });
+                    }
                 }
 
-                const allDiagnosticsFulfilled = Object.values(expected).every(
+                const allDiagnosticsFulfilled = Object.values(remainingExpectedDiagnostics).every(
                     diagnostics => diagnostics.length === 0
                 );
 
@@ -126,8 +124,6 @@ suite("DiagnosticsManager Test Suite", function () {
 
     activateExtensionForSuite({
         async setup(ctx) {
-            this.timeout(60000 * 5);
-
             workspaceContext = ctx;
             toolchain = workspaceContext.globalToolchain;
             folderContext = await folderInRootWorkspace("diagnostics", workspaceContext);
@@ -160,8 +156,6 @@ suite("DiagnosticsManager Test Suite", function () {
     });
 
     suite("Parse diagnostics", function () {
-        this.timeout(60000 * 2);
-
         suite("Parse from task output", () => {
             const expectedWarningDiagnostic = new vscode.Diagnostic(
                 new vscode.Range(new vscode.Position(1, 8), new vscode.Position(1, 8)),
@@ -228,7 +222,7 @@ suite("DiagnosticsManager Test Suite", function () {
                     suiteSetup(async function () {
                         // Swift 5.10 and 6.0 on Windows have a bug where the
                         // diagnostics are not emitted on their own line.
-                        const swiftVersion = workspaceContext.globalToolchain.swiftVersion;
+                        const swiftVersion = folderContext.toolchain.swiftVersion;
                         if (
                             swiftVersion.isLessThan(new Version(5, 10, 0)) ||
                             (process.platform === "win32" &&
@@ -237,25 +231,36 @@ suite("DiagnosticsManager Test Suite", function () {
                         ) {
                             this.skip();
                         }
-                        this.timeout(5 * 60 * 1000); // Allow 5 minutes to build
+
+                        resetSettings = await updateSettings({ "swift.diagnosticsStyle": style });
 
                         // Clean up any lingering diagnostics
-                        workspaceContext.diagnostics.clear();
-                        await workspaceContext.focusFolder(null);
+                        if (vscode.languages.getDiagnostics(mainUri).length > 0) {
+                            const clearPromise = new Promise<void>(resolve => {
+                                const diagnosticDisposable =
+                                    vscode.languages.onDidChangeDiagnostics(() => {
+                                        if (vscode.languages.getDiagnostics(mainUri).length === 0) {
+                                            diagnosticDisposable?.dispose();
+                                            resolve();
+                                        }
+                                    });
+                            });
+                            workspaceContext.diagnostics.clear();
+                            await clearPromise;
+                        }
+                    });
 
-                        resetSettings = await updateSettings({ "swift.diagnosticsStyle": style });
+                    setup(() => {
+                        resetBuildAllTaskCache();
                     });
 
                     test("succeeds", async function () {
                         await Promise.all([
                             waitForDiagnostics(expected()),
                             createBuildAllTask(folderContext).then(task =>
-                                executeTaskAndWaitForResult(task).catch(() => {
-                                    /* Ignore */
-                                })
+                                executeTaskAndWaitForResult(task)
                             ),
                         ]);
-                        await waitForNoRunningTasks();
                     });
 
                     callback && callback();

test/integration-tests/testexplorer/TestExplorerIntegration.test.ts

@@ -52,7 +52,7 @@ import { FolderContext } from "../../../src/FolderContext";
 import { lineBreakRegex } from "../../../src/utilities/tasks";
 
 suite("Test Explorer Suite", function () {
-    const MAX_TEST_RUN_TIME_MINUTES = 5;
+    const MAX_TEST_RUN_TIME_MINUTES = 6;
 
     this.timeout(1000 * 60 * MAX_TEST_RUN_TIME_MINUTES);
 
@@ -427,7 +427,7 @@ suite("Test Explorer Suite", function () {
                 this.timeout(1000 * 60 * MAX_TEST_RUN_TIME_MINUTES * 5);
 
                 test("@slow runs an swift-testing test multiple times", async function () {
-                    const testItems = await gatherTests(
+                    const testItems = gatherTests(
                         testExplorer.controller,
                         "PackageTests.MixedXCTestSuite/testPassing"
                     );
@@ -495,7 +495,7 @@ suite("Test Explorer Suite", function () {
                 if (!targetProfile) {
                     throw new Error(`Unable to find run profile named ${TestKind.standard}`);
                 }
-                const testItems = await gatherTests(
+                const testItems = gatherTests(
                     testExplorer.controller,
                     "PackageTests.DuplicateSuffixTests/testPassing"
                 );
@@ -560,7 +560,7 @@ suite("Test Explorer Suite", function () {
                 this.timeout(1000 * 60 * MAX_TEST_RUN_TIME_MINUTES * 5);
 
                 test("@slow runs an XCTest multiple times", async function () {
-                    const testItems = await gatherTests(
+                    const testItems = gatherTests(
                         testExplorer.controller,
                         "PackageTests.PassingXCTestSuite/testPassing"
                     );

test/integration-tests/testexplorer/utilities.ts

@@ -247,10 +247,10 @@ function getTestItem(
  * @param tests A list of test IDs
  * @returns A collection of resolved `vscode.TestItem`s
  */
-export async function gatherTests(
+export function gatherTests(
     controller: vscode.TestController,
     ...tests: string[]
-): Promise<vscode.TestItem[]> {
+): vscode.TestItem[] {
     const testItems = tests.map(test => {
         const testItem = getTestItem(controller, test);
         if (!testItem) {
@@ -295,7 +295,7 @@ export async function runTest(
     if (!targetProfile) {
         throw new Error(`Unable to find run profile named ${runProfile}`);
     }
-    const testItems = await gatherTests(testExplorer.controller, ...tests);
+    const testItems = gatherTests(testExplorer.controller, ...tests);
     const request = new vscode.TestRunRequest(testItems);
 
     // The first promise is the return value, the second promise builds and runs

test/integration-tests/utilities/testutilities.ts

@@ -93,7 +93,17 @@ const extensionBootstrapper = (() => {
                 restoreSettings = await updateSettings({
                     "swift.debugger.setupCodeLLDB": "never",
                 });
+            } else if (requiresDebugger) {
+                await workspaceContext.launchProvider.promptForCodeLldbSettings(
+                    workspaceContext.globalToolchain
+                );
             }
+
+            // Make sure no running tasks before setting up
+            await waitForNoRunningTasks({ timeout: 10000 });
+            // Clear build all cache before starting suite
+            resetBuildAllTaskCache();
+
             if (!setup) {
                 return;
             }
@@ -141,8 +151,6 @@ const extensionBootstrapper = (() => {
                 if (autoTeardown) {
                     await autoTeardown();
                 }
-                await waitForNoRunningTasks();
-                resetBuildAllTaskCache();
             } catch (error) {
                 if (workspaceContext) {
                     printLogs(workspaceContext.outputChannel, "Error during test/suite teardown");