feat(smoke-tests): test updating from the latest release to the newly packaged app COMPASS-8532 COMPASS-8535

.evergreen/functions.yml

@@ -678,8 +678,8 @@ functions:
 
           if [[ "$IS_WINDOWS" == "true" ]]; then
             # TODO: windows_setup
-            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests=auto-update-from
-            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi --tests=auto-update-from
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests auto-update-from
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi --tests auto-update-from
           fi
 
           if [[ "$IS_OSX" == "true" ]]; then

packages/compass-e2e-tests/tests/auto-update.test.ts

@@ -13,66 +13,94 @@ function wait(ms: number) {
 }
 
 describe('Auto-update', function () {
-  it('auto-update from', async function () {
-    if (process.env.TEST_NAME !== 'auto-update-from') {
-      // we don't want this test to execute along with all the others under
-      // normal circumstances because it is destructive - it overwrites Compass
-      // itself
-      this.skip();
-    }
-
-    // run the app and wait for it to auto-update
-    console.log('starting compass the first time');
-    const compass = await init('auto-update from', { firstRun: true });
-    const { browser } = compass;
-    try {
-      await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();
-
-      if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
-        const restartButton = browser.$(Selectors.AutoUpdateRestartButton);
-        await restartButton.waitForDisplayed();
-
-        // We could click the restart button to apply the update and restart the
-        // app, but restarting the app confuses webdriverio or at least our test
-        // helpers. So we're going to just restart the app manually.
-        await browser.pause(1000);
-      } else {
-        // When auto-update is not supported the toast contains a link to
-        // download
-        const linkElement = browser.$(Selectors.AutoUpdateDownloadLink);
-        await linkElement.waitForDisplayed();
-        expect(await linkElement.getAttribute('href')).to.equal(
-          'https://www.mongodb.com/try/download/compass?utm_source=compass&utm_medium=product'
-        );
+  for (const testName of ['auto-update-from', 'auto-update-to']) {
+    it(testName, async function () {
+      if (process.env.TEST_NAME !== testName) {
+        // we don't want this test to execute along with all the others under
+        // normal circumstances because it is destructive - it overwrites Compass
+        // itself
+        this.skip();
       }
-    } finally {
-      await browser.screenshot(screenshotPathName('auto-update-from'));
-      await cleanup(compass);
-    }
 
-    if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
-      console.log(
-        'pause to make sure the app properly exited before starting again'
-      );
-      await wait(10_000);
-
-      console.log('starting compass a second time');
-      // run the app again and check that the version changed
-      const compass = await init('auto-update from restart', {
-        firstRun: false,
-      });
+      // run the app and wait for it to auto-update
+      console.log('starting compass the first time');
+      const compass = await init(testName, { firstRun: true });
       const { browser } = compass;
       try {
         await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();
-        await browser
-          .$(Selectors.AutoUpdateReleaseNotesLink)
-          .waitForDisplayed();
+
+        if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
+          const restartButton = browser.$(Selectors.AutoUpdateRestartButton);
+          await restartButton.waitForDisplayed();
+
+          if (process.env.EXPECTED_UPDATE_VERSION) {
+            expect(
+              await browser.$(Selectors.AutoUpdateToast).getText()
+            ).to.contain(
+              `Compass is ready to update to ${process.env.EXPECTED_UPDATE_VERSION}!`
+            );
+          }
+
+          // We could click the restart button to apply the update and restart the
+          // app, but restarting the app confuses webdriverio or at least our test
+          // helpers. So we're going to just restart the app manually.
+          await browser.pause(1000);
+        } else {
+          // When auto-update is not supported the toast contains a link to
+          // download
+          const linkElement = browser.$(Selectors.AutoUpdateDownloadLink);
+          await linkElement.waitForDisplayed();
+          expect(await linkElement.getAttribute('href')).to.equal(
+            'https://www.mongodb.com/try/download/compass?utm_source=compass&utm_medium=product'
+          );
+
+          if (process.env.EXPECTED_UPDATE_VERSION) {
+            expect(
+              await browser.$(Selectors.AutoUpdateToast).getText()
+            ).to.contain(
+              `Compass ${process.env.EXPECTED_UPDATE_VERSION} is available`
+            );
+          }
+        }
       } finally {
-        await browser.screenshot(
-          screenshotPathName('auto-update-from-restart')
-        );
+        await browser.screenshot(screenshotPathName(testName));
         await cleanup(compass);
       }
-    }
-  });
+
+      if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
+        console.log(
+          'pause to make sure the app properly exited before starting again'
+        );
+        await wait(10_000);
+
+        console.log('starting compass a second time');
+        // run the app again and check that the version changed
+        const compass = await init(`${testName} restart`, {
+          firstRun: false,
+        });
+        const { browser } = compass;
+        try {
+          await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();
+          const releaseNotesLink = browser.$(
+            Selectors.AutoUpdateReleaseNotesLink
+          );
+          await releaseNotesLink.waitForDisplayed();
+          // for now we only know the new version in the auto-update-to case
+          if (process.env.EXPECTED_UPDATE_VERSION) {
+            expect(
+              await browser.$(Selectors.AutoUpdateToast).getText()
+            ).to.contain(
+              `Compass ${process.env.EXPECTED_UPDATE_VERSION} installed successfully`
+            );
+            expect(await releaseNotesLink.getAttribute('href')).to.equal(
+              `https://github.com/mongodb-js/compass/releases/tag/v${process.env.EXPECTED_UPDATE_VERSION}`
+            );
+          }
+        } finally {
+          await browser.screenshot(screenshotPathName(`${testName}-restart`));
+          await cleanup(compass);
+        }
+      }
+    });
+  }
 });

packages/compass-smoke-tests/src/build-info.ts

@@ -5,17 +5,20 @@ import path from 'node:path';
 import { handler as writeBuildInfo } from 'hadron-build/commands/info';
 
 import { type PackageKind } from './packages';
-import { type SmokeTestsContext } from './context';
+import { type SmokeTestsContextWithSandbox } from './context';
 import { pick } from 'lodash';
 
-function assertObjectHasKeys(
-  obj: unknown,
-  name: string,
-  keys: readonly string[]
-) {
+const SUPPORTED_CHANNELS = ['dev', 'beta', 'stable'] as const;
+
+export type Channel = typeof SUPPORTED_CHANNELS[number];
+
+function assertObjectHasKeys<
+  Keys extends readonly string[],
+  Obj extends Record<Keys[number], unknown>
+>(obj: unknown, name: string, keys: Keys): asserts obj is Obj {
   assert(
     typeof obj === 'object' && obj !== null,
-    'Expected buildInfo to be an object'
+    `Expected ${name} to be an object`
   );
 
   for (const key of keys) {
@@ -25,13 +28,21 @@ function assertObjectHasKeys(
 
 // subsets of the hadron-build info result
 
-export const commonKeys = ['productName'] as const;
-export type CommonBuildInfo = Record<typeof commonKeys[number], string>;
+export const commonKeys = ['productName', 'version', 'channel'] as const;
+export type CommonBuildInfo = Record<typeof commonKeys[number], string> & {
+  channel: Channel;
+};
 
 export function assertCommonBuildInfo(
   buildInfo: unknown
 ): asserts buildInfo is CommonBuildInfo {
   assertObjectHasKeys(buildInfo, 'buildInfo', commonKeys);
+  assert(
+    SUPPORTED_CHANNELS.includes(buildInfo.channel as Channel),
+    `Expected ${JSON.stringify(
+      buildInfo.channel
+    )} to be in ${SUPPORTED_CHANNELS.join(',')}`
+  );
 }
 
 export const windowsFilenameKeys = [
@@ -221,7 +232,7 @@ export function readPackageDetails(
 }
 
 export function writeAndReadPackageDetails(
-  context: SmokeTestsContext
+  context: SmokeTestsContextWithSandbox
 ): PackageDetails {
   const compassDir = path.resolve(__dirname, '../../compass');
   const infoArgs = {