WIP - not ready - e2e cypress run in CI

.github/workflows/e2e.yaml

@@ -0,0 +1,29 @@
+name: e2e
+on:
+  workflow_call:
+jobs:
+  e2e-tests:
+    env:
+      CI: true
+      CONNECT_LICENSE: ${{ secrets.CONNECT_LICENSE }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runs-on: [ubuntu-latest]
+    runs-on: ${{ matrix.runs-on }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: ./.github/actions/setup
+      - uses: extractions/setup-just@v2
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist
+
+      - name: Extract dist
+        run: unzip dist/*-linux-amd64.vsix -d dist/
+
+      - run: just test/e2e/build-images
+      - run: just test/e2e/cypress-run

.github/workflows/pull-request.yaml

@@ -33,15 +33,10 @@ jobs:
     uses: ./.github/workflows/upload.yaml
     secrets: inherit
 
-  # Integration Tests
-  # bats:
-  #   needs: build
-  #   secrets: inherit
-  #   uses: ./.github/workflows/bats.yaml
-
-  vscode-ui:
+  # E2E Tests
+  e2e:
     needs:
       - build
       - package
     secrets: inherit
-    uses: ./.github/workflows/vscode-ui.yaml
+    uses: ./.github/workflows/e2e.yaml

.gitignore

@@ -26,5 +26,10 @@ node_modules
 .pnpm-store
 .Rproj.user
 
+# Possible e2e tests assets
+test/e2e/content-workspace/**/.posit
+test/e2e/cypress/downloads/
+test/e2e/cypress/screenshots/
+
 # Possible e2e tests deployment assets
 test/e2e/content-workspace/**/.posit

test/e2e/Dockerfile.cypress

@@ -1,13 +1,26 @@
 FROM cypress/browsers:node-18.20.3-chrome-125.0.6422.141-1-ff-126.0.1-edge-125.0.2535.85-1
 
-RUN mkdir /app
-WORKDIR /app
+# Install Python and required system dependencies
+RUN apt-get update && apt-get install -y \
+    python3 \
+    python3-pip \
+    python3-venv \
+    && rm -rf /var/lib/apt/lists/*
 
-COPY package*.json ./
-RUN npm ci
+# Create and activate virtual environment
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-ENV PATH /app/node_modules/.bin:$PATH
+# Install rsconnect, used for bootstraping first user and API key
+RUN pip install rsconnect-python
 
+# Setup the env for Cypress
 WORKDIR /app/e2e
 
-CMD ["cypress", "run"]
+COPY test/e2e/package*.json ./
+RUN npm ci
+
+ENV PATH /app/e2e/node_modules/.bin:$PATH
+
+CMD ["cypress", "run", "--browser", "chrome"]

test/e2e/cypress.config.js

@@ -5,16 +5,29 @@ module.exports = defineConfig({
     baseUrl: "http://localhost:8080",
     supportFile: "support/index.js",
     specPattern: "tests/**/*.cy.{js,jsx,ts,tsx}",
+    defaultCommandTimeout: 20000,
     // eslint-disable-next-line no-unused-vars
     setupNodeEvents(on, config) {
-      // implement node event listeners here
+      on("before:browser:launch", (browser = {}, launchOptions) => {
+        // Allow the non SSL access point to code server http://code-server:8080
+        // to be treated as secure, else, vscode complains about security and does not load.
+        // >  'crypto.subtle' is not available so webviews will not work.
+        //    This is likely because the editor is not running in a secure context
+        //    (https://developer.mozilla.org/en-US/docs/Web/Security/Secure_Contexts).
+        if (browser.name === "chrome") {
+          launchOptions.args.push(
+            "--unsafely-treat-insecure-origin-as-secure=http://code-server:8080",
+          );
+        }
+        return launchOptions;
+      });
     },
   },
   env: {
     BOOTSTRAP_ADMIN_API_KEY: "", // To be updated by Cypress when spinning up
-    BOOTSTRAP_SECRET_KEY: "bootstrap-secret.key", // To be updated by Cypress when spinning up
-    CONNECT_SERVER_URL: "http://localhost:3939",
-    CONNECT_MANAGER_URL: "http://localhost:4723",
+    BOOTSTRAP_SECRET_KEY: "bootstrap-secret.key",
+    CONNECT_SERVER_URL: "http://localhost:3939", // Updated by docker-compose env var for CI runs
+    CONNECT_MANAGER_URL: "http://localhost:4723", // Updated by docker-compose env var for CI runs
   },
   chromeWebSecurity: false,
 });

test/e2e/docker-compose.yml

@@ -31,3 +31,17 @@ services:
       - ../../dist:/home/coder/vsix
     ports:
       - "8080:8080"
+
+  # Cypress instance, meant to be used only on CI
+  # or to run e2e tests without the Cypress UI
+  cypress:
+    container_name: publisher-e2e.cypress
+    build:
+      context: ../..
+      dockerfile: test/e2e/Dockerfile.cypress
+    environment:
+      - CYPRESS_BASE_URL=http://code-server:8080
+      - CYPRESS_CONNECT_SERVER_URL=http://connect-publisher-e2e:3939
+      - CYPRESS_CONNECT_MANAGER_URL=http://connect-publisher-e2e:4723
+    volumes:
+      - ./:/app/e2e

test/e2e/justfile

@@ -20,7 +20,16 @@ stop:
 lint:
     npm run lint
 
-# Install local dependencies, start Connect in development mode, and open Cypress UI locally (not from a container)
+build-images:
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    just build "connect-publisher-e2e"
+    just build "code-server"
+    just build "cypress"
+
+# Install local dependencies, start Connect in development mode,
+# and open Cypress UI locally (not from a container)
 dev:
     #!/usr/bin/env bash
     set -euo pipefail
@@ -34,12 +43,28 @@ dev:
     USE_PLATFORM="linux/amd64" just ../../package
     just clear-credentials
     just install
-    just build "connect-publisher-e2e"
-    just build "code-server"
     just start "connect-publisher-e2e"
     just start "code-server"
     npm run cypress:open
 
+# Run Cypress tests using the CLI
+# Meant for CI or automatic runs where a build for linux/amd64 already exists
+cypress-run:
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    function cleanup() {
+        just clear-credentials
+    }
+    trap cleanup EXIT
+
+    just clear-credentials
+    just install
+    just start "connect-publisher-e2e"
+    just start "code-server"
+    docker compose run --rm cypress
+    just stop
+
 clear-credentials:
     #!/usr/bin/env bash
     set -euo pipefail

test/e2e/tests/credentials.cy.js

@@ -47,7 +47,7 @@ describe("Credentials Section", () => {
 
     cy.findInPublisherWebview(
       '[data-automation="admin-code-server-list"]',
-    ).then(($credRecord) => {
+    ).should(($credRecord) => {
       expect($credRecord.find(".tree-item-title").text()).to.equal(
         "admin-code-server",
       );

test/e2e/tests/deployments.cy.js

@@ -56,19 +56,6 @@ describe("Deployments Section", () => {
       .should("be.visible")
       .click();
 
-    cy.loadProjectConfigFile("static").then((config) => {
-      expect(config.title).to.equal("static");
-      expect(config.type).to.equal("html");
-      expect(config.entrypoint).to.equal("index.html");
-      expect(config.files[0]).to.equal("/index.html");
-      expect(config.files[1]).to.match(
-        /\/.posit\/publish\/static-[A-Z0-9]{4}\.toml/,
-      );
-      expect(config.files[2]).to.match(
-        /\/.posit\/publish\/deployments\/deployment-[A-Z0-9]{4}\.toml/,
-      );
-    });
-
     cy.publisherWebview()
       .findByTestId("deploy-button")
       .should("be.visible")
@@ -83,5 +70,12 @@ describe("Deployments Section", () => {
       .should("not.exist");
 
     cy.findByText("Deployment was successful").should("be.visible");
+
+    cy.loadProjectConfigFile("static").then((config) => {
+      expect(config.title).to.equal("static");
+      expect(config.type).to.equal("html");
+      expect(config.entrypoint).to.equal("index.html");
+      expect(config.files[0]).to.equal("/index.html");
+    });
   });
 });