ENH - Generalise action (#34)

* 🔧 Get name from package * 🔧 Update pre-commits * ✏️ Remove new line * ♻️ Jupyterlab is now on main * 👷 Fix branch in actions * ⏪ Revert changes - we want to use yarn to link package * 👷 Minor enhancements to action summary * 🐛 Pass the path to the linked package * 🔧 Update external package path * ♻️ Move linked package summary to action * Move working directory to top of step * 📝 Update readmes * 🐛 Capture path instead * 🔧 Use vanilla jq for name * 👷 Use yarn 3 syntax * Should only link once * Docs improvements * Update .github/actions/link-package/action.yml Co-authored-by: gabalafou <[email protected]> * Better format inputs docs * 📝 Remove hard references to Lumino --------- Co-authored-by: gabalafou <[email protected]>
Quansight-Labs · Jul 12, 2023 · b748a27 · b748a27
1 parent 55014d6
commit b748a27
Show file tree

Hide file tree

Showing 10 changed files with 75 additions and 47 deletions.
diff --git a/.github/actions/accessibility-test-jupyterlab/action.yml b/.github/actions/accessibility-test-jupyterlab/action.yml
@@ -1,7 +1,7 @@
 name: Run accessibility tests on JupyterLab
 description: |
   Builds JupyterLab from source, then runs accessibility tests against it using
-  Playwright. Outputs the results to a markdown file which is then copied to the
+  Playwright. Outputs the results to a Markdown file which is then copied to the
   GitHub workflow run summary.
 # see https://github.com/actions/checkout more details on the inputs accepted
 inputs:
@@ -55,6 +55,7 @@ runs:
         npx playwright install --with-deps
         echo "::endgroup::"
         # saving the "system npm" path to call later (ensure we do not use conda npm)
+        # this is only needed for the tests, NOT to build JupyterLab
         echo "npm_path=$(which npm)" >> $GITHUB_ENV
       shell: bash -l {0}
       working-directory: ./testing/jupyterlab
@@ -89,7 +90,7 @@ runs:
         echo "::endgroup::"
       shell: bash -l {0}
 
-    - name: Build JupyterLab from source 🏗
+    - name: Get and build JupyterLab from source 🏗
       uses: ./.github/actions/build-lab
       with:
         repository: ${{ inputs.repository }}

diff --git a/.github/actions/build-lab/action.yml b/.github/actions/build-lab/action.yml
@@ -15,12 +15,12 @@ runs:
       uses: actions/checkout@v3
       with:
         repository: ${{ inputs.repository || 'jupyterlab/jupyterlab' }}
-        ref: ${{ inputs.ref || 'master' }}
+        ref: ${{ inputs.ref || 'main' }}
         path: jupyterlab
 
     - name: Build JupyterLab 🏗
       run: |
-        echo "::group:: Building JupyterLab 🏗"
+        echo "::group:: Building JupyterLab from echo '${{ inputs.ref || 'main' }}' 🏗"
         conda run --prefix $CONDA_PREFIX ./scripts/ci_install.sh
         conda run --prefix $CONDA_PREFIX jlpm run build
         echo "::endgroup::"

diff --git a/.github/actions/link-package/action.yml b/.github/actions/link-package/action.yml
@@ -1,11 +1,12 @@
 # Note: this is not meant to be used as a standalone action, as this only links a package
 # to an existing JupyterLab build.
-# You must install JupyterLab and its dependencies separately see .github/actions/build-lab/action.yml
+# You must install JupyterLab and its dependencies separately for example with the
+# .github/actions/build-lab/action.yml
 
-name: Build Lumino from source and link to JupyterLab
+name: Build an external package from source and link to JupyterLab
 description: |
-  Build Lumino from source then link to JupyterLab.
-  Useful to test changes in Lumino against a given build of JupyterLab.
+  Build external package from source then link to JupyterLab.
+  Useful to test changes in external packages against a given build of JupyterLab.
 
 inputs:
   external_pkg_repo:
@@ -26,29 +27,44 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Checkout package repo 🛎
+    - name: Checkout external package repo 🛎
       uses: actions/checkout@v3
       with:
         repository: ${{ inputs.external_pkg_repo }}
         ref: ${{ inputs.external_pkg_ref }}
         path: external-pkg
 
-    # following https://github.com/jupyterlab/lumino/blob/main/CONTRIBUTING.md#setting-up-a-development-environment
-    # and https://jupyterlab.readthedocs.io/en/latest/developer/contributing.html#id17
-    - name: Build package from source 🏗
+    # https://jupyterlab.readthedocs.io/en/latest/developer/contributing.html#id18
+    - name: Build external package from source 🏗
+      id: build-pkg
       run: |
         echo "::group::Build package from source 🛠"
         conda run --prefix $CONDA_PREFIX yarn
         conda run --prefix $CONDA_PREFIX yarn build:src
-        conda run --prefix $CONDA_PREFIX YARN_IGNORE_PATH=1 yarn link
         echo "::endgroup::"
+        echo $(pwd)
+        sudo apt install -y jq
+        echo "pkg-name=$(jq '.name' package.json)" >> $GITHUB_OUTPUT
       shell: bash -l {0}
       working-directory: external-pkg
 
-    - name: Link JupyterLab and package 🔗
+    - name: Link JupyterLab and external package 🔗
       run: |
-        # todo later on: replace the lumino reference here and output linked packages
-        conda run --prefix $CONDA_PREFIX jlpm link "lumino-top-level"
+        conda run --prefix $CONDA_PREFIX yarn link "${{ github.workspace }}/external-pkg" -p
         conda run --prefix $CONDA_PREFIX jlpm run build
       shell: bash -l {0}
       working-directory: jupyterlab
+
+    - name: Add details to action summary 📝
+      run: |
+        step_summary=$(cat <<EOF
+        ## Linked external package 🔗
+        - Package repository: [${{ inputs.external_pkg_repo }}](https://github.com/${{ github.event.inputs.external_pkg_repo }})
+        - Package \`ref\`: \`${{ inputs.external_pkg_ref }}\`
+        - Linked package name: \`${{ steps.build-pkg.outputs.pkg-name }}\`
+
+        EOF
+        )
+        echo "${step_summary}">>  $GITHUB_STEP_SUMMARY
+
+      shell: bash -l {0}
diff --git a/.github/workflows/accessibility-test-jupyterlab.yml b/.github/workflows/accessibility-test-jupyterlab.yml
@@ -19,7 +19,7 @@ on:
         description: The Jupyterlab repository's branch, tag, or SHA to checkout.
         type: string
         required: true
-        default: "master"
+        default: "main"
       test_project:
         description: |
           Leave blank to run the full test suite, or enter a Playwright project
@@ -53,7 +53,7 @@ on:
       ref:
         description: The branch, tag, or SHA to checkout.
         required: false
-        default: "master"
+        default: "main"
       test_project:
         description: |
           Leave blank to run the full test suite, or enter a Playwright project
@@ -82,6 +82,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  FORCE_COLOR: "1" # Make tools pretty.
+
 jobs:
   print-info:
     runs-on: ubuntu-latest
@@ -105,30 +108,24 @@ jobs:
     steps:
       # We have to check out the repo in order to locally reference the action,
       # i.e., .github/actions/...
-      - name: Check out repository 🛎
+      - name: Check out testing repository 🛎
         uses: actions/checkout@v3
 
       - name: Log action details in the summary 📝
         run: |
           step_summary=$(cat <<EOF
           # JupyterLab accessibility tests ♿️
-          ## Test details
-          - workflow trigger: \`${{ github.event_name }}\`
+          ## Tests execution details 🗂
+          - Workflow trigger: \`${{ github.event_name }}\`
           - \`ref\`: \`${{ github.ref }}\`
 
           - JupyterLab repository: [${{ github.event.inputs.repository || 'jupyterlab/jupyterlab' }}](https://github.com/${{ github.event.inputs.repository || 'jupyterlab/jupyterlab' }})
-          - JupyterLab repository \`ref\`: \`${{ github.event.inputs.ref || 'master' }}\`
+          - JupyterLab \`ref\`: \`${{ github.event.inputs.ref || 'main' }}\`
 
-          ---
-          \n
           EOF
           )
           echo "${step_summary}">>  $GITHUB_STEP_SUMMARY
-          if [[ -n "${{ github.event.inputs.external_pkg_ref }}" ]]; then
-            echo "**Linked external package 🔗 **" >> $GITHUB_STEP_SUMMARY
-            echo "- package repository: [${{ github.event.inputs.external_pkg_repo }}](https://github.com/${{ github.event.inputs.external_pkg_repo }})" >> $GITHUB_STEP_SUMMARY
-            echo "- \`ref\`: \`${{ github.event.inputs.external_pkg_ref }}\` " >> $GITHUB_STEP_SUMMARY
-          fi
+          echo "" >> $GITHUB_STEP_SUMMARY # adding a blank line for readability
 
       - name: Run testing action ✅
         uses: ./.github/actions/accessibility-test-jupyterlab

diff --git a/.gitignore b/.gitignore
@@ -133,3 +133,5 @@ reports/
 
 # editor
 .vscode
+
+.ruff_cache
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -16,7 +16,7 @@ ci:
 repos:
   # Autoformat: markdown, yaml to ensure that it doesn't need to be updated in other repos
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.0.0-alpha.6
+    rev: v3.0.0
     hooks:
       - id: prettier
 

diff --git a/README.md b/README.md
@@ -16,10 +16,11 @@ Information | Links
 - [Jupyter accessibility testing tools](#jupyter-accessibility-testing-tools)
   - [Plans for the future 🚀](#plans-for-the-future-)
   - [Repository orientation 🗺️](#repository-orientation-️)
+    - [Running the tests ✨](#running-the-tests-)
   - [Pre-commit hooks 🧹](#pre-commit-hooks-)
   - [License 📖](#license-)
 
-Welcome to the Jupyter Accessibility testing tool's repository. 👋🏽
+Welcome to the Jupyter Accessibility testing tools repository. 👋🏽
 This repository is a place for accessibility testing within [Jupyter](https://jupyter.org).
 
 > **Important**
@@ -29,7 +30,7 @@ This repository is a place for accessibility testing within [Jupyter](https://ju
 
 Automated accessibility tests cannot address accessibility issues on their own, but used correctly they can be a useful tool.
 
-Work in this repository is modeled after the [JupyterLab Benchmarks](https://github.com/jupyterlab/benchmarks/) repo.
+Work in this repository is modelled after the [JupyterLab Benchmarks](https://github.com/jupyterlab/benchmarks/) repo.
 
 As described in the [Jupyter Accessibility Roadmap](https://github.com/jupyter/accessibility/blob/main/docs/funding/czi-grant-roadmap.md),
 the plan is to start by adding tests for JupyterLab.
@@ -43,7 +44,7 @@ we hope to extend this testing to other parts of the Jupyter ecosystem beyond Ju
 
 This repository is organized as follows:
 
-```txt
+```bash
 .
 ├── .github
 │   ├── .github/workflows                               # set of GitHub actions to run the accessibility tests based on certain type of triggers
@@ -63,9 +64,13 @@ This repository is organized as follows:
 └── LICENSE                                             # license file
 ```
 
+### Running the tests ✨
+
+👉🏽 To learn about how to run and inspect the JupyterLab accessibility tests check the [JupyterLab testing README](./testing/jupyterlab/README.md).
+
 ## Pre-commit hooks 🧹
 
-This repository uses the `prettier` [pre-commit hook](https://pre-commit.com/) to standardize our YAML and markdown structure.
+This repository uses several [pre-commit hooks](https://pre-commit.com/) to standardize our codebase. Follow these steps to install the hooks:
 
 1. Before you can run the hooks, you need to install the pre-commit package manager:
 
@@ -84,7 +89,7 @@ This repository uses the `prettier` [pre-commit hook](https://pre-commit.com/) t
    pre-commit install
    ```
 
-3. Optional- run the hooks against the files in this repository
+3. Optional - run the hooks against the files in this repository
 
    ```bash
    # run the pre-commit hooks

diff --git a/images/GH-actions-run-workflow-button.png b/images/GH-actions-run-workflow-button.png
diff --git a/testing/jupyterlab/README.md b/testing/jupyterlab/README.md
@@ -34,33 +34,39 @@ You can run tests locally on your machine, or remotely in the cloud.
    workflow](https://github.com/Quansight-Labs/jupyter-a11y-testing/actions/workflows/accessibility-test-jupyterlab.yml) on the GitHub Actions UI.
 
 2. Click on the **Run Workflow** button. This should open a dropdown form.
+   ![GitHub actions UI - "Run accessibility tests on JupyterLab" workflow with a banner with the text: This workflow has a workflow_dispatch event trigger. And a Run workflow dropdown button.](./../../images/GH-actions-run-workflow-button.png)
 
-3. In the dropdown form, enter:
+3. In the dropdown form, enter the following inputs:
 
-   1. The JupyterLab repository or fork that you want to test (`default value`: `jupyterlab/jupyterlab`)
-   2. A valid reference to check the repository against should be one of:
-      - `branch` for example `main` (`default value`: `master` which is the [JupyterLab's repository default branch](https://github.com/jupyterlab/jupyterlab))
-      - `SHA` for example `4b4b0387febeff95780a3fdb100f4cd6848d29a2`
-      - `tag` for example `v1.0.0`
-   3. (Optional) - The subset of tests that you want to run (e.g., enter `regression` to run only the accessibility regression tests) (`default value`: `""` which runs all the tests).
-   4. (Optional) - A valid Lumino repository or fork to link to JupyterLab (see [Testing Changes to External Pages](https://jupyterlab.readthedocs.io/en/latest/developer/contributing.html#id17) section of JupyterLab's documentation). For example `jupyterlab/lumino` (`default value`: `""` to not link an external package).
-   5. (Optional) - A valid reference for Lumino to test against a JupyterLab build (see more details in the [Testing Changes to External Pages](https://jupyterlab.readthedocs.io/en/latest/developer/contributing.html#id17) section of JupyterLab's documentation). Should be one of `branch`, `tag`, `SHA` (`default value`: `""`).
+   | Variable            | Description                                                                                                                                                                                                        | Default value           | Required |
+   | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------- | -------- |
+   | `repository`        | The JupyterLab repository or fork that you want to test in the form of `<organisation or username>/<repository>`.                                                                                                  | `jupyterlab/jupyterlab` | Yes      |
+   | `ref`               | A valid reference to check the repository against should be one of:<br>- `branch`: a valid repository branch <br>- `SHA`: for example `4b4b0387febeff95780a3fdb100f4cd6848d29a2` <br>- `tag`: for example `v1.0.0` | `main`[^1]              | Yes      |
+   | `test_project`      | The name of the tests to run, should be one of: `""` (runs all tests) or `regression` (runs only the accessibility regression tests)                                                                               | `""`                    | No       |
+   | `external_pkg_repo` | A valid repository or fork to link to JupyterLab[^2] in the form of `<organisation or username>/<repository>` for example: `jupyterlab/lumino`                                                                     | `""`                    | No       |
+   | `external_pkg_ref`  | A valid reference for the external package or extension defined in Step 4. Should be one of:<br>- `branch`<br>- `tag`<br>- `SHA`                                                                                   | `""`                    | No       |
 
 4. Once you have entered all the information, click on the **Run workflow** button.
 
+[^1]: `main` is the default branch for JupyterLab hence this is the default value for the `ref` parameter.
+[^2]: This is useful if you want to test the accessibility of a package that is not part of the JupyterLab repository. See the [Testing Changes to External Pages](https://jupyterlab.readthedocs.io/en/latest/developer/contributing.html#id17) section of JupyterLab's documentation for more information.
+
 #### Inspecting the test results
 
 Once the accessibility tests have been completed you can use the following to inspect the test results:
 
-- **Markdown summary**: scroll to the bottom of the GitHub actions UI, where you can find this summary. From here you can use the links to read the manual test scripts for the test cases run.
+- **Markdown summary**: scroll to the bottom of the GitHub actions UI, where you can find the workflow run Summary. From here you can use the links to read the manual test scripts for the test cases run.
 - **Zipped `json` and `HTML` reports**: these can be downloaded from the GitHub actions UI. The `json` report can be used to generate a custom report using the [Playwright Reporter](https://playwright.dev/docs/test-reporters#custom-reporter). The `HTML` report can be opened in a browser to view the test results.
 - [**Annotations**](https://playwright.dev/docs/test-reporters#github-actions-annotations): this will appear directly in the GitHub actions UI.
 
 Follow these steps to download and read the reports locally:
 
 1. Scroll to the bottom of the GitHub actions UI and right-click on the test results' artifact.
 2. Unzip the downloaded file. Ideally in a folder where you have `npm` and `playwright` installed. For example, in the `testing/jupyterlab` directory.
-   (**Tip**: see the instructions in [Running the tests locally](#running-the-tests-locally-) to install the corresponding dependencies)
+
+   > **Tip**
+   > See the instructions in [Running the tests locally](#running-the-tests-locally-) to install the corresponding dependencies.
+
 3. Using the command line, change to the directory where you unzipped the test results.
 4. Use the following command to open the `HTML` report in your browser:
 

diff --git a/testing/jupyterlab/markdown-reporter.ts b/testing/jupyterlab/markdown-reporter.ts
@@ -49,7 +49,7 @@ export default class MarkdownReporter implements Reporter {
 
   private _serializeReport(): string {
     const lines: string[] = [];
-    lines.push("## Test Results");
+    lines.push("## Test Results 🔍");
     lines.push("");
     const projectSuites = this.rootSuite.suites;
 
@@ -63,6 +63,7 @@ export default class MarkdownReporter implements Reporter {
           lines.push(testResultString);
         }
         lines.push("");
+        lines.push("---");
       }
     }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -133,3 +133,5 @@ reports/

		# editor
		.vscode

		.ruff_cache