Merge pull request #65 from MatrixAI/feature-test-load-balancing

feat: test load balancing using jest's `--shard` option

Author: emmacasolin

.gitlab-ci.yml

@@ -72,24 +72,36 @@ check:nix-dry:
     - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != 'master' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
       when: manual
 
-check:test:
+check:test-generate:
   stage: check
-  needs: []
   script:
+    - mkdir -p ./tmp
     - >
         nix-shell --run '
-        npm run build --verbose;
-        npm test -- --ci --coverage;
+        ./scripts/test-pipelines.sh > ./tmp/test-pipelines.yml
         '
   artifacts:
     when: always
-    reports:
-      junit:
-        - ./tmp/junit/junit.xml
-      coverage_report:
-        coverage_format: cobertura
-        path: ./tmp/coverage/cobertura-coverage.xml
-  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
+    paths:
+      - ./tmp/test-pipelines.yml
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Manually run on commits other than master and staging and ignore version commits
+    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+      when: manual
+
+check:test:
+  stage: check
+  needs:
+    - check:test-generate
+  inherit:
+    variables: false
+  trigger:
+    include:
+      - artifact: tmp/test-pipelines.yml
+        job: check:test-generate
+    strategy: depend
   rules:
     # Runs on staging commits and ignores version commits
     - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
@@ -125,82 +137,35 @@ build:merge:
     # Runs on tag pipeline where the tag is a prerelease or release version
     - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
 
-build:linux:
+build:build-generate:
   stage: build
-  needs: []
   script:
+    - mkdir -p ./tmp
     - >
         nix-shell --run '
-        npm run build --verbose;
-        npm test -- --ci --coverage;
+        ./scripts/build-pipelines.sh > ./tmp/build-pipelines.yml
         '
   artifacts:
     when: always
-    reports:
-      junit:
-        - ./tmp/junit/junit.xml
-      coverage_report:
-        coverage_format: cobertura
-        path: ./tmp/coverage/cobertura-coverage.xml
     paths:
-      # Only the build:linux preserves the dist
-      - ./dist
-  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
-  rules:
-    # Runs on staging commits and ignores version commits
-    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-    # Runs on tag pipeline where the tag is a prerelease or release version
-    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-
-build:windows:
-  stage: build
-  needs: []
-  tags:
-    - windows
-  before_script:
-    - choco install nodejs --version=16.14.2 -y
-    - refreshenv
-  script:
-    - npm config set msvs_version 2019
-    - npm install --ignore-scripts
-    - $env:Path = "$(npm bin);" + $env:Path
-    - npm run build --verbose
-    - npm test -- --ci
-  artifacts:
-    when: always
-    reports:
-      junit:
-        - ./tmp/junit/junit.xml
+      - ./tmp/build-pipelines.yml
   rules:
     # Runs on staging commits and ignores version commits
     - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
     # Runs on tag pipeline where the tag is a prerelease or release version
     - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
 
-build:macos:
+build:platforms:
   stage: build
-  needs: []
-  tags:
-    - shared-macos-amd64
-  image: macos-11-xcode-12
-  variables:
-    HOMEBREW_NO_INSTALL_UPGRADE: "true"
-    HOMEBREW_NO_INSTALL_CLEANUP: "true"
-  before_script:
-    - eval "$(brew shellenv)"
-    - brew install node@16
-    - brew link --overwrite node@16
-    - hash -r
-  script:
-    - npm install --ignore-scripts
-    - export PATH="$(npm bin):$PATH"
-    - npm run build --verbose
-    - npm test -- --ci
-  artifacts:
-    when: always
-    reports:
-      junit:
-        - ./tmp/junit/junit.xml
+  needs:
+    - build:build-generate
+  inherit:
+    variables: false
+  trigger:
+    include:
+      - artifact: tmp/build-pipelines.yml
+        job: build:build-generate
+    strategy: depend
   rules:
     # Runs on staging commits and ignores version commits
     - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
@@ -210,9 +175,7 @@ build:macos:
 build:prerelease:
   stage: build
   needs:
-    - build:linux
-    - build:windows
-    - build:macos
+    - build:platforms
   # Don't interrupt publishing job
   interruptible: false
   before_script:
@@ -235,9 +198,7 @@ build:prerelease:
 integration:builds:
   stage: integration
   needs:
-    - build:linux
-    - build:windows
-    - build:macos
+    - build:platforms
   script:
     - mkdir -p ./builds
     - >
@@ -434,11 +395,7 @@ integration:merge:
   stage: integration
   needs:
     - build:merge
-    - job: build:linux
-      optional: true
-    - job: build:windows
-      optional: true
-    - job: build:macos
+    - job: build:platforms
       optional: true
     - job: integration:nix
       optional: true
@@ -514,9 +471,7 @@ release:deployment:tag:
 release:distribution:
   stage: release
   needs:
-    - build:linux
-    - build:windows
-    - build:macos
+    - build:platforms
     - integration:builds
     - integration:merge
     - release:deployment:tag

jest.config.js

@@ -38,7 +38,12 @@ module.exports = {
   },
   reporters: [
     'default',
-    ['jest-junit', { outputDirectory: '<rootDir>/tmp/junit' }],
+    ['jest-junit', {
+      outputDirectory: '<rootDir>/tmp/junit',
+      classNameTemplate: '{classname}',
+      titleTemplate: '{title}',
+      addFileAttribute: 'true',
+    }],
   ],
   collectCoverageFrom: ['src/**/*.{ts,tsx,js,jsx}', '!src/**/*.d.ts'],
   coverageReporters: ['text', 'cobertura'],

package-lock.json

@@ -49,13 +49,13 @@
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-import": "^2.26.0",
     "eslint-plugin-prettier": "^4.0.0",
-    "jest": "^27.2.5",
-    "jest-junit": "^13.2.0",
+    "jest": "^28.1.1",
+    "jest-junit": "^14.0.0",
     "jest-mock-process": "^1.4.0",
     "pkg": "5.6.0",
     "prettier": "^2.6.2",
     "rimraf": "^3.0.2",
-    "ts-jest": "^27.0.5",
+    "ts-jest": "^28.0.5",
     "ts-node": "^10.4.0",
     "tsconfig-paths": "^3.9.0",
     "typedoc": "^0.22.15",

package.json

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+
+shopt -s globstar
+shopt -s nullglob
+
+# Quote the heredoc to prevent shell expansion
+cat << "EOF"
+workflow:
+  rules:
+    # Disable merge request pipelines
+    - if: $CI_MERGE_REQUEST_ID
+      when: never
+    - when: always
+default:
+  interruptible: true
+variables:
+  GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}"
+  GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git"
+  GIT_SUBMODULE_STRATEGY: "recursive"
+  # Cache .npm
+  NPM_CONFIG_CACHE: "./tmp/npm"
+  # Prefer offline node module installation
+  NPM_CONFIG_PREFER_OFFLINE: "true"
+  # `ts-node` has its own cache
+  # It must use an absolute path, otherwise ts-node calls will CWD
+  TS_CACHED_TRANSPILE_CACHE: "${CI_PROJECT_DIR}/tmp/ts-node-cache"
+  TS_CACHED_TRANSPILE_PORTABLE: "true"
+  # Homebrew cache only used by macos runner
+  HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew"
+# Cached directories shared between jobs & pipelines per-branch per-runner
+cache:
+  key: $CI_COMMIT_REF_SLUG
+  paths:
+    - ./tmp/npm/
+    - ./tmp/ts-node-cache/
+    # Homebrew cache is only used by the macos runner
+    - ./tmp/Homebrew
+    # `jest` cache is configured in jest.config.js
+    - ./tmp/jest/
+stages:
+  - build       # Cross-platform library compilation, unit tests
+image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
+EOF
+
+printf "\n"
+
+# Using shards to optimise tests
+# In the future we can incorporate test durations rather than using
+# a static value for the parallel keyword
+
+cat << "EOF"
+build:linux:
+  stage: build
+  needs: []
+  parallel: 2
+  script:
+    - >
+        nix-shell --run '
+        npm run build --verbose;
+        npm test -- --ci --runInBand --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL;
+        '
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - ./tmp/junit/junit.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: ./tmp/coverage/cobertura-coverage.xml
+    paths:
+      # Only the build:linux preserves the dist
+      - ./dist
+  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+EOF
+
+printf "\n"
+
+cat << "EOF"
+build:windows:
+  stage: build
+  needs: []
+  parallel: 2
+  tags:
+    - windows
+  before_script:
+    - choco install nodejs --version=16.14.2 -y
+    - refreshenv
+  script:
+    - npm config set msvs_version 2019
+    - npm install --ignore-scripts
+    - $env:Path = "$(npm bin);" + $env:Path
+    - npm run build --verbose
+    - npm test -- --ci --runInBand --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - ./tmp/junit/junit.xml
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+EOF
+
+printf "\n"
+
+cat << "EOF"
+build:macos:
+  stage: build
+  needs: []
+  parallel: 2
+  tags:
+    - shared-macos-amd64
+  image: macos-11-xcode-12
+  variables:
+    HOMEBREW_NO_INSTALL_UPGRADE: "true"
+    HOMEBREW_NO_INSTALL_CLEANUP: "true"
+  before_script:
+    - eval "$(brew shellenv)"
+    - brew install node@16
+    - brew link --overwrite node@16
+    - hash -r
+  script:
+    - npm install --ignore-scripts
+    - export PATH="$(npm bin):$PATH"
+    - npm run build --verbose
+    - npm test -- --ci --runInBand --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - ./tmp/junit/junit.xml
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+EOF
+
+printf "\n"

scripts/build-pipelines.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+shopt -s globstar
+shopt -s nullglob
+
+# Quote the heredoc to prevent shell expansion
+cat << "EOF"
+workflow:
+  rules:
+    # Disable merge request pipelines
+    - if: $CI_MERGE_REQUEST_ID
+      when: never
+    - when: always
+default:
+  interruptible: true
+variables:
+  GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}"
+  GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git"
+  GIT_SUBMODULE_STRATEGY: "recursive"
+  # Cache .npm
+  NPM_CONFIG_CACHE: "./tmp/npm"
+  # Prefer offline node module installation
+  NPM_CONFIG_PREFER_OFFLINE: "true"
+  # `ts-node` has its own cache
+  # It must use an absolute path, otherwise ts-node calls will CWD
+  TS_CACHED_TRANSPILE_CACHE: "${CI_PROJECT_DIR}/tmp/ts-node-cache"
+  TS_CACHED_TRANSPILE_PORTABLE: "true"
+# Cached directories shared between jobs & pipelines per-branch per-runner
+cache:
+  key: $CI_COMMIT_REF_SLUG
+  paths:
+    - ./tmp/npm/
+    - ./tmp/ts-node-cache/
+    # `jest` cache is configured in jest.config.js
+    - ./tmp/jest/
+stages:
+  - check       # Linting, unit tests
+image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
+EOF
+
+printf "\n"
+
+# Using shards to optimise tests
+# In the future we can incorporate test durations rather than using
+# a static value for the parallel keyword
+cat << "EOF"
+check:test:
+  stage: check
+  needs: []
+  parallel: 2
+  script:
+    - >
+        nix-shell --run '
+        npm run build --verbose;
+        npm test -- --ci --runInBand --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL;
+        '
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - ./tmp/junit/junit.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: ./tmp/coverage/cobertura-coverage.xml
+  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Manually run on commits other than master and staging and ignore version commits
+    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+      when: manual
+EOF
+
+printf "\n"