diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml new file mode 100644 index 00000000..07b6fed9 --- /dev/null +++ b/.github/workflows/node.js.yml @@ -0,0 +1,375 @@ +name: Build + +on: [push, pull_request] + # push: + # branches: + # - main + # pull_request: + # branches: + # - main + +# Cancel in-progress runs for pull requests when developers push +# additional changes, and serialize builds in branches. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-concurrency-to-cancel-any-in-progress-job-or-run +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +# Note: if: success() is used in several jobs - +# this ensures that it only executes if all previous jobs succeeded. + +# if: steps.cache-node-modules.outputs.cache-hit != 'true' +# will skip running `yarn install` if it successfully fetched from cache + +jobs: + prettier: + name: Format with Prettier + runs-on: ubuntu-20.04 + timeout-minutes: 5 + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Run prettier with actionsx/prettier + uses: actionsx/prettier@v3 + with: + args: --check --loglevel=warn . + + doctoc: + name: Doctoc markdown files + runs-on: ubuntu-20.04 + timeout-minutes: 5 + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v44 + with: + files: | + docs/** + + - name: Install Node.js + if: steps.changed-files.outputs.any_changed == 'true' + uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: "yarn" + + - name: Install doctoc + run: yarn global add doctoc@2.2.1 + + - name: Run doctoc + if: steps.changed-files.outputs.any_changed == 'true' + run: yarn doctoc + + lint-helm: + name: Lint Helm chart + runs-on: ubuntu-20.04 + timeout-minutes: 5 + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v44 + with: + files: | + ci/helm-chart/** + + - name: Install helm + if: steps.changed-files.outputs.any_changed == 'true' + uses: azure/setup-helm@v3.5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install helm kubeval plugin + if: steps.changed-files.outputs.any_changed == 'true' + run: helm plugin install https://github.com/instrumenta/helm-kubeval + + - name: Lint Helm chart + if: steps.changed-files.outputs.any_changed == 'true' + run: helm kubeval ci/helm-chart + + lint-ts: + name: Lint TypeScript files + runs-on: ubuntu-20.04 + timeout-minutes: 5 + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v44 + with: + files: | + **/*.ts + **/*.js + files_ignore: | + lib/vscode/** + + - name: Install Node.js + if: steps.changed-files.outputs.any_changed == 'true' + uses: actions/setup-node@v4 + with: + node-version-file: .node-version + + - name: Fetch dependencies from cache + if: steps.changed-files.outputs.any_changed == 'true' + id: cache-node-modules + uses: actions/cache@v4 + with: + path: "**/node_modules" + key: yarn-build-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + yarn-build- + + - name: Install dependencies + if: steps.changed-files.outputs.any_changed == 'true' && steps.cache-node-modules.outputs.cache-hit != 'true' + run: SKIP_SUBMODULE_DEPS=1 yarn --frozen-lockfile + + - name: Lint TypeScript files + if: steps.changed-files.outputs.any_changed == 'true' + run: yarn lint:ts + + lint-actions: + name: Lint GitHub Actions + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Check workflow files + run: | + bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/7fdc9630cc360ea1a469eed64ac6d78caeda1234/scripts/download-actionlint.bash) + ./actionlint -color -shellcheck= -ignore "set-output" + shell: bash + + build: + name: Build code-editor + runs-on: ubuntu-20.04 + timeout-minutes: 60 + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + submodules: true + + - name: Install system dependencies + run: sudo apt update && sudo apt install -y libkrb5-dev + + - name: Install quilt + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: quilt + version: 1.0 + + - name: Patch Code + run: quilt push -a + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version-file: .node-version + + # node-gyp is missing in (at least) npm 9.8.1. + # TODO: Remove once we update to npm>=10? + - name: Install node-gyp + run: npm install -g node-gyp + + - name: Fetch dependencies from cache + id: cache-node-modules + uses: actions/cache@v4 + with: + path: "**/node_modules" + key: yarn-build-code-editor-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + yarn-build-code-editor- + + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: yarn --frozen-lockfile + + # Get Code's git hash. When this changes it means the content is + # different and we need to rebuild. + - name: Get latest vscode rev + id: vscode-rev + run: echo "rev=$(git rev-parse HEAD:./vscode)" >> $GITHUB_OUTPUT + + # We need to rebuild when we have a new version of Code, when any of + # the patches changed, or when the code-editor version changes (since + # it gets embedded into the code). Use VSCODE_CACHE_VERSION to + # force a rebuild. + - name: Fetch prebuilt Code package from cache + id: cache-vscode + uses: actions/cache@v4 + with: + path: lib/vscode-reh-web-* + key: vscode-reh-package-${{ secrets.VSCODE_CACHE_VERSION }}-${{ steps.vscode-rev.outputs.rev }}-${{ hashFiles('patches/*.diff', 'ci/build/build-vscode.sh') }} + + - name: Build code-editor + run: yarn build + + # The release package does not contain any native modules + # and is neutral to architecture/os/libc version. + - name: Create release package + run: yarn release + if: success() + + # https://github.com/actions/upload-artifact/issues/38 + - name: Compress release package + run: tar -czf package.tar.gz release + + - name: Upload npm package artifact + uses: actions/upload-artifact@v4 + with: + name: npm-package + path: ./package.tar.gz + + test-e2e: + name: Run e2e tests + needs: build + runs-on: ubuntu-20.04 + timeout-minutes: 25 + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Install system dependencies + run: sudo apt update && sudo apt install -y libkrb5-dev + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version-file: .node-version + + - name: Fetch dependencies from cache + id: cache-node-modules + uses: actions/cache@v4 + with: + path: "**/node_modules" + key: yarn-build-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + yarn-build- + + - name: Download npm package + uses: actions/download-artifact@v4 + with: + name: npm-package + + - name: Decompress npm package + run: tar -xzf package.tar.gz + + - name: Install release package dependencies + run: cd release && npm install --unsafe-perm --omit=dev + + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: SKIP_SUBMODULE_DEPS=1 yarn --frozen-lockfile + + - name: Install Playwright OS dependencies + run: | + ./test/node_modules/.bin/playwright install-deps + ./test/node_modules/.bin/playwright install + + - name: Run end-to-end tests + run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e + + - name: Upload test artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: failed-test-videos + path: ./test/test-results + + - name: Remove release packages and test artifacts + run: rm -rf ./release ./test/test-results + + test-e2e-proxy: + name: Run e2e tests behind proxy + needs: build + runs-on: ubuntu-20.04 + timeout-minutes: 25 + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Install system dependencies + run: sudo apt update && sudo apt install -y libkrb5-dev + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version-file: .node-version + + - name: Fetch dependencies from cache + id: cache-node-modules + uses: actions/cache@v4 + with: + path: "**/node_modules" + key: yarn-build-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + yarn-build- + + - name: Download npm package + uses: actions/download-artifact@v4 + with: + name: npm-package + + - name: Decompress npm package + run: tar -xzf package.tar.gz + + - name: Install release package dependencies + run: cd release && npm install --unsafe-perm --omit=dev + + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: SKIP_SUBMODULE_DEPS=1 yarn --frozen-lockfile + + - name: Install Playwright OS dependencies + run: | + ./test/node_modules/.bin/playwright install-deps + ./test/node_modules/.bin/playwright install + + - name: Cache Caddy + uses: actions/cache@v4 + id: caddy-cache + with: + path: | + ~/.cache/caddy + key: cache-caddy-2.5.2 + + - name: Install Caddy + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: steps.caddy-cache.outputs.cache-hit != 'true' + run: | + gh release download v2.5.2 --repo caddyserver/caddy --pattern "caddy_2.5.2_linux_amd64.tar.gz" + mkdir -p ~/.cache/caddy + tar -xzf caddy_2.5.2_linux_amd64.tar.gz --directory ~/.cache/caddy + + - name: Start Caddy + run: sudo ~/.cache/caddy/caddy start --config ./ci/Caddyfile + + - name: Run end-to-end tests + run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e:proxy --global-timeout 840000 + + - name: Stop Caddy + if: always() + run: sudo ~/.cache/caddy/caddy stop --config ./ci/Caddyfile + + - name: Upload test artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: failed-test-videos-proxy + path: ./test/test-results + + - name: Remove release packages and test artifacts + run: rm -rf ./release ./test/test-results \ No newline at end of file diff --git a/.node-version b/.node-version new file mode 100644 index 00000000..a58d2d2c --- /dev/null +++ b/.node-version @@ -0,0 +1 @@ +18.18.2 \ No newline at end of file diff --git a/ci/build/build-code-editor.sh b/ci/build/build-code-editor.sh new file mode 100755 index 00000000..8c192bee --- /dev/null +++ b/ci/build/build-code-editor.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Builds code-editor into out and the frontend into dist. + +# MINIFY controls whether a minified version of vscode is built. +MINIFY=${MINIFY-true} + +main() { + cd "$(dirname "${0}")/../.." + +echo `pwd` +echo `ls` + + pushd vscode + + # yarn --cwd vscode gulp vscode-reh-web-linux-x64-min + yarn gulp "vscode-reh-web-linux-x64${MINIFY:+-min}" + + # If out/node/entry.js does not already have the shebang, + # we make sure to add it and make it executable. +# if ! grep -q -m1 "^#!/usr/bin/env node" out/node/entry.js; then +# sed -i.bak "1s;^;#!/usr/bin/env node\n;" out/node/entry.js && rm out/node/entry.js.bak +# chmod +x out/node/entry.js +# fi +} + +main "$@" \ No newline at end of file diff --git a/ci/build/build-release.sh b/ci/build/build-release.sh new file mode 100644 index 00000000..773b9396 --- /dev/null +++ b/ci/build/build-release.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script requires vscode to be built with matching MINIFY. + +# MINIFY controls whether minified vscode is bundled. +MINIFY="${MINIFY-true}" + +# KEEP_MODULES controls whether the script cleans all node_modules requiring a yarn install +# to run first. +KEEP_MODULES="${KEEP_MODULES-0}" + +main() { + cd "$(dirname "${0}")/../.." + + source ./ci/lib.sh + + VSCODE_SRC_PATH="vscode" + VSCODE_OUT_PATH="$RELEASE_PATH/vscode" + + create_shrinkwraps + + mkdir -p "$RELEASE_PATH" + + bundle_vscode + +} + +bundle_vscode() { + mkdir -p "$VSCODE_OUT_PATH" + + local rsync_opts=() + if [[ ${DEBUG-} = 1 ]]; then + rsync_opts+=(-vh) + fi + + # Some extensions have a .gitignore which excludes their built source from the + # npm package so exclude any .gitignore files. + rsync_opts+=(--exclude .gitignore) + + # Exclude Node as we will add it ourselves for the standalone and will not + # need it for the npm package. + rsync_opts+=(--exclude /node) + + # Exclude Node modules. + if [[ $KEEP_MODULES = 0 ]]; then + rsync_opts+=(--exclude node_modules) + fi + + rsync "${rsync_opts[@]}" ./lib/vscode-reh-web-*/ "$VSCODE_OUT_PATH" + + # Use the package.json for the web/remote server. It does not have the right + # version though so pull that from the main package.json. + jq --slurp '.[0] * {version: .[1].version}' \ + "$VSCODE_SRC_PATH/remote/package.json" \ + "$VSCODE_SRC_PATH/package.json" > "$VSCODE_OUT_PATH/package.json" + + mv "$VSCODE_SRC_PATH/remote/npm-shrinkwrap.json" "$VSCODE_OUT_PATH/npm-shrinkwrap.json" + + # Include global extension dependencies as well. + rsync "$VSCODE_SRC_PATH/extensions/package.json" "$VSCODE_OUT_PATH/extensions/package.json" + mv "$VSCODE_SRC_PATH/extensions/npm-shrinkwrap.json" "$VSCODE_OUT_PATH/extensions/npm-shrinkwrap.json" + rsync "$VSCODE_SRC_PATH/extensions/postinstall.mjs" "$VSCODE_OUT_PATH/extensions/postinstall.mjs" +} + +create_shrinkwraps() { + # yarn.lock or package-lock.json files (used to ensure deterministic versions of dependencies) are + # not packaged when publishing to the NPM registry. + # To ensure deterministic dependency versions (even when code-server is installed with NPM), we create + # an npm-shrinkwrap.json file from the currently installed node_modules. This ensures the versions used + # from development (that the yarn.lock guarantees) are also the ones installed by end-users. + # These will include devDependencies, but those will be ignored when installing globally (for code-server), and + # because we use --omit=dev when installing vscode. + + # We first generate the shrinkwrap file for code-server itself - which is the current directory + create_shrinkwrap_keeping_yarn_lock + + # Then the shrinkwrap files for the bundled VSCode + pushd "$VSCODE_SRC_PATH/remote/" + create_shrinkwrap_keeping_yarn_lock + popd + + pushd "$VSCODE_SRC_PATH/extensions/" + create_shrinkwrap_keeping_yarn_lock + popd +} + +create_shrinkwrap_keeping_yarn_lock() { + # HACK@edvincent: Generating a shrinkwrap alters the yarn.lock which we don't want (with NPM URLs rather than the Yarn URLs) + # But to generate a valid shrinkwrap, it has to exist... So we copy it to then restore it + cp yarn.lock yarn.lock.temp + npm shrinkwrap + cp yarn.lock.temp yarn.lock + rm yarn.lock.temp +} + +main "$@" \ No newline at end of file diff --git a/ci/build/clean.sh b/ci/build/clean.sh new file mode 100755 index 00000000..656246fb --- /dev/null +++ b/ci/build/clean.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + rm -rf vscode/.build + +} + +main "$@" \ No newline at end of file diff --git a/ci/build/patch.sh b/ci/build/patch.sh new file mode 100755 index 00000000..8eabe47b --- /dev/null +++ b/ci/build/patch.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + quilt push -a + +} + +main "$@" \ No newline at end of file diff --git a/ci/dev/postinstall.sh b/ci/dev/postinstall.sh new file mode 100755 index 00000000..16c11de7 --- /dev/null +++ b/ci/dev/postinstall.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Install dependencies in $1. +install-deps() { + local args=(install) + if [[ ${CI-} ]]; then + args+=(--frozen-lockfile) + fi + if [[ "$1" == "vscode" ]]; then + args+=(--no-default-rc) + fi + # If there is no package.json then yarn will look upward and end up installing + # from the root resulting in an infinite loop (this can happen if you have not + # checked out the submodule yet for example). + if [[ ! -f "$1/package.json" ]]; then + echo "$1/package.json is missing; did you run git submodule update --init?" + exit 1 + fi + pushd "$1" + echo "Installing dependencies for $PWD" + yarn "${args[@]}" + popd +} + +main() { + cd "$(dirname "$0")/../.." + source ./ci/lib.sh + +# install-deps test +# install-deps test/e2e/extensions/test-extension + # We don't need these when running the integration tests + # so you can pass SKIP_SUBMODULE_DEPS + if [[ ! ${SKIP_SUBMODULE_DEPS-} ]]; then + install-deps vscode + fi +} + +main "$@" \ No newline at end of file diff --git a/ci/lib.sh b/ci/lib.sh new file mode 100644 index 00000000..672268a4 --- /dev/null +++ b/ci/lib.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +pushd() { + builtin pushd "$@" > /dev/null +} + +popd() { + builtin popd > /dev/null +} + +vscode_version() { + jq -r .version lib/vscode/package.json +} + +os() { + osname=$(uname | tr '[:upper:]' '[:lower:]') + case $osname in + linux) + # Alpine's ldd doesn't have a version flag but if you use an invalid flag + # (like --version) it outputs the version to stderr and exits with 1. + # TODO: Better to check /etc/os-release; see ../install.sh. + ldd_output=$(ldd --version 2>&1 || true) + if echo "$ldd_output" | grep -iq musl; then + osname="alpine" + fi + ;; + darwin) osname="macos" ;; + cygwin* | mingw*) osname="windows" ;; + esac + echo "$osname" +} + +arch() { + cpu="$(uname -m)" + case "$cpu" in + aarch64) cpu=arm64 ;; + x86_64) cpu=amd64 ;; + esac + echo "$cpu" +} + +rsync() { + command rsync -a --del "$@" +} + +ARCH="$(arch)" +export ARCH +OS=$(os) +export OS + +# RELEASE_PATH is the destination directory for the release from the root. +# Defaults to release +RELEASE_PATH="${RELEASE_PATH-release}" \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 00000000..6d58eb77 --- /dev/null +++ b/package.json @@ -0,0 +1,134 @@ +{ + "name": "code-editor", + "license": "MIT", + "version": "0.0.0", + "description": "Run VS Code on a remote server.", + "homepage": "https://github.com/coder/code-editor", + "bugs": { + "url": "https://github.com/coder/code-editor/issues" + }, + "repository": "https://github.com/coder/code-editor", + "scripts": { + "clean": "./ci/build/clean.sh", + "build": "./ci/build/build-code-editor.sh", + "compile": "yarn --cwd vscode compile", + "install": "yarn --cwd vscode install --frozen-lockfile", + "build:vscode": "./ci/build/build-vscode.sh", + "patch": "./ci/build/patch.sh", + "local-server": "./vscode/scripts/code-server.sh --launch", + "unittest": "./vscode/scripts/test.sh" + }, + "main": "out/node/entry.js", + "devDependencies": { + "@schemastore/package": "^0.0.10", + "@types/compression": "^1.7.3", + "@types/cookie-parser": "^1.4.4", + "@types/express": "^4.17.17", + "@types/http-proxy": "1.17.7", + "@types/js-yaml": "^4.0.6", + "@types/node": "^18.0.0", + "@types/pem": "^1.14.1", + "@types/proxy-from-env": "^1.0.1", + "@types/safe-compare": "^1.1.0", + "@types/semver": "^7.5.2", + "@types/trusted-types": "^2.0.4", + "@types/ws": "^8.5.5", + "@typescript-eslint/eslint-plugin": "^6.7.2", + "@typescript-eslint/parser": "^6.7.2", + "audit-ci": "^6.6.1", + "doctoc": "^2.2.1", + "eslint": "^8.49.0", + "eslint-config-prettier": "^9.0.0", + "eslint-import-resolver-typescript": "^3.6.0", + "eslint-plugin-import": "^2.28.1", + "eslint-plugin-prettier": "^5.0.0", + "prettier": "^3.0.3", + "prettier-plugin-sh": "^0.14.0", + "ts-node": "^10.9.1", + "typescript": "^5.2.2" + }, + "dependencies": { + "@coder/logger": "^3.0.1", + "argon2": "^0.31.1", + "compression": "^1.7.4", + "cookie-parser": "^1.4.6", + "env-paths": "^2.2.1", + "express": "5.0.0-beta.3", + "http-proxy": "^1.18.1", + "httpolyglot": "^0.1.2", + "i18next": "^23.5.1", + "js-yaml": "^4.1.0", + "limiter": "^2.1.0", + "pem": "^1.14.8", + "proxy-agent": "^6.3.1", + "qs": "6.12.1", + "rotating-file-stream": "^3.1.1", + "safe-buffer": "^5.2.1", + "safe-compare": "^1.1.4", + "semver": "^7.5.4", + "ws": "^8.14.2", + "xdg-basedir": "^4.0.0" + }, + "resolutions": { + "@types/node": "^18.0.0" + }, + "bin": { + "code-editor": "out/node/entry.js" + }, + "keywords": [ + "vscode", + "development", + "ide", + "coder", + "vscode-remote", + "browser-ide", + "remote-development" + ], + "engines": { + "node": "18" + }, + "jest": { + "transform": { + "^.+\\.ts$": "/test/node_modules/ts-jest" + }, + "testEnvironment": "node", + "testPathIgnorePatterns": [ + "/node_modules/", + "/lib/", + "/out/", + "test/e2e" + ], + "collectCoverage": true, + "collectCoverageFrom": [ + "/src/**/*.ts" + ], + "coverageDirectory": "/coverage", + "coverageReporters": [ + "json", + "json-summary", + "text", + "clover" + ], + "coveragePathIgnorePatterns": [ + "/out" + ], + "coverageThreshold": { + "global": { + "lines": 60 + } + }, + "modulePathIgnorePatterns": [ + "/release-packages", + "/release", + "/release-standalone", + "/release-npm-package", + "/release-gcp", + "/release-images", + "/lib" + ], + "moduleNameMapper": { + "^.+\\.(css|less)$": "/test/utils/cssStub.ts" + }, + "globalSetup": "/test/utils/globalUnitSetup.ts" + } + } \ No newline at end of file