diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74cbb88c0..67008d03d 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,8 +201,35 @@ jobs: retention-days: 14 if-no-files-found: ignore + # Release gate: block marketplace publish until the Install/Update Matrix + # reports green for this tag commit. The matrix workflow (triggered by the + # same tag push) sets the 'Install/Update Matrix' commit status; we poll it. + matrix-gate: + runs-on: ubuntu-latest + if: success() && startsWith(github.ref, 'refs/tags/') + timeout-minutes: 60 + steps: + - name: Wait for Install/Update Matrix status on this commit + uses: actions/github-script@v7 + with: + script: | + const ref = context.sha; + // Poll up to ~50 min (100 x 30s): the matrix builds 3 platforms + + // runs 12 cells, which on a cold cache can exceed 30 min. + for (let i = 0; i < 100; i++) { + const { data } = await github.rest.repos.getCombinedStatusForRef({ + owner: context.repo.owner, repo: context.repo.repo, ref }); + const s = data.statuses.find((x) => x.context === 'Install/Update Matrix'); + if (s && s.state === 'success') { core.info('Install/Update Matrix green'); return; } + if (s && s.state === 'failure') { core.setFailed('Install/Update Matrix failed — blocking release'); return; } + if (s && s.state === 'error') { core.setFailed('Install/Update Matrix status errored — blocking release'); return; } + core.info('waiting for Install/Update Matrix status...'); + await new Promise((r) => setTimeout(r, 30000)); + } + core.setFailed('Timed out waiting for Install/Update Matrix status'); + release-vsstudio-marketplace: - needs: [build, release-smoke] + needs: [build, release-smoke, matrix-gate] runs-on: ubuntu-latest if: success() && startsWith( github.ref, 'refs/tags/') strategy: @@ -256,7 +283,7 @@ jobs: text: "Tag: ${{ github.ref_name }} release to Visual Studio Marketplace status: ${{ needs.release-vsstudio-marketplace.result == 'success' && 'succeeded' || 'failed' }} :${{ needs.release-vsstudio-marketplace.result == 'success' && 'tada' || 'disappointed' }}:" release-openvsx-marketplace: - needs: build + needs: [build, matrix-gate] runs-on: ubuntu-latest if: success() && startsWith( github.ref, 'refs/tags/') strategy: diff --git a/.github/workflows/install-update-matrix.yml b/.github/workflows/install-update-matrix.yml new file mode 100644 index 000000000..c9b0c71c0 --- /dev/null +++ b/.github/workflows/install-update-matrix.yml @@ -0,0 +1,511 @@ +name: Install/Update Matrix + +on: + pull_request: + # "**" matches multi-segment base branches (e.g. feat/x); "*" would not. + branches: ["**"] + push: + tags: ["*"] + schedule: + - cron: "30 6 * * *" # daily, 06:30 UTC (offset from vsix-smoke's 06:00) + workflow_dispatch: {} + +permissions: + contents: read + pull-requests: write + statuses: write + +concurrency: + group: install-update-matrix-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Compute the cell matrix. Upgrade baselines are data-driven from the live + # App Insights version distribution when APPINSIGHTS_API_KEY is set; otherwise + # a hardcoded fallback is used (fork PRs / telemetry outage). + plan: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + baselines: ${{ steps.plan.outputs.baselines }} + source: ${{ steps.plan.outputs.source }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Build matrix (data-driven baselines) + id: plan + env: + APPINSIGHTS_API_KEY: ${{ secrets.APPINSIGHTS_API_KEY }} + run: | + python3 test-matrix/build-matrix.py >> "$GITHUB_OUTPUT" + # Split the per-version×OS impact map out to a file for the aggregate + # job (it's large + contains install-count business metrics, so it + # travels as an artifact rather than a step output that gets echoed). + grep '^impact=' "$GITHUB_OUTPUT" | tail -1 | sed 's/^impact=//' > impact.json + test -s impact.json || echo '{}' > impact.json + - name: Echo chosen baselines + run: echo "source=${{ steps.plan.outputs.source }} baselines=${{ steps.plan.outputs.baselines }}" + - name: Upload impact map + uses: actions/upload-artifact@v4 + with: + name: impact-map + path: impact.json + retention-days: 3 + + # Build one platform-specific VSIX per OS (native deps make a linux vsix + # uninstallable on mac/windows), each on its matching runner. + build: + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, target: linux-x64 } + - { os: macos-latest, target: darwin-arm64 } + - { os: windows-latest, target: win32-x64 } + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/common-build + with: + vsce-target: ${{ matrix.target }} + - name: Package VSIX + run: npx @vscode/vsce package --target ${{ matrix.target }} -o pu-${{ matrix.target }}.vsix + env: + NODE_OPTIONS: --max-old-space-size=8192 + - uses: actions/upload-artifact@v4 + with: + name: vsix-${{ matrix.target }} + path: pu-${{ matrix.target }}.vsix + retention-days: 3 + + vscode-cells: + needs: [plan, build] + strategy: + fail-fast: false + # Cells (incl. data-driven upgrade baselines) come from the plan job. + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + - name: Install deps + compile test suite + run: | + npm ci + npm run compile + - uses: actions/download-artifact@v4 + with: + name: vsix-${{ matrix.target }} + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Run cell + shell: bash + run: | + set -e + eval "$(bash test-matrix/setup-dbt-env.sh)" + vsixes=( pu-*.vsix ); VSIX="${vsixes[0]}" + OUT="result-${{ matrix.osl }}-${{ matrix.vscode }}-${{ matrix.mode }}-${{ matrix.from }}.json" + ARGS=(--mode "${{ matrix.mode }}" --target "$PWD/$VSIX" --vscode-version "${{ matrix.vscode }}" --out "$OUT") + [ -n "${{ matrix.from }}" ] && ARGS+=(--from "${{ matrix.from }}") + if [ "${{ matrix.osl }}" = "linux" ]; then + sudo apt-get update && sudo apt-get install -y xvfb + xvfb-run -a node test-matrix/vscode-cell.mjs "${ARGS[@]}" || true + else + node test-matrix/vscode-cell.mjs "${ARGS[@]}" || true + fi + - uses: actions/upload-artifact@v4 + with: + name: result-${{ matrix.osl }}-${{ matrix.vscode }}-${{ matrix.mode }}-${{ matrix.from }} + path: result-*.json + if-no-files-found: warn + + codeserver-cell: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: vsix-linux-x64 + - name: code-server fresh + upgrade + run: | + vsixes=( pu-*.vsix ); VSIX="${vsixes[0]}" + bash test-matrix/codeserver-cell.sh --mode fresh --vsix-file "$PWD/$VSIX" --out result-codeserver-fresh.json || true + bash test-matrix/codeserver-cell.sh --mode upgrade --from 0.61.4 --vsix-file "$PWD/$VSIX" --out result-codeserver-upgrade.json || true + - uses: actions/upload-artifact@v4 + with: + name: result-codeserver + path: result-codeserver-*.json + if-no-files-found: warn + + # Non-blocking fork lane: Cursor (Anysphere VSCode fork), Linux only. Drives + # the extracted Cursor AppImage headlessly (xvfb) — @vscode/test-electron can't + # target a fork. A failure here renders as ⚠️ and never blocks merge/release. + cursor-cell: + needs: [plan, build] + runs-on: ubuntu-latest + timeout-minutes: 20 + continue-on-error: true + strategy: + fail-fast: false + matrix: + # Fresh install + one representative upgrade (highest live baseline). + include: + - { mode: fresh, from: "" } + - { mode: upgrade, from: "" } # 'from' filled from plan baselines at run time + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + - name: Install deps + compile test suite + run: | + npm ci + npm run compile + - uses: actions/download-artifact@v4 + with: + name: vsix-linux-x64 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Run Cursor cell + shell: bash + env: + BASELINES: ${{ needs.plan.outputs.baselines }} + run: | + set -e + sudo apt-get update && sudo apt-get install -y xvfb + eval "$(bash test-matrix/setup-dbt-env.sh)" + MODE="${{ matrix.mode }}" + # Representative upgrade baseline = highest version from the plan job. + FROM="" + if [ "$MODE" = "upgrade" ]; then + FROM="$(printf '%s' "$BASELINES" | tr ',' '\n' | tail -1)" + fi + OUT="result-cursor-${MODE}-${FROM}.json" + FROM_JSON="$([ -n "$FROM" ] && echo "\"$FROM\"" || echo null)" + # Write an ACCURATE failure result (real reason + measured duration), not a + # hardcoded "cell wedged"/720s placeholder, so the board's reason is honest. + emit_fail() { # $1=reason $2=duration_seconds + local r; r="$(printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g')" + printf '{"runtime":"cursor","os":"linux","scenario":"%s","from":%s,"to":"pr-build","install_ok":false,"deps_resolved":{},"activation_ok":false,"dbt_flow_ok":false,"status":"fail","reason":"%s","duration_s":%s,"log_artifact":"%s"}\n' \ + "$MODE" "$FROM_JSON" "$r" "$2" "$OUT" > "$OUT" + } + # Provision runs in $(...), so `set -e` can't see its exit code — check it + # explicitly and capture the real stderr reason (download/extract/launcher + # error) instead of silently continuing with an empty binary path. + t0=$SECONDS + if ! prov="$(bash test-matrix/provision/cursor.sh 2>/tmp/cursor-prov.err)"; then + reason="$(grep -E 'FAIL' /tmp/cursor-prov.err | tail -1)" + [ -z "$reason" ] && reason="$(tail -1 /tmp/cursor-prov.err 2>/dev/null)" + emit_fail "provision failed: ${reason:-unknown error}" "$((SECONDS - t0))" + echo "=== RESULT_JSON ==="; cat "$OUT"; cat /tmp/cursor-prov.err >&2 || true + exit 0 + fi + eval "$prov" # sets CURSOR_BIN, CURSOR_VERSION + echo "Provisioned Cursor ${CURSOR_VERSION:-?} at ${CURSOR_BIN:-?}" + vsixes=( pu-*.vsix ); VSIX="${vsixes[0]}" + ARGS=(--bin "$CURSOR_BIN" --mode "$MODE" --target "$PWD/$VSIX" --out "$OUT") + [ -n "$FROM" ] && ARGS+=(--from "$FROM") + # Hard outer guard: `timeout -s KILL` caps wall time even if the whole + # Cursor process tree wedges (the node-level backstop can't help if a + # synchronous child blocks the event loop). After it returns, reap any + # straggler Cursor/Xvfb processes so nothing keeps the job alive. + t1=$SECONDS + timeout -s KILL 12m xvfb-run -a node test-matrix/cursor-cell.mjs "${ARGS[@]}" || true + pkill -9 -f 'squashfs-root|cursor|Xvfb' 2>/dev/null || true + if [ ! -f "$OUT" ]; then + emit_fail "cell produced no result within 12m (process wedged or was killed)" "$((SECONDS - t1))" + fi + echo "=== RESULT_JSON ==="; cat "$OUT" + - uses: actions/upload-artifact@v4 + with: + name: result-cursor-${{ matrix.mode }} + path: result-cursor-*.json + if-no-files-found: warn + - name: Upload Cursor console log (diagnostics) + if: always() + uses: actions/upload-artifact@v4 + with: + name: cursor-console-${{ matrix.mode }} + path: cursor-console-*.log + if-no-files-found: ignore + + # Non-blocking fork lane: Windsurf (Codeium VSCode fork), Linux only. Same + # mechanism as Cursor — install via Open VSX download+unzip (the bundled CLI + # is unreliable headless), launch the extracted binary under xvfb, scan stdout + # for the activation marker. Reuses cursor-cell.mjs via --runtime windsurf. + windsurf-cell: + needs: [plan, build] + runs-on: ubuntu-latest + timeout-minutes: 20 + continue-on-error: true + strategy: + fail-fast: false + matrix: + include: + - { mode: fresh, from: "" } + - { mode: upgrade, from: "" } # 'from' filled from plan baselines at run time + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + - name: Install deps + compile test suite + run: | + npm ci + npm run compile + - uses: actions/download-artifact@v4 + with: + name: vsix-linux-x64 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Run Windsurf cell + shell: bash + env: + BASELINES: ${{ needs.plan.outputs.baselines }} + run: | + set -e + sudo apt-get update && sudo apt-get install -y xvfb + eval "$(bash test-matrix/setup-dbt-env.sh)" + MODE="${{ matrix.mode }}" + FROM="" + if [ "$MODE" = "upgrade" ]; then + FROM="$(printf '%s' "$BASELINES" | tr ',' '\n' | tail -1)" + fi + OUT="result-windsurf-${MODE}-${FROM}.json" + FROM_JSON="$([ -n "$FROM" ] && echo "\"$FROM\"" || echo null)" + # Write an ACCURATE failure result (real reason + measured duration), not a + # hardcoded "cell wedged"/720s placeholder, so the board's reason is honest. + emit_fail() { # $1=reason $2=duration_seconds + local r; r="$(printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g')" + printf '{"runtime":"windsurf","os":"linux","scenario":"%s","from":%s,"to":"pr-build","install_ok":false,"deps_resolved":{},"activation_ok":false,"dbt_flow_ok":false,"status":"fail","reason":"%s","duration_s":%s,"log_artifact":"%s"}\n' \ + "$MODE" "$FROM_JSON" "$r" "$2" "$OUT" > "$OUT" + } + # Provision runs in $(...), so `set -e` can't see its exit code — check it + # explicitly and capture the real stderr reason. This is exactly the failure + # mode that hid the Windsurf->Devin repackage: an empty WINDSURF_BIN slid + # through and the cell reported a fake "wedged/720s" instead of the launcher + # error. Now a provision failure produces an honest, fast result. + t0=$SECONDS + if ! prov="$(bash test-matrix/provision/windsurf.sh 2>/tmp/windsurf-prov.err)"; then + reason="$(grep -E 'FAIL' /tmp/windsurf-prov.err | tail -1)" + [ -z "$reason" ] && reason="$(tail -1 /tmp/windsurf-prov.err 2>/dev/null)" + emit_fail "provision failed: ${reason:-unknown error}" "$((SECONDS - t0))" + echo "=== RESULT_JSON ==="; cat "$OUT"; cat /tmp/windsurf-prov.err >&2 || true + exit 0 + fi + eval "$prov" # sets WINDSURF_BIN, WINDSURF_VERSION + echo "Provisioned Windsurf ${WINDSURF_VERSION:-?} at ${WINDSURF_BIN:-?}" + vsixes=( pu-*.vsix ); VSIX="${vsixes[0]}" + ARGS=(--runtime windsurf --bin "$WINDSURF_BIN" --mode "$MODE" --target "$PWD/$VSIX" --out "$OUT") + [ -n "$FROM" ] && ARGS+=(--from "$FROM") + t1=$SECONDS + timeout -s KILL 12m xvfb-run -a node test-matrix/cursor-cell.mjs "${ARGS[@]}" || true + pkill -9 -f 'Windsurf|windsurf|Devin|devin|Xvfb' 2>/dev/null || true + if [ ! -f "$OUT" ]; then + emit_fail "cell produced no result within 12m (process wedged or was killed)" "$((SECONDS - t1))" + fi + echo "=== RESULT_JSON ==="; cat "$OUT" + - uses: actions/upload-artifact@v4 + with: + name: result-windsurf-${{ matrix.mode }} + path: result-windsurf-*.json + if-no-files-found: warn + - name: Upload Windsurf console log (diagnostics) + if: always() + uses: actions/upload-artifact@v4 + with: + name: windsurf-console-${{ matrix.mode }} + path: windsurf-console-*.log + if-no-files-found: ignore + + # Non-blocking fork lane: Kiro (AWS agentic VSCode fork), Linux only. Same + # mechanism as Cursor/Windsurf via --runtime kiro. + kiro-cell: + needs: [plan, build] + runs-on: ubuntu-latest + timeout-minutes: 20 + continue-on-error: true + strategy: + fail-fast: false + matrix: + include: + - { mode: fresh, from: "" } + - { mode: upgrade, from: "" } + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + - name: Install deps + compile test suite + run: | + npm ci + npm run compile + - uses: actions/download-artifact@v4 + with: + name: vsix-linux-x64 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Run Kiro cell + shell: bash + env: + BASELINES: ${{ needs.plan.outputs.baselines }} + run: | + set -e + sudo apt-get update && sudo apt-get install -y xvfb + eval "$(bash test-matrix/setup-dbt-env.sh)" + MODE="${{ matrix.mode }}" + FROM="" + if [ "$MODE" = "upgrade" ]; then + FROM="$(printf '%s' "$BASELINES" | tr ',' '\n' | tail -1)" + fi + OUT="result-kiro-${MODE}-${FROM}.json" + FROM_JSON="$([ -n "$FROM" ] && echo "\"$FROM\"" || echo null)" + # Write an ACCURATE failure result (real reason + measured duration), not a + # hardcoded "cell wedged"/720s placeholder, so the board's reason is honest. + emit_fail() { # $1=reason $2=duration_seconds + local r; r="$(printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g')" + printf '{"runtime":"kiro","os":"linux","scenario":"%s","from":%s,"to":"pr-build","install_ok":false,"deps_resolved":{},"activation_ok":false,"dbt_flow_ok":false,"status":"fail","reason":"%s","duration_s":%s,"log_artifact":"%s"}\n' \ + "$MODE" "$FROM_JSON" "$r" "$2" "$OUT" > "$OUT" + } + # Provision runs in $(...), so `set -e` can't see its exit code — check it + # explicitly and capture the real stderr reason instead of silently + # continuing with an empty binary path. + t0=$SECONDS + if ! prov="$(bash test-matrix/provision/kiro.sh 2>/tmp/kiro-prov.err)"; then + reason="$(grep -E 'FAIL' /tmp/kiro-prov.err | tail -1)" + [ -z "$reason" ] && reason="$(tail -1 /tmp/kiro-prov.err 2>/dev/null)" + emit_fail "provision failed: ${reason:-unknown error}" "$((SECONDS - t0))" + echo "=== RESULT_JSON ==="; cat "$OUT"; cat /tmp/kiro-prov.err >&2 || true + exit 0 + fi + eval "$prov" # sets KIRO_BIN, KIRO_VERSION + echo "Provisioned Kiro ${KIRO_VERSION:-?} at ${KIRO_BIN:-?}" + vsixes=( pu-*.vsix ); VSIX="${vsixes[0]}" + ARGS=(--runtime kiro --bin "$KIRO_BIN" --mode "$MODE" --target "$PWD/$VSIX" --out "$OUT") + [ -n "$FROM" ] && ARGS+=(--from "$FROM") + t1=$SECONDS + timeout -s KILL 12m xvfb-run -a node test-matrix/cursor-cell.mjs "${ARGS[@]}" || true + pkill -9 -f 'Kiro|kiro|Xvfb' 2>/dev/null || true + if [ ! -f "$OUT" ]; then + emit_fail "cell produced no result within 12m (process wedged or was killed)" "$((SECONDS - t1))" + fi + echo "=== RESULT_JSON ==="; cat "$OUT" + - uses: actions/upload-artifact@v4 + with: + name: result-kiro-${{ matrix.mode }} + path: result-kiro-*.json + if-no-files-found: warn + - name: Upload Kiro console log (diagnostics) + if: always() + uses: actions/upload-artifact@v4 + with: + name: kiro-console-${{ matrix.mode }} + path: kiro-console-*.log + if-no-files-found: ignore + + aggregate: + needs: [plan, vscode-cells, codeserver-cell, cursor-cell, windsurf-cell, kiro-cell] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: result-* + path: results + merge-multiple: true + # Per-version×OS impact map from the plan job (best-effort: the board still + # renders without it, just without the "% users impacted" figures). + - uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: impact-map + path: impact + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Aggregate + id: agg + run: | + set +e + IMPACT_ARG="" + [ -f impact/impact.json ] && IMPACT_ARG="--impact-file impact/impact.json" + python3 test-matrix/aggregate.py --results-dir results --out-dir agg \ + --target "${{ github.event_name == 'pull_request' && 'pr-build' || 'latest' }}" \ + --trigger "${{ github.event_name }}" $IMPACT_ARG + code=$? + # Render the board on the run's Summary page (formatted tables, no + # download). Runs whether the gate passes or fails so a red run still + # shows its board. Markdown is GitHub-flavored, so the | tables render. + if [ -f agg/matrix.md ]; then + cat agg/matrix.md >> "$GITHUB_STEP_SUMMARY" + else + echo "## VSIX Install + Update Matrix" >> "$GITHUB_STEP_SUMMARY" + echo ":x: no board produced (no result files)." >> "$GITHUB_STEP_SUMMARY" + fi + echo "blocking_failed=$([ $code -ne 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT" + exit 0 + - uses: actions/upload-artifact@v4 + with: + name: matrix-report + path: agg/ + - name: Post sticky PR comment + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const body = '\n' + fs.readFileSync('agg/matrix.md', 'utf8'); + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number }); + const existing = comments.find(c => c.body.includes('')); + if (existing) await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body }); + else await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, body }); + - name: Set commit status (blocking gate) + if: github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/') + uses: actions/github-script@v7 + with: + script: | + const pr = context.payload.pull_request; + if (pr && pr.head.repo.full_name !== context.payload.repository.full_name) { + core.info('fork PR — cannot set commit status, skipping'); + return; + } + const failed = '${{ steps.agg.outputs.blocking_failed }}' === 'true'; + const sha = pr ? pr.head.sha : context.sha; + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, repo: context.repo.repo, + sha, + state: failed ? 'failure' : 'success', + description: failed ? 'Blocking-lane cell failed' : 'Install/Update matrix green', + context: 'Install/Update Matrix', + target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + }); + # Slack notification intentionally REMOVED until the matrix is thoroughly + # tested and a dedicated channel is approved. Do NOT reuse the repo's shared + # SLACK_WEBHOOK_URL secret (a prod alerts channel). To add it back later: + # 1) create a dedicated webhook + set a MATRIX_SLACK_WEBHOOK repo secret + # 2) add a step that posts agg/slack.json to ${{ secrets.MATRIX_SLACK_WEBHOOK }} + # ONLY (never a fallback to SLACK_WEBHOOK_URL). + - name: Fail job if blocking lane failed + if: steps.agg.outputs.blocking_failed == 'true' + run: | + echo "::error::Blocking-lane matrix cell(s) failed" + exit 1 diff --git a/.gitignore b/.gitignore index 667c53f9b..7731d2050 100755 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,8 @@ docker-setup/.env # GitHub issues dashboard cache monitoring/github_issues/.cache/ + +# Install/update matrix local env +.matrix-venv/ +# Generated telemetry snapshot (install counts) — never commit (public repo) +test-matrix/active-versions.json diff --git a/docker-setup/docker-compose.yml b/docker-setup/docker-compose.yml index c273effeb..80a8cec9f 100644 --- a/docker-setup/docker-compose.yml +++ b/docker-setup/docker-compose.yml @@ -9,6 +9,13 @@ services: - PORT=3001 volumes: - ..:/home/coder/extension-src:ro - - ${DBT_PROJECT_PATH:-/home/coder/jaffle-shop-duckdb}:/home/coder/project + # Mount your own dbt project by setting DBT_PROJECT_PATH to an ABSOLUTE host + # path. When it is unset we mount the compose dir (".") purely as a harmless + # placeholder so the bind resolves on every OS — a non-existent default host + # path like "/home/coder/jaffle-shop-duckdb" is rejected by macOS Docker + # Desktop ("mounts denied"). The placeholder has no dbt_project.yml, so + # start-code-server.sh falls back to the built-in jaffle project it seeds + # from test-fixtures. + - ${DBT_PROJECT_PATH:-.}:/home/coder/project - ${ALTIMATE_CREDENTIALS_FILE:-/dev/null}:/home/coder/.altimate-host/altimate.json:ro restart: unless-stopped diff --git a/docker-setup/start-code-server.sh b/docker-setup/start-code-server.sh index 2b6feb890..d8183a239 100755 --- a/docker-setup/start-code-server.sh +++ b/docker-setup/start-code-server.sh @@ -110,11 +110,15 @@ if [ -f "$HOME/.altimate-host/altimate.json" ]; then " fi -# Determine project directory -if [ -d "/home/coder/project" ] && [ "$(ls -A /home/coder/project 2>/dev/null)" ]; then +# Determine project directory. Use the mounted project only if it is a real dbt +# project (has dbt_project.yml) — a bare non-empty check would wrongly pick the +# placeholder dir mounted when DBT_PROJECT_PATH is unset. Otherwise fall back to +# the built-in jaffle-shop-duckdb seeded above (note: HYPHENS, matching the dir +# created by the seeding loop — the old underscore name pointed at nothing). +if [ -f "/home/coder/project/dbt_project.yml" ]; then PROJECT_DIR="/home/coder/project" else - PROJECT_DIR="/home/coder/jaffle_shop_duckdb" + PROJECT_DIR="/home/coder/jaffle-shop-duckdb" fi # Start code-server with the project open diff --git a/package.json b/package.json index 55e3fd324..03e43c8b1 100644 --- a/package.json +++ b/package.json @@ -1209,7 +1209,10 @@ "compile": "tsc -p ./", "docker:deploy": "./docker-setup/deploy.sh", "docker:logs": "cd docker-setup && docker compose logs -f || docker-compose logs -f", - "docker:stop": "cd docker-setup && docker compose down || docker-compose down" + "docker:stop": "cd docker-setup && docker compose down || docker-compose down", + "matrix:vscode": "node test-matrix/vscode-cell.mjs", + "matrix:aggregate": "python3 test-matrix/aggregate.py", + "matrix:test-aggregate": "python3 -m pytest tests/matrix/ -q" }, "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.2", diff --git a/src/test/matrix/activation.test.ts b/src/test/matrix/activation.test.ts new file mode 100644 index 000000000..9f40b743b --- /dev/null +++ b/src/test/matrix/activation.test.ts @@ -0,0 +1,98 @@ +import * as assert from "assert"; +import * as fs from "fs"; +import * as path from "path"; +import * as vscode from "vscode"; + +const EXTENSION_ID = "innoverio.vscode-dbt-power-user"; +// A command this extension contributes (proves contributions loaded), from package.json contributes.commands. +const STABLE_COMMAND = "dbtPowerUser.openInsights"; +// Success/failure markers emitted to the file-backed "Log - dbt" LogOutputChannel and the exthost log. +const INIT_OK = "Initialized dbt project"; +const INIT_FAIL = "Unable to register dbt project"; + +function sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)); +} + +function readAllLogs(uddDir: string): string { + // VSCode writes LogOutputChannels + console output under /logs/**. + const out: string[] = []; + const stack = [path.join(uddDir, "logs")]; + while (stack.length) { + const dir = stack.pop()!; + let entries: fs.Dirent[] = []; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + continue; + } + for (const e of entries) { + const p = path.join(dir, e.name); + if (e.isDirectory()) { + stack.push(p); + } else if (e.name.endsWith(".log")) { + try { + out.push(fs.readFileSync(p, "utf8")); + } catch { + /* ignore unreadable rotating log */ + } + } + } + } + return out.join("\n"); +} + +suite("Matrix: installed VSIX activation + dbt project init", function () { + this.timeout(120_000); + + test("extension is installed and activates", async function () { + const ext = vscode.extensions.getExtension(EXTENSION_ID); + assert.ok( + ext, + `${EXTENSION_ID} should be installed in the test extensions-dir`, + ); + await ext!.activate(); + assert.strictEqual(ext!.isActive, true, "extension should be active"); + }); + + test("contributed command is registered", async function () { + const cmds = await vscode.commands.getCommands(true); + assert.ok( + cmds.includes(STABLE_COMMAND), + `command ${STABLE_COMMAND} should be registered`, + ); + }); + + test("dbt fixture workspace is open", function () { + const folders = vscode.workspace.workspaceFolders ?? []; + assert.ok(folders.length > 0, "a workspace folder should be open"); + assert.ok( + folders.some((f) => + fs.existsSync(path.join(f.uri.fsPath, "dbt_project.yml")), + ), + "the open workspace should contain dbt_project.yml", + ); + }); + + test("dbt project initializes (log shows 'Initialized dbt project')", async function () { + const uddDir = process.env.MATRIX_UDD; + assert.ok(uddDir, "MATRIX_UDD env must point at the --user-data-dir"); + const deadline = Date.now() + 90_000; + let logs = ""; + while (Date.now() < deadline) { + logs = readAllLogs(uddDir!); + if (logs.includes(INIT_FAIL)) { + assert.fail( + `dbt project registration failed: found '${INIT_FAIL}' in logs`, + ); + } + if (logs.includes(INIT_OK)) { + return; // success + } + await sleep(2000); + } + assert.fail( + `did not observe '${INIT_OK}' within 90s (dbt project did not initialize)`, + ); + }); +}); diff --git a/src/test/matrix/index.ts b/src/test/matrix/index.ts new file mode 100644 index 000000000..f5e6c592d --- /dev/null +++ b/src/test/matrix/index.ts @@ -0,0 +1,19 @@ +import { glob } from "glob"; +import Mocha from "mocha"; +import * as path from "path"; + +export async function run(): Promise { + const mocha = new Mocha({ ui: "tdd", timeout: 120_000, color: true }); + const testsRoot = path.resolve(__dirname); + const files = await glob("**/*.test.js", { cwd: testsRoot }); + for (const file of files) { + mocha.addFile(path.resolve(testsRoot, file)); + } + return new Promise((resolve, reject) => { + mocha.run((failures) => + failures > 0 + ? reject(new Error(`${failures} test(s) failed.`)) + : resolve(), + ); + }); +} diff --git a/src/test/suite/repro/crossIdeAppName.repro.test.ts b/src/test/suite/repro/crossIdeAppName.repro.test.ts new file mode 100644 index 000000000..b519bab49 --- /dev/null +++ b/src/test/suite/repro/crossIdeAppName.repro.test.ts @@ -0,0 +1,110 @@ +import { + afterEach, + beforeEach, + describe, + expect, + jest, + test, +} from "@jest/globals"; +import * as vscode from "vscode"; +import { isCursor } from "../../../mcp/utils"; + +/** + * Cross-IDE family reproduction — behaviour keyed on `vscode.env.appName`. + * + * App Insights telemetry tags every event with + * customDimensions.ide = vscode.env.appName (src/telemetry/index.ts:12) + * and the per-IDE error totals (VSCode 16.2M, Cursor 2.16M, Windsurf 104k, + * Kiro 5.3k, code-server 6k, …) come from that single field. The extension + * therefore runs inside several VS Code forks, each reporting a different + * `vscode.env.appName`. + * + * The only branch in the extension `src/` tree (outside tests) that *diverges* + * on that value is `isCursor()`: + * + * // src/mcp/utils.ts:3-4 + * export const isCursor = (): boolean => { + * return env.appName === "Cursor"; + * }; + * + * This gate is consumed when wiring the MCP "data lookup" feature, whose + * user-facing copy is hard-coded to mention "Cursor" + * (src/mcp/index.ts:72). The gate is an EXACT, case-sensitive string match, + * so it is `true` only for the literal appName "Cursor" and `false` for every + * other host — including the other forks the telemetry proves are in the wild + * (Windsurf, Kiro, code-server, VSCode-Insiders, Antigravity) and including + * Cursor reported with any non-canonical casing/spacing. + * + * IMPORTANT about the test harness: `src/mcp/utils.ts` imports `env` as a + * *named* import (`import { env } from "vscode"`), but the jest vscode mock + * (src/test/mock/vscode.ts) does NOT export `env`. So `env` is `undefined` + * in the test runtime and `env.appName` would throw `TypeError` unless a test + * supplies `vscode.env`. We mirror the established pattern from + * telemetryService.test.ts: assign `(vscode as any).env = {...}` before each + * call. Under ts-jest's CommonJS interop the named import is read at use-site + * (`vscode_1.env.appName`), so this assignment is visible to the real code. + * + * Conclusion: `isCursor` is WORKING AS INTENDED — it is a deliberate, + * Cursor-only feature gate, not a bug. These tests PIN its exact divergent + * behaviour across fork appName values so any future broadening (e.g. adding + * Windsurf/Kiro support, or making it case-insensitive) is a visible, + * intentional change. is_real_bug = false. + */ +describe("Cross-IDE: isCursor() keyed on vscode.env.appName", () => { + const setAppName = (appName: string | undefined) => { + (vscode as { env?: unknown }).env = { appName }; + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + delete (vscode as { env?: unknown }).env; + }); + + test("returns true only for the canonical Cursor appName", () => { + setAppName("Cursor"); + expect(isCursor()).toBe(true); + }); + + test("returns false for stock Visual Studio Code", () => { + setAppName("Visual Studio Code"); + expect(isCursor()).toBe(false); + }); + + test("returns false for VSCode-Insiders (Visual Studio Code - Insiders)", () => { + setAppName("Visual Studio Code - Insiders"); + expect(isCursor()).toBe(false); + }); + + test("returns false for the Windsurf fork", () => { + setAppName("Windsurf"); + expect(isCursor()).toBe(false); + }); + + test("returns false for the Kiro fork", () => { + setAppName("Kiro"); + expect(isCursor()).toBe(false); + }); + + test("returns false for code-server (browser host)", () => { + setAppName("code-server"); + expect(isCursor()).toBe(false); + }); + + test("returns false for the Antigravity fork", () => { + setAppName("Antigravity"); + expect(isCursor()).toBe(false); + }); + + test("is case-sensitive: lowercase 'cursor' is NOT matched (pins current strict behaviour)", () => { + setAppName("cursor"); + expect(isCursor()).toBe(false); + }); + + test("does NOT substring-match: 'Cursor (Anysphere)' is NOT matched (pins current exact-equality behaviour)", () => { + setAppName("Cursor (Anysphere)"); + expect(isCursor()).toBe(false); + }); +}); diff --git a/src/test/suite/repro/dbtCloudJsonParse.repro.test.ts b/src/test/suite/repro/dbtCloudJsonParse.repro.test.ts new file mode 100644 index 000000000..9d41eefe1 --- /dev/null +++ b/src/test/suite/repro/dbtCloudJsonParse.repro.test.ts @@ -0,0 +1,246 @@ +import { + DBTCommandFactory, + DBTFusionCommandProjectIntegration, +} from "@altimateai/dbt-integration"; +import { beforeEach, describe, expect, jest, test } from "@jest/globals"; + +/** + * REPRODUCTION: "Unexpected end of JSON input" via per-line JSON.parse + * ------------------------------------------------------------------ + * Production family (App Insights, 30d, innoverio.vscode-dbt-power-user): + * "Unexpected end of JSON input" -> catchAllError 131,317 + * + * REAL root cause (verified against the bundled npm package + * node_modules/@altimateai/dbt-integration, version 0.3.2 — source map points + * to src/dbtFusionCommandIntegration.ts:517): + * + * DBTFusionCommandProjectIntegration.executeSQL() runs + * `dbt show --inline --limit N --output json --log-format json` + * and, inside the QueryExecution callback, parses stdout with: + * + * let d = i.trim().split("\n").map(h => JSON.parse(h.trim())); // <-- BUG + * + * There is NO per-line try/catch and NO empty-line filter here (unlike the + * sibling Cloud/Core paths in the same file, which wrap each parse in + * `try { ... } catch {}` and filter falsy results). So any line that is + * blank after trimming reaches JSON.parse("") and throws V8's exact message + * "Unexpected end of JSON input". `i.trim()` strips only the OUTER + * whitespace, so these all crash the whole batch: + * - completely empty stdout ("") + * - whitespace-only stdout (" ") + * - an INTERIOR blank line between two valid JSON objects + * (common in dbt's --log-format json debug stream) + * The raw SyntaxError propagates UNWRAPPED through + * QueryExecution.executeQuery(), which is why App Insights logs the bare + * field string. + * + * (A *truncated* content line like '{"a":1' throws a DIFFERENT V8 message — + * "Expected ',' or '}' after property value..." — so it is NOT part of THIS + * family. The 131k "Unexpected end of JSON input" hits come specifically + * from blank/empty lines reaching JSON.parse(""). That distinction is pinned + * below so the reproduction is precise, not hand-wavy.) + * + * This test drives the REAL, unmodified library code. No `dbt` subprocess is + * spawned: we inject a `cliDBTCommandExecutionStrategyFactory` whose + * strategy.execute() returns crafted stdout — exactly what the spawned + * `dbt show` process would have streamed back. Everything downstream + * (trim/split/JSON.parse, preview extraction, tabular transform) is the + * library's own code. The parser is NOT re-implemented here. + * + * Every expected value below was confirmed empirically against the real + * package before being written. + */ + +// V8's exact field message recorded 131,317 times in App Insights. +const RAW_JSON_FIELD_ERROR = "Unexpected end of JSON input"; + +const dbtConfiguration = { + getQueryTemplate: () => "select * from ({query}) as query limit {limit}", + getQueryLimit: () => 500, +} as any; + +const terminal = { + show: async () => {}, + log: () => {}, + trace: () => {}, + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, + dispose: () => {}, +} as any; + +const pythonEnvironment = { + pythonPath: "python3", + getEnvironmentVariables: () => ({}), +} as any; + +const pythonEnvironmentProvider = { + getCurrentEnvironment: () => pythonEnvironment, + onEnvironmentChanged: () => () => {}, +} as any; + +/** + * Build a real DBTFusionCommandProjectIntegration whose command execution + * returns `stdout`/`stderr` verbatim (no `dbt` binary involved). + */ +function makeFusionIntegration(stdout: string, stderr = "") { + const execute = jest.fn(async () => ({ + stdout, + stderr, + fullOutput: stdout + stderr, + })); + // wrapCommand() calls this factory with (projectRoot, dbtPath) and uses the + // returned strategy as the command's execution strategy. + const strategyFactory = (_path: string, _dbtPath: string) => ({ execute }); + + const commandFactory = new DBTCommandFactory(dbtConfiguration); + + const instance = new DBTFusionCommandProjectIntegration( + /* _executionInfrastructure */ {} as any, + /* dbtCommandFactory */ commandFactory, + /* cliDBTCommandExecutionStrategyFactory */ strategyFactory as any, + /* pythonEnvironment */ pythonEnvironment, + /* _pythonEnvironmentProvider */ pythonEnvironmentProvider, + /* terminal */ terminal, + /* projectRoot */ "/tmp/fusion-proj", + /* projectConfigDiagnostics */ [], + /* deferConfig */ { + deferToProduction: false, + favorState: false, + } as any, + /* onDiagnosticsChanged */ () => {}, + ); + + // The Fusion constructor does not assign these; wrapCommand()/executeSQL() + // read them. They are plain (non-frozen) instance fields. + (instance as any).dbtConfiguration = dbtConfiguration; + (instance as any).dbtPath = "dbt"; + (instance as any).profilesDir = undefined; + + return { instance, execute }; +} + +/** Drive the public path end-to-end; returns whatever it produced/threw. */ +async function runExecuteSQL(stdout: string, stderr = "") { + const { instance } = makeFusionIntegration(stdout, stderr); + const queryExecution = await instance.executeSQL("select 1", 10, "my_model"); + return queryExecution.executeQuery(); +} + +// One valid dbt-show line carrying the preview rows (preview is itself a +// JSON-encoded string, exactly as dbt emits it). +const previewLine = JSON.stringify({ + data: { preview: JSON.stringify([{ col_a: 1 }]) }, +}); +// One valid line carrying the compiled sql. +const sqlLine = JSON.stringify({ data: { sql: "select 1 as col_a" } }); + +describe('dbt Fusion executeSQL — "Unexpected end of JSON input" via per-line JSON.parse', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + // ----- PINS: current (buggy) behaviour. GREEN today. ----- + + test("completely empty stdout throws the exact production field error (unwrapped)", async () => { + // "".trim() -> "" -> split("\n") -> [""] -> JSON.parse("") throws. + // No empty-result guard, no error wrapper: the raw SyntaxError surfaces. + let caught: Error | undefined; + try { + await runExecuteSQL(""); + } catch (e) { + caught = e as Error; + } + expect(caught).toBeInstanceOf(SyntaxError); + expect((caught as Error).message).toBe(RAW_JSON_FIELD_ERROR); + }); + + test("whitespace-only stdout throws the exact production field error", async () => { + // " ".trim() -> "" -> JSON.parse("") -> "Unexpected end of JSON input". + await expect(runExecuteSQL(" ")).rejects.toThrow(RAW_JSON_FIELD_ERROR); + }); + + test("an INTERIOR blank line between two valid JSON objects crashes the whole batch", async () => { + // i.trim() removes only outer whitespace; an interior empty line survives + // split("\n") and is handed to JSON.parse(""). Both surrounding lines are + // perfectly valid yet the entire result is discarded with the field error. + await expect(runExecuteSQL(previewLine + "\n\n" + sqlLine)).rejects.toThrow( + RAW_JSON_FIELD_ERROR, + ); + }); + + test("a spaced (non-empty) blank line between valid objects also crashes the parse", async () => { + // A line that is " " (a single space) is non-trimmable as part of the + // OUTER string, survives split("\n"), and after per-line h.trim() becomes + // "" -> JSON.parse("") -> the field error. This is the realistic shape of + // dbt's --log-format json debug stream interleaving blank progress lines + // between data lines. (Confirmed empirically against the real package.) + await expect( + runExecuteSQL(previewLine + "\n \n" + sqlLine), + ).rejects.toThrow(RAW_JSON_FIELD_ERROR); + }); + + test("a truncated content line throws a DIFFERENT V8 message (NOT this family)", async () => { + // Boundary pin: '{"a":1' is malformed-but-non-empty, so V8 reports + // "Expected ',' or '}' ...", not "Unexpected end of JSON input". This keeps + // the family attribution honest — only blank/empty lines yield the 131k msg. + let caught: Error | undefined; + try { + await runExecuteSQL('{"a":1'); + } catch (e) { + caught = e as Error; + } + expect(caught).toBeInstanceOf(SyntaxError); + expect((caught as Error).message).not.toBe(RAW_JSON_FIELD_ERROR); + expect((caught as Error).message).toBe( + "Expected ',' or '}' after property value in JSON at position 6 (line 1 column 7)", + ); + }); + + // ----- CONTROL: the path works when every line is valid JSON. ----- + + test("valid preview+sql stdout parses into the expected tabular ExecuteSQLResult", async () => { + const result = await runExecuteSQL(previewLine + "\n" + sqlLine); + // Exact-value assertions only (the library's own tabular transform output). + expect(result.table.column_names).toEqual(["col_a"]); + expect(result.table.column_types).toEqual(["string"]); + expect(result.table.rows).toEqual([[1]]); + expect(result.raw_sql).toBe("select 1"); + expect(result.compiled_sql).toBe("select 1 as col_a"); + expect(result.modelName).toBe("my_model"); + }); + + test("a valid line lacking a preview yields the specific previewLine error, NOT a JSON error", async () => { + // Proves the JSON-input crash above is specific to unparseable/blank lines: + // a well-formed line with no preview reaches the intended domain guard. + await expect(runExecuteSQL(sqlLine)).rejects.toThrow( + "Could not find previewLine in " + sqlLine, + ); + }); + + // ----- CORRECT behaviour, encoded with test.failing. ----- + // GREEN now (the calls currently throw). Each flips RED the moment the + // library skips blank/empty lines before JSON.parse and adds an empty-result + // guard — matching the resilient pattern the sibling parse paths already use. + + test.failing( + "an interior blank line SHOULD be skipped, leaving the valid batch intact", + async () => { + const result = await runExecuteSQL(previewLine + "\n\n" + sqlLine); + expect(result.table.column_names).toEqual(["col_a"]); + expect(result.table.rows).toEqual([[1]]); + expect(result.compiled_sql).toBe("select 1 as col_a"); + }, + ); + + test.failing( + "empty stdout SHOULD resolve to an empty result rather than throwing", + async () => { + // After a fix, an empty show output should surface as a benign empty + // table, never a raw 'Unexpected end of JSON input'. + const result = await runExecuteSQL(""); + expect(result.table.rows).toEqual([]); + }, + ); +}); diff --git a/src/test/suite/repro/formatterDiffProcessing.repro.test.ts b/src/test/suite/repro/formatterDiffProcessing.repro.test.ts new file mode 100644 index 000000000..28c9d3509 --- /dev/null +++ b/src/test/suite/repro/formatterDiffProcessing.repro.test.ts @@ -0,0 +1,250 @@ +/** + * Reproduction: formatDbtModelApplyDiffError family. + * + * Telemetry (30d, innoverio.vscode-dbt-power-user): + * formatDbtModelApplyDiffError = 31,334 events. + * + * The error is emitted from DbtDocumentFormattingEditProvider.executeSqlFmt + * when processDiffOutput(document, diffOutput) throws while turning a sqlfmt + * unified-diff into vscode.TextEdit[]. This suite drives the REAL + * processDiffOutput against REAL parse-diff output (the same dependency the + * production code uses) over a minimal fake TextDocument, and pins the exact + * TextEdit ranges / newText it produces. + * + * Findings encoded here: + * 1. GENUINE BUG (wrong content): when sqlfmt's formatted output has no + * trailing newline, git/diff appends a "\ No newline at end of file" + * marker line. parse-diff's eof handler models that marker as a synthetic + * change whose `type` copies the most-recent change's type. When the last + * real change was an "add", the marker becomes a synthetic ADD change with + * content "\ No newline at end of file". processDiffOutput's filter keeps + * every add/normal change and NEVER calls isDeleteChange (the only place + * the marker is special-cased), so the marker leaks into newContent as + * " No newline at end of file" (its leading "\" stripped by content.slice). + * => the formatted text is corrupted. Pinned with a normal test() (current + * buggy behaviour) and the CORRECT behaviour encoded with test.failing(). + * 2. Working-as-intended baselines: pure insertion, replacement near EOF, + * and out-of-range chunk clamping are pinned with exact assertions so the + * reproduction is objective. + */ +import { beforeEach, describe, expect, jest, test } from "@jest/globals"; +import parseDiff from "parse-diff"; +import "reflect-metadata"; +import { Position, Range, TextEdit } from "vscode"; +import { DbtDocumentFormattingEditProvider } from "../../../document_formatting_edit_provider/dbtDocumentFormattingEditProvider"; + +/** + * Minimal stand-in for vscode.TextDocument that exposes only the surface + * processDiffOutput touches: lineCount, lineAt(n).range / .rangeIncludingLineBreak. + * Line ranges use 0-based line numbers; .range.end is end-of-text on the line, + * .rangeIncludingLineBreak.end is the start of the next line (i.e. includes "\n"). + */ +function makeDocument(lines: string[]) { + const lineCount = lines.length; + return { + lineCount, + lineAt(n: number) { + if (n < 0 || n >= lineCount) { + throw new RangeError( + `Illegal value for line: ${n} (lineCount=${lineCount})`, + ); + } + const text = lines[n]; + const start = new Position(n, 0); + const end = new Position(n, text.length); + // Next-line start; for the last line vscode clamps to end-of-text, but + // for these reproductions we only feed in-range end lines. + const breakEnd = + n + 1 < lineCount + ? new Position(n + 1, 0) + : new Position(n, text.length); + return { + text, + range: new Range(start, end), + rangeIncludingLineBreak: new Range(start, breakEnd), + }; + }, + } as any; +} + +function newProvider(): any { + // processDiffOutput uses none of the constructor deps, so stubs are safe. + return new DbtDocumentFormattingEditProvider({} as any, {} as any, {} as any); +} + +function runProcessDiffOutput(doc: any, diff: string): TextEdit[] { + return newProvider().processDiffOutput(doc, diff); +} + +describe("formatDbtModelApplyDiffError repro: processDiffOutput on real sqlfmt diffs", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + test("pure insertion chunk (oldLines===0) emits a single insert TextEdit at line 0 start", () => { + // sqlfmt added two lines to an otherwise-1-line doc. + const diff = [ + "--- a/model.sql", + "+++ b/model.sql", + "@@ -0,0 +1,2 @@", + "+line one", + "+line two", + ].join("\n"); + + // Sanity-check the dependency really produced an insertion chunk. + const parsed = parseDiff(diff); + expect(parsed).toHaveLength(1); + expect(parsed[0].chunks[0].oldLines).toBe(0); + expect(parsed[0].chunks[0].oldStart).toBe(0); + + const doc = makeDocument(["select 1"]); + const edits = runProcessDiffOutput(doc, diff); + + expect(edits).toHaveLength(1); + // insertLine = Math.min(oldStart=0, lineCount=1) - 1 = -1 -> max(-1,0) = 0 + expect(edits[0].range.start.line).toBe(0); + expect(edits[0].range.start.character).toBe(0); + // Insert is a zero-width range. + expect(edits[0].range.end.line).toBe(0); + expect(edits[0].range.end.character).toBe(0); + expect(edits[0].newText).toBe("line one\nline two\n"); + }); + + test("replacement chunk near EOF replaces the exact old line range with formatted content", () => { + // 3 old lines collapse into 2 formatted lines (" select" kept, a/b joined). + const diff = [ + "--- a/model.sql", + "+++ b/model.sql", + "@@ -1,3 +1,2 @@", + " select", + "- a,", + "- b", + "+ a, b", + ].join("\n"); + + const parsed = parseDiff(diff); + expect(parsed[0].chunks[0].oldStart).toBe(1); + expect(parsed[0].chunks[0].oldLines).toBe(3); + + const doc = makeDocument(["select", " a,", " b"]); + const edits = runProcessDiffOutput(doc, diff); + + expect(edits).toHaveLength(1); + // startLine = max(oldStart-1, 0) = 0 ; endLine = min(0 + 3 - 1, 2) = 2 + expect(edits[0].range.start.line).toBe(0); + expect(edits[0].range.start.character).toBe(0); + // rangeIncludingLineBreak.end of line 2 (last line) -> end-of-text (col 3). + expect(edits[0].range.end.line).toBe(2); + expect(edits[0].range.end.character).toBe(3); + // newContent = normal(" select").slice(1) + add("+ a, b").slice(1) joined. + expect(edits[0].newText).toBe("select\n a, b\n"); + }); + + test("chunk referencing lines beyond document.lineCount is clamped to the last line", () => { + // sqlfmt diff says lines 5-6, but the document only has 3 lines. + const diff = [ + "--- a/model.sql", + "+++ b/model.sql", + "@@ -5,2 +5,1 @@", + "-x", + "-y", + "+xy", + ].join("\n"); + + const parsed = parseDiff(diff); + expect(parsed[0].chunks[0].oldStart).toBe(5); + expect(parsed[0].chunks[0].oldLines).toBe(2); + + const doc = makeDocument(["a", "b", "c"]); // lineCount = 3 + const edits = runProcessDiffOutput(doc, diff); + + // startLine = max(5-1,0) = 4 ; startLine(4) >= lineCount(3) -> early return. + // So no edit is produced (the out-of-range guard fires before lineAt). + expect(edits).toHaveLength(0); + }); + + // ----- GENUINE BUG: "\ No newline at end of file" marker corrupts output ----- + + test('PIN (current buggy behaviour): no-newline marker after an add leaks " No newline at end of file" into the replacement text', () => { + // sqlfmt formatted output has no trailing newline, so diff appends the + // "\ No newline at end of file" marker right after the "+select 2" add. + const diff = [ + "--- a/model.sql", + "+++ b/model.sql", + "@@ -1,1 +1,1 @@", + "-select 1", + "+select 2", + "\\ No newline at end of file", + ].join("\n"); + + // Confirm parse-diff models the marker as a synthetic ADD change (this is + // exactly what makes processDiffOutput's add/normal filter swallow it). + const parsed = parseDiff(diff); + const changes = parsed[0].chunks[0].changes; + const marker = changes[changes.length - 1]; + expect(marker.type).toBe("add"); + expect(marker.content).toBe("\\ No newline at end of file"); + + const doc = makeDocument(["select 1"]); + const edits = runProcessDiffOutput(doc, diff); + + expect(edits).toHaveLength(1); + // BUG: the marker line is treated as content. content.slice(1) drops the + // leading "\" and yields " No newline at end of file", which is appended + // as a bogus second line of the formatted SQL. + expect(edits[0].newText).toBe("select 2\n No newline at end of file\n"); + }); + + test.failing( + "CORRECT behaviour (flips RED when fixed): no-newline marker must NOT appear in the formatted text", + () => { + const diff = [ + "--- a/model.sql", + "+++ b/model.sql", + "@@ -1,1 +1,1 @@", + "-select 1", + "+select 2", + "\\ No newline at end of file", + ].join("\n"); + + const doc = makeDocument(["select 1"]); + const edits = runProcessDiffOutput(doc, diff); + + expect(edits).toHaveLength(1); + // The marker is metadata, not file content. The formatted line is just + // "select 2"; whether or not a trailing newline is preserved, the marker + // string must never bleed into the document text. + expect(edits[0].newText).not.toContain("No newline at end of file"); + expect(edits[0].newText.trimEnd()).toBe("select 2"); + }, + ); + + test("isDeleteChange (the marker special-case) is never consulted by processDiffOutput's filter", () => { + // Demonstrates the root cause: the add/normal filter is what selects content, + // and a synthetic ADD-typed marker bypasses isDeleteChange entirely. + const provider = newProvider(); + const markerAsAdd = { + type: "add", + add: true, + ln: 1, + content: "\\ No newline at end of file", + } as parseDiff.Change; + + // isAddChange returns true for the marker -> it is KEPT by the content filter. + expect(provider.isAddChange(markerAsAdd)).toBe(true); + // isDeleteChange would correctly reject the marker, but only for type "del". + expect(provider.isDeleteChange(markerAsAdd)).toBe(false); + + const markerAsDel = { + type: "del", + del: true, + ln: 1, + content: "\\ No newline at end of file", + } as parseDiff.Change; + // The special-case only works on the del-typed form, which the add/normal + // filter never even looks at -> the guard is effectively dead for add-typed + // markers. + expect(provider.isDeleteChange(markerAsDel)).toBe(false); + expect(provider.isAddChange(markerAsDel)).toBe(false); + }); +}); diff --git a/src/test/suite/repro/formatterSqlfmtMissing.repro.test.ts b/src/test/suite/repro/formatterSqlfmtMissing.repro.test.ts new file mode 100644 index 000000000..b9cef1d55 --- /dev/null +++ b/src/test/suite/repro/formatterSqlfmtMissing.repro.test.ts @@ -0,0 +1,226 @@ +/** + * Reproduction of the highest-volume formatter field error: + * formatDbtModelApplyDiffError (App Insights: 31,334 / 30d) + * dominant message: "sqlfmt not found" (35,757 / 30d) + * + * Field facts (innoverio.vscode-dbt-power-user, 30 days): + * - formatDbtModelApplyDiffError is sent from + * DbtDocumentFormattingEditProvider.executeSqlFmt when sqlfmt cannot be + * located on the user's machine. + * - The single most common Error message attached to that telemetry event is + * literally "sqlfmt not found" — thrown at line 68 of + * src/document_formatting_edit_provider/dbtDocumentFormattingEditProvider.ts + * when findSqlFmtPath() resolves to undefined. + * + * This test drives the REAL provider through the REAL executeSqlFmt code path. + * The only things mocked are the OS-discovery primitives (fs.existsSync, the + * `which` lookup, and child_process.exec used by `uv tool dir`) so that sqlfmt + * is provably absent — exactly the condition that produces the field error. + * + * Verdict: working-as-intended. A missing external binary (shandy-sqlfmt) is + * surfaced to the user as an actionable error message plus a telemetry event. + * is_real_bug = false. This test GUARDS that behaviour so a regression + * (silent failure, wrong event name, or a thrown exception escaping the + * provider) would be caught. + */ +import { + afterEach, + beforeEach, + describe, + expect, + jest, + test, +} from "@jest/globals"; + +// --- Mock the OS-discovery primitives so sqlfmt is provably NOT found. --- +// fs.existsSync must return false for every candidate path that +// discoverSqlFmtPath() probes (venv bin dir, ~/.local/bin, pipx, uv tool dir). +jest.mock("fs", () => ({ + __esModule: true, + default: { existsSync: jest.fn(() => false) }, + existsSync: jest.fn(() => false), +})); + +// `which("sqlfmt")` is the final PATH fallback; reject so it contributes +// nothing — mirroring a machine where sqlfmt is not on PATH. +jest.mock("which", () => ({ + __esModule: true, + default: jest.fn(() => Promise.reject(new Error("not found: sqlfmt"))), +})); + +// child_process.exec backs execAsync("uv tool dir"). Invoke the callback with +// an error so promisify(exec) rejects and findSqlFmtInUvTools() returns +// undefined (the catch branch in the source). util.promisify uses the real +// util; only exec is faked. +jest.mock("child_process", () => ({ + __esModule: true, + exec: jest.fn( + ( + _cmd: string, + _opts: unknown, + cb?: (e: Error | null, out?: unknown) => void, + ) => { + const callback = typeof _opts === "function" ? _opts : cb; + if (callback) { + callback(new Error("uv: command not found")); + } + }, + ), +})); + +import { window, workspace } from "vscode"; +import { DbtDocumentFormattingEditProvider } from "../../../document_formatting_edit_provider/dbtDocumentFormattingEditProvider"; + +// Minimal TextDocument stand-in. executeSqlFmt only reaches document.getText() +// AFTER a sqlfmt binary is found, so in the "not found" path getText() is never +// called — but we provide it anyway to be faithful to the interface. +const makeDocument = () => + ({ + getText: jest.fn(() => "select 1 as a"), + lineCount: 1, + }) as any; + +// PythonEnvironment stand-in. getResolvedConfigValue("sqlFmtPath") returns +// undefined (no user-configured path) so the provider falls through to +// findSqlFmtPath(). pythonPath is a non-existent interpreter — its dirname is +// probed via fs.existsSync (mocked false), so it yields no sqlfmt either. +const makePythonEnvironment = () => + ({ + getResolvedConfigValue: jest.fn(() => undefined), + pythonPath: "/nonexistent/venv/bin/python", + }) as any; + +const makeTelemetry = () => + ({ + sendTelemetryEvent: jest.fn(), + sendTelemetryError: jest.fn(), + }) as any; + +// The command factory must NEVER be invoked in the "not found" path — sqlfmt is +// never located, so createCommandProcessExecution is unreachable. We assert +// that below. +const makeCommandFactory = () => + ({ + createCommandProcessExecution: jest.fn(), + }) as any; + +describe("formatDbtModelApplyDiffError repro — 'sqlfmt not found'", () => { + let telemetry: ReturnType; + let commandFactory: ReturnType; + let pythonEnvironment: ReturnType; + let provider: DbtDocumentFormattingEditProvider; + + beforeEach(() => { + (window.showErrorMessage as jest.Mock).mockClear(); + + // executeSqlFmt reads workspace.getConfiguration("dbt").get( + // "sqlFmtAdditionalParams", []) then calls .join(" ") on the result. + // The bundled vscode mock's get() ignores the default arg and returns + // undefined, which would NPE on .join — so we pin it to an empty array, + // faithfully representing the default (no extra sqlfmt params configured). + jest.spyOn(workspace, "getConfiguration").mockReturnValue({ + get: jest.fn((_key: string, _default?: unknown) => []), + has: jest.fn(), + update: jest.fn(), + } as any); + + telemetry = makeTelemetry(); + commandFactory = makeCommandFactory(); + pythonEnvironment = makePythonEnvironment(); + provider = new DbtDocumentFormattingEditProvider( + commandFactory, + telemetry, + pythonEnvironment, + ); + }); + + afterEach(() => { + jest.restoreAllMocks(); + jest.clearAllMocks(); + }); + + test("sends formatDbtModelApplyDiffError telemetry with the exact 'sqlfmt not found' Error", async () => { + const result = await provider.provideDocumentFormattingEdits( + makeDocument(), + {} as any, + {} as any, + ); + + // Telemetry is sent exactly once, with the documented event name and the + // literal "sqlfmt not found" error that dominates the field data. + expect(telemetry.sendTelemetryError).toHaveBeenCalledTimes(1); + const [eventName, errorArg] = (telemetry.sendTelemetryError as jest.Mock) + .mock.calls[0] as [string, unknown]; + expect(eventName).toBe("formatDbtModelApplyDiffError"); + expect(errorArg).toBeInstanceOf(Error); + expect((errorArg as Error).message).toBe("sqlfmt not found"); + + // The "happy path" telemetry event must NOT fire when the binary is absent. + expect(telemetry.sendTelemetryEvent).not.toHaveBeenCalled(); + + // The provider returns an empty edit list (no formatting applied), so the + // user's document is left untouched rather than corrupted. + expect(result).toEqual([]); + }); + + test("shows the actionable install/setup error message and never spawns sqlfmt", async () => { + await provider.provideDocumentFormattingEdits( + makeDocument(), + {} as any, + {} as any, + ); + + expect(window.showErrorMessage).toHaveBeenCalledTimes(1); + const shown = (window.showErrorMessage as jest.Mock).mock + .calls[0][0] as string; + // The message guides the user to install sqlfmt or set dbt.sqlFmtPath, and + // carries the underlying "sqlfmt not found" detail. + expect(shown).toContain("Could not run sqlfmt."); + expect(shown).toContain("dbt.sqlFmtPath"); + expect(shown).toContain("shandy-sqlfmt[jinjafmt]"); + expect(shown).toContain("Error: sqlfmt not found."); + + // Because sqlfmt was never located, the command process is never created — + // we never try to spawn a non-existent binary. + expect(commandFactory.createCommandProcessExecution).not.toHaveBeenCalled(); + }); + + test("pins working-as-intended: a user-configured sqlFmtPath bypasses discovery and skips the not-found error", async () => { + // Counter-case proving the not-found branch is gated on discovery, not an + // unconditional failure. With a configured path, sqlfmt is considered + // present: the formatDbtModel telemetry fires and the command IS created. + // getFirstWorkspacePath() reads workspace.workspaceFolders[0].uri.fsPath, + // so a folder must be present for the command-creation path. + (workspace as any).workspaceFolders = [{ uri: { fsPath: "/tmp/project" } }]; + pythonEnvironment.getResolvedConfigValue = jest.fn(() => "/usr/bin/sqlfmt"); + const completeMock = jest.fn(() => Promise.resolve({ stderr: "" })); + commandFactory.createCommandProcessExecution = jest.fn(() => ({ + complete: completeMock, + })); + const configuredProvider = new DbtDocumentFormattingEditProvider( + commandFactory, + telemetry, + pythonEnvironment, + ); + + const result = await configuredProvider.provideDocumentFormattingEdits( + makeDocument(), + {} as any, + {} as any, + ); + + expect(telemetry.sendTelemetryError).not.toHaveBeenCalled(); + expect(telemetry.sendTelemetryEvent).toHaveBeenCalledTimes(1); + expect((telemetry.sendTelemetryEvent as jest.Mock).mock.calls[0]).toEqual([ + "formatDbtModel", + { sqlFmtPath: "setting" }, + ]); + expect(commandFactory.createCommandProcessExecution).toHaveBeenCalledTimes( + 1, + ); + // Empty stderr + no diff output => no edits, clean return. + expect(result).toEqual([]); + + (workspace as any).workspaceFolders = []; + }); +}); diff --git a/src/test/suite/repro/manifestWarningsParse.repro.test.ts b/src/test/suite/repro/manifestWarningsParse.repro.test.ts new file mode 100644 index 000000000..a203f4705 --- /dev/null +++ b/src/test/suite/repro/manifestWarningsParse.repro.test.ts @@ -0,0 +1,258 @@ +/** + * Reproduction: RebuildManifestErrorsAndWarningsJSONParsingError + * (App Insights, innoverio.vscode-dbt-power-user, 30d: 61,116 + 13,132 non-cloud variant) + * + * WHERE IT LIVES (source AND runtime verified against the real npm package) + * ------------------------------------------------------------------------ + * The telemetry name is produced by the shared helper `parseJSON(tag, line, notify)` defined on + * the base integration class (`L`) in node_modules/@altimateai/dbt-integration/dist/index.js. + * Verified minified source body (at byte offset 14034): + * + * parseJSON(r, t, e = true) { // r=tag, t=line, e=notify(default true) + * try { return JSON.parse(t); } + * catch (n) { + * if (this.terminal.error(r + "Error", // -> "RebuildManifestErrorsAndWarningsJSONParsingError" + * "An error occured while parsing following json: " + t, + * n), // <-- exactly 3 args: name, message, error + * e) throw n; // throws ONLY when notify(e) is truthy + * } + * // NO explicit return -> falls off the end -> returns `undefined` when notify is false + * } + * + * The caller is `DBTFusionCommandProjectIntegration.rebuildManifest()` (verified at offset 46742 / + * dist/index.js line 77), which parses the dbt parse-command stderr LINE-BY-LINE with notify=false: + * + * const o = stderr.trim().split("\n").map(s=>s.trim()).filter(s=>!!s) + * .map(s => this.parseJSON("RebuildManifestErrorsAndWarningsJSONParsing", s, false)); + * const a = o.filter(s => s && s.hasOwnProperty("info") && s.info.hasOwnProperty("level") + * && s.info.hasOwnProperty("msg") && ["error","fatal"].includes(s.info.level)) + * .map(s => s.info.msg); // -> error diagnostics + * const i = o.filter(s => s && ... && s.info.level === "warn").map(s => s.info.msg); // -> warning diagnostics + * + * The error NAME is built as `tag + "Error"`, which is exactly + * "RebuildManifestErrorsAndWarningsJSONParsingError". `parseJSON` is inherited from base `L`, so it + * is reachable off the exported integration classes — runtime-verified that it resolves on + * DBTFusionCommandProjectIntegration, DBTCloudProjectIntegration, DBTBaseProjectIntegration and + * CloudFusionIntegration (all defined on `L`). We use the ACTUAL rebuild caller class here, + * DBTFusionCommandProjectIntegration. + * + * EXACT RUNTIME-OBSERVED BEHAVIOUR (notify=false, the rebuild path) + * ---------------------------------------------------------------- + * valid line -> returns the parsed object, terminal.error NOT called + * "{}" -> returns {} (parses fine), terminal.error NOT called, caller drops it (no info) + * bad line -> terminal.error called with name="...Error", + * message="An error occured while parsing following json: " + line, + * error=; then returns `undefined` (NOT null, NOT throw) + * Note: this helper does NOT pass a `notify` flag NOR an `{adapter}` dimension to terminal.error + * (only 3 args). The adapter dimension lives on OTHER telemetry calls in rebuildManifest's catch, + * not on this per-line parse. + * + * WHAT THE TELEMETRY MEANS / BUG CLASSIFICATION + * --------------------------------------------- + * When dbt emits a line that is not valid JSON (a truncated jsonl record, a banner/log line, or a + * python traceback leaking onto stderr — the same upstream condition behind the 131k "Unexpected + * end of JSON input" family), JSON.parse throws, the helper fires the telemetry, and (because the + * caller passes notify=false) returns `undefined`. The caller's `.filter(s => s && ...)` then drops + * every undefined, so the offending line — which may have carried a real compile error or warning — + * is silently discarded from the diagnostics list. + * + * It does not crash (the throw is suppressed for notify=false), so at "does the extension blow up" + * level it is working-as-intended. But it is a genuine, high-volume (61k + 13k events / 30d) + * data-loss / observability defect: a failed error/warning line vanishes and the dropped-line count + * is never surfaced to the user. is_real_bug = true on that basis. + * + * STRATEGY + * -------- + * - Every assertion calls the REAL exported helper off the class prototype (no re-implementation), + * with a minimal `this` providing only the members the method touches: `this.terminal.error`. + * - Normal test()s PIN current behaviour with exact values (parsed object; the precise telemetry + * name + message; SyntaxError; `undefined` return for notify=false; throw for notify=true; and + * the caller filter dropping the undefined). + * - One test.failing() encodes the CORRECT fixed behaviour: a parse failure must not collapse to a + * bare `undefined` indistinguishable from a benign non-diagnostic line. Green now (bug present), + * flips red when fixed. + * + * ASSERTION STABILITY + * ------------------- + * Telemetry NAME, the exact message string, the `undefined` return, and the canonical V8-stable + * SyntaxError message "Unexpected end of JSON input" (premature-EOF / truncated case) are stable + * and asserted exactly. For arbitrary garbage we assert only `instanceof SyntaxError` because V8's + * position-specific text ("Unexpected token 'R'...") varies by Node version. + */ +import { DBTFusionCommandProjectIntegration } from "@altimateai/dbt-integration"; +import { describe, expect, test } from "@jest/globals"; + +const TELEMETRY_EVENT = "RebuildManifestErrorsAndWarningsJSONParsingError"; +const REBUILD_TAG = "RebuildManifestErrorsAndWarningsJSONParsing"; + +type Captured = { + name: string; + message: string; + error: unknown; + extraArgCount: number; +} | null; + +/** Invoke the REAL parseJSON helper off the prototype with a minimal `this`. */ +function callRealParseJSON( + line: string, + notify: boolean, +): { result: unknown; threw: unknown; telemetry: Captured } { + let telemetry: Captured = null; + const ctx = { + terminal: { + error: ( + name: string, + message: string, + error: unknown, + ...rest: unknown[] + ) => { + telemetry = { name, message, error, extraArgCount: rest.length }; + }, + }, + }; + const fn = (DBTFusionCommandProjectIntegration as any).prototype.parseJSON; + let result: unknown; + let threw: unknown = undefined; + try { + result = fn.call(ctx, REBUILD_TAG, line, notify); + } catch (e) { + threw = e; + } + return { result, threw, telemetry }; +} + +/** Mirror of the caller's error/fatal keep-filter (dist/index.js line 77). */ +function callerKeepsAsError(s: any): boolean { + return !!( + s && + Object.prototype.hasOwnProperty.call(s, "info") && + s.info && + Object.prototype.hasOwnProperty.call(s.info, "level") && + Object.prototype.hasOwnProperty.call(s.info, "msg") && + ["error", "fatal"].includes(s.info.level) + ); +} + +describe("RebuildManifestErrorsAndWarningsJSONParsingError (real @altimateai/dbt-integration)", () => { + test("the parseJSON helper is reachable on the rebuild caller class prototype (not re-implemented here)", () => { + expect(typeof DBTFusionCommandProjectIntegration).toBe("function"); + expect( + typeof (DBTFusionCommandProjectIntegration as any).prototype.parseJSON, + ).toBe("function"); + }); + + test("valid error JSONL line: returns the parsed object, no telemetry, caller keeps it", () => { + const line = JSON.stringify({ + info: { level: "error", msg: "Compilation Error in model foo" }, + }); + const { result, threw, telemetry } = callRealParseJSON(line, false); + expect(threw).toBeUndefined(); + expect(result).toEqual({ + info: { level: "error", msg: "Compilation Error in model foo" }, + }); + expect(telemetry).toBeNull(); + expect(callerKeepsAsError(result)).toBe(true); + }); + + test("valid but non-diagnostic JSONL line ('{}'): parses fine, no telemetry, caller drops it", () => { + const { result, threw, telemetry } = callRealParseJSON("{}", false); + expect(threw).toBeUndefined(); + expect(result).toEqual({}); + expect(telemetry).toBeNull(); + expect(callerKeepsAsError(result)).toBe(false); + }); + + test("truncated JSONL line ('Unexpected end of JSON input') with notify=false: fires telemetry, does NOT throw, returns undefined", () => { + // A real jsonl line cut off mid-stream so it ends right after a key, where MORE tokens were + // expected — this is what reliably yields the V8-stable "Unexpected end of JSON input" message + // and is the exact upstream condition behind that 131k family. (A line truncated mid-VALUE + // instead yields V8's position-specific text, which is not version-stable, so we don't use it + // for the message assertion.) + const truncated = '{"info":'; + const { result, threw, telemetry } = callRealParseJSON(truncated, false); + + expect(telemetry).not.toBeNull(); + expect(telemetry!.name).toBe(TELEMETRY_EVENT); + // Exact message string built by the helper (verified in source). + expect(telemetry!.message).toBe( + `An error occured while parsing following json: ${truncated}`, + ); + expect(telemetry!.error).toBeInstanceOf(SyntaxError); + // Canonical, V8-stable message for premature EOF. + expect((telemetry!.error as Error).message).toBe( + "Unexpected end of JSON input", + ); + // The helper passes ONLY (name, message, error) to terminal.error — no notify flag, no adapter. + expect(telemetry!.extraArgCount).toBe(0); + + // notify=false -> the `if(...,e)throw n` does NOT throw; the method falls off the end. + expect(threw).toBeUndefined(); + // THE DATA LOSS: returns undefined (not null), so the caller's `.filter(s => s && ...)` drops it. + expect(result).toBeUndefined(); + expect(callerKeepsAsError(result)).toBe(false); + }); + + test("non-JSON line (banner/log) with notify=false: fires telemetry with a SyntaxError, returns undefined", () => { + const logline = "Running with dbt=1.7.0"; + const { result, threw, telemetry } = callRealParseJSON(logline, false); + expect(telemetry).not.toBeNull(); + expect(telemetry!.name).toBe(TELEMETRY_EVENT); + expect(telemetry!.message).toBe( + `An error occured while parsing following json: ${logline}`, + ); + expect(telemetry!.error).toBeInstanceOf(SyntaxError); + expect(telemetry!.extraArgCount).toBe(0); + expect(threw).toBeUndefined(); + expect(result).toBeUndefined(); + }); + + test("notify=true: still fires the SAME telemetry but RE-THROWS the SyntaxError (other callers' path)", () => { + // The rebuild path uses notify=false; the default (notify=true) re-throws. Pin both branches. + const bad = "definitely not json"; + const { threw, telemetry } = callRealParseJSON(bad, true); + expect(telemetry).not.toBeNull(); + expect(telemetry!.name).toBe(TELEMETRY_EVENT); + expect(telemetry!.message).toBe( + `An error occured while parsing following json: ${bad}`, + ); + expect(threw).toBeInstanceOf(SyntaxError); + }); + + test("THE BUG, pinned: a failed parse is dropped by the caller IDENTICALLY to a benign non-diagnostic line", () => { + // A failed parse yields undefined -> dropped by `.filter(s => s && ...)`. + const failed = callRealParseJSON( + '{"info":{"level":"error","msg":"boom"}', + false, + ).result; + // A successfully-parsed non-diagnostic line is ALSO dropped by the same filter. + const nonDiag = callRealParseJSON( + JSON.stringify({ status: "ok" }), + false, + ).result; + + expect(callerKeepsAsError(failed)).toBe(false); + expect(callerKeepsAsError(nonDiag)).toBe(false); + // The dropped error message "boom" never reaches the diagnostics list, and downstream there is + // no way to tell the failure (failed=undefined) apart from the benign skip (nonDiag={status}). + expect(failed).toBeUndefined(); + expect(nonDiag).toEqual({ status: "ok" }); + }); + + // CORRECT behaviour, encoded as test.failing so the suite is GREEN now (bug present) and flips + // RED when the helper stops collapsing a parse failure to a bare `undefined`. A correct + // implementation would surface the failure distinctly to the rebuild caller (e.g. return a + // sentinel it can detect and report, or push a synthetic diagnostic) instead of returning the + // same falsy value that the caller's filter silently drops — making the ~74k/mo of dropped + // diagnostics observable instead of silent. + test.failing( + "FIXED-BEHAVIOUR GUARD: a parse failure must not return a bare undefined that the caller filter silently drops", + () => { + const failed = callRealParseJSON( + '{"info":{"level":"error","msg":"boom"}', + false, + ).result; + // Today this is undefined (the bug). A correct fix returns a distinguishable failure marker. + expect(failed).toBeDefined(); + }, + ); +}); diff --git a/src/test/suite/repro/utilsErrorPaths.repro.test.ts b/src/test/suite/repro/utilsErrorPaths.repro.test.ts new file mode 100644 index 000000000..1392e097c --- /dev/null +++ b/src/test/suite/repro/utilsErrorPaths.repro.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, test } from "@jest/globals"; +import { extendErrorWithSupportLinks } from "../../../utils"; + +/** + * Reproduction tests for the `extendErrorWithSupportLinks` error-formatting + * helper (src/utils.ts:96). This helper decorates the user-facing message on + * the error paths that feed the top App Insights field errors + * (catchAllError 18.07M, formatDbtModelApplyDiffError, RebuildManifest..., + * "Unexpected end of JSON input", etc.). It is invoked from ~20 call sites + * (commands/index.ts, dbtProject.ts, the formatter provider, doc gen, lineage, + * conversation provider, ...), several of which pass `(err as Error).message` + * or joined arrays — i.e. values that are NOT guaranteed to be strings at + * runtime even though the parameter is typed `string`. + * + * Two real defects are pinned here: + * + * (1) Dead trailing-space de-duplication. The source uses `error[-1]` to test + * whether the message already ends in a space. JavaScript does NOT support + * negative string indexing (that is Python), so `error[-1]` is ALWAYS + * `undefined` and the `=== " "` branch is never taken. An + * already-space-terminated message gets a SECOND space appended, producing + * a double space before the support link. The intended behaviour + * (`endsWith(" ")` => no extra space) is encoded with `test.failing` so the + * suite is green now and flips red the moment the bug is fixed. + * + * (2) Non-string coercion. With non-string runtime inputs (undefined, null, + * plain object) the `+ " "` concatenation coerces the value via + * `String(value)`, yielding misleading messages like "undefined ...", + * "null ..." and "[object Object] ..." that mask the real primary error + * instead of surfacing it. The function never throws, so this compounds — + * rather than crashes — the top field errors. + * + * The const below is the exact support-link suffix from the source; the leading + * space is contributed by the function, not by this constant. + */ +const SUPPORT_TAIL = + "If the issue persists, please [contact us](https://www.altimate.ai/support) via chat or Slack"; + +describe("extendErrorWithSupportLinks - error-formatting robustness (repro)", () => { + // ---- Baseline: the documented happy path still holds --------------------- + test("plain string message gets exactly one space then the support link", () => { + expect(extendErrorWithSupportLinks("problem")).toBe( + "problem " + SUPPORT_TAIL, + ); + }); + + // ---- Defect 1: dead trailing-space de-duplication ------------------------ + // PIN (current buggy behaviour): an already-space-terminated message gets a + // SECOND space, because `error[-1] === " "` can never be true in JS. + test("BUG PIN: already-space-terminated message gets a doubled space", () => { + const out = extendErrorWithSupportLinks("problem "); + // Two spaces between "problem" and the link -> dedup logic is dead code. + expect(out).toBe("problem " + SUPPORT_TAIL); + expect(out.startsWith("problem If")).toBe(true); // double space + expect(out.startsWith("problem If")).toBe(false); // not single + }); + + // CORRECT behaviour the fix should produce: a message that already ends in a + // space must NOT receive a second one. This flips to passing once the source + // uses `endsWith(" ")` (or `error[error.length - 1]`) instead of `error[-1]`. + test.failing( + "EXPECTED AFTER FIX: trailing-space message keeps a single space", + () => { + expect(extendErrorWithSupportLinks("problem ")).toBe( + "problem " + SUPPORT_TAIL, + ); + }, + ); + + // ---- Defect 2: non-string inputs are silently coerced -------------------- + // These call sites are real: e.g. commands/index.ts passes + // `(err as Error).message`, which is `undefined` when the thrown value is not + // an Error or has no message; dbtLineageService passes joined arrays; several + // paths forward a caught value of `unknown` type. + + // VERIFIED by running jest: undefined/null do NOT coerce to "undefined ..." + // — the property access `error[-1]` itself throws before any concatenation, + // because you cannot read a property off undefined/null. This is the more + // severe variant: the error-decorator THROWS on exactly the inputs produced + // by `(err as Error).message` when the thrown value is not an Error (message + // is undefined). A helper meant to make errors friendlier instead raises a + // secondary TypeError, which is what feeds the catchAll wrapper. + test("BUG PIN: undefined input throws TypeError (reading '-1'), not a friendly message", () => { + expect(() => + extendErrorWithSupportLinks(undefined as unknown as string), + ).toThrow(TypeError); + expect(() => + extendErrorWithSupportLinks(undefined as unknown as string), + ).toThrow("Cannot read properties of undefined (reading '-1')"); + }); + + test("BUG PIN: null input throws TypeError (reading '-1'), not a friendly message", () => { + expect(() => + extendErrorWithSupportLinks(null as unknown as string), + ).toThrow(TypeError); + expect(() => + extendErrorWithSupportLinks(null as unknown as string), + ).toThrow("Cannot read properties of null (reading '-1')"); + }); + + // CORRECT behaviour the fix should produce: null/undefined must NOT throw — + // the decorator should degrade gracefully to the support link. Flips to + // passing once the helper normalises nullish input (e.g. `String(error ?? "")`). + test.failing("EXPECTED AFTER FIX: null/undefined inputs do not throw", () => { + expect(() => + extendErrorWithSupportLinks(undefined as unknown as string), + ).not.toThrow(); + expect(() => + extendErrorWithSupportLinks(null as unknown as string), + ).not.toThrow(); + }); + + test("BUG PIN: a plain object becomes '[object Object]'", () => { + expect(extendErrorWithSupportLinks({} as unknown as string)).toBe( + "[object Object] " + SUPPORT_TAIL, + ); + }); + + test("BUG PIN: an error-like object surfaces '[object Object]', not its .message", () => { + // The function stringifies the whole object instead of reading `.message`, + // so the actual error text ("boom") is lost from the user-facing message. + const out = extendErrorWithSupportLinks({ + message: "boom", + } as unknown as string); + expect(out).toBe("[object Object] " + SUPPORT_TAIL); + expect(out.includes("boom")).toBe(false); + }); + + // CORRECT behaviour the fix should produce for an Error-like value: the real + // message ("boom") should appear and the misleading "[object Object]" should + // not. Flips to passing once the helper normalises non-string input + // (e.g. `error instanceof Error ? error.message : String(error ?? "")`). + test.failing( + "EXPECTED AFTER FIX: error-like input surfaces its message, not [object Object]", + () => { + const out = extendErrorWithSupportLinks({ + message: "boom", + } as unknown as string); + expect(out.includes("boom")).toBe(true); + expect(out.includes("[object Object]")).toBe(false); + }, + ); + + // CORRECT behaviour the fix should produce for null/undefined: the call must + // succeed and return a string that still ends with the support link (so the + // user is never shown a raw "null"/"undefined" and the decorator never + // throws). Currently this throws at `error[-1]`, so the test fails today and + // flips to passing once the helper normalises nullish input. + test.failing( + "EXPECTED AFTER FIX: undefined yields a clean string ending in the support link", + () => { + const out = extendErrorWithSupportLinks(undefined as unknown as string); + expect(out.endsWith(SUPPORT_TAIL)).toBe(true); + expect(out).not.toContain("undefined "); + }, + ); +}); diff --git a/src/test/suite/telemetryRedaction.repro.test.ts b/src/test/suite/telemetryRedaction.repro.test.ts new file mode 100644 index 000000000..62e77a1cf --- /dev/null +++ b/src/test/suite/telemetryRedaction.repro.test.ts @@ -0,0 +1,122 @@ +import { + afterEach, + beforeEach, + describe, + expect, + jest, + test, +} from "@jest/globals"; +import * as vscode from "vscode"; +import { TelemetryService } from "../../telemetry"; + +// Reproduction tests for the extension's own pre-send secret scrubbing in +// TelemetryService.removeGenericSecretsFromStackTrace (src/telemetry/index.ts). +// +// Why this matters: error stacks are sent to App Insights via +// sendTelemetryErrorEvent. Field data shows error events at very high volume +// (millions / 30d), so any stack that slips a secret past this scrubber is a +// real privacy exposure. VS Code's TelemetryLogger is the primary redactor, so +// severity is defence-in-depth — but the extension's own scrubber is provably +// incomplete, and these tests pin exactly how. +// +// Root cause (src/telemetry/index.ts:144): +// error.replace(/(key|token|sig|secret|...)/i, "****") +// has NO /g flag, so only the FIRST keyword occurrence in the whole stack is +// touched; every later secret-bearing line passes through verbatim. It also +// only masks the matched KEYWORD, never the secret VALUE that follows it. + +describe("TelemetryService secret-redaction (repro: incomplete stack scrubbing)", () => { + let telemetry: TelemetryService; + let reporterErrorSpy: jest.Mock; + + beforeEach(() => { + // TelemetryService's constructor builds a real TelemetryReporter, which reads + // vscode.env.appName and calls vscode.env.createTelemetryLogger. Mirror the + // env mock used by the existing telemetryService.test.ts so construction works. + (vscode as { env?: unknown }).env = { + appName: "Visual Studio Code", + createTelemetryLogger: jest.fn().mockReturnValue({ + onDidChangeEnableStates: jest + .fn() + .mockReturnValue({ dispose: jest.fn() }), + isErrorsEnabled: false, + isUsageEnabled: false, + logUsage: jest.fn(), + logError: jest.fn(), + dispose: jest.fn(), + }), + }; + + telemetry = new TelemetryService(); + // Swap the live reporter for a spy so nothing leaves the process; capture the + // exact properties dict (including the scrubbed `stack`) handed to App Insights. + reporterErrorSpy = jest.fn(); + (telemetry as unknown as { telemetryReporter: unknown }).telemetryReporter = + { + sendTelemetryEvent: jest.fn(), + sendTelemetryErrorEvent: reporterErrorSpy, + dispose: jest.fn(), + }; + }); + + afterEach(() => { + telemetry.dispose(); + delete (vscode as { env?: unknown }).env; + jest.restoreAllMocks(); + }); + + // Event name ends in "Error" per the repo's eslint convention for + // sendTelemetryError's first argument. + function scrubbedStackFor(stack: string): string { + const err = new Error("boom"); + err.stack = stack; + telemetry.sendTelemetryError("reproSecretScrubError", err); + expect(reporterErrorSpy).toHaveBeenCalledTimes(1); + const [, properties] = reporterErrorSpy.mock.calls[0] as [ + string, + Record, + ]; + return properties.stack; + } + + // Pins CURRENT behaviour: with two "token" keywords, only the first is masked + // and the second leaks through. Passes today (documents the bug); becomes a + // tripwire that fails the moment the scrubber is made global. + test("BUG: only the first secret keyword in a multi-line stack is masked", () => { + const out = scrubbedStackFor( + [ + "Error: request failed", + " at a (token=AAAA111)", + " at b (token=BBBB222)", + ].join("\n"), + ); + expect(out).toContain("****"); // first occurrence masked + expect(out).toContain("token=BBBB222"); // later occurrence leaks (the bug) + expect(out.split("****").length - 1).toBe(1); // exactly one substitution + }); + + // Also pins that only the keyword is masked, not the secret value after it. + test("BUG: the secret VALUE survives even when its keyword is masked", () => { + const out = scrubbedStackFor("Error\n at x (secret=SUPERSECRET123)"); + expect(out).toContain("****"); // 'secret' keyword masked + expect(out).toContain("SUPERSECRET123"); // the actual value still leaks + }); + + // Desired behaviour. Currently fails (scrubber is not global), so marked + // `failing` to keep the suite green; flips to a hard failure when index.ts adds + // the /g flag — the signal to convert this to a normal test(). + test.failing( + "FIXME(field-repro): every secret keyword in the stack should be masked", + () => { + const out = scrubbedStackFor( + [ + "Error: request failed", + " at a (token=AAAA111)", + " at b (password=BBBB222)", + ].join("\n"), + ); + expect(out).not.toMatch(/AAAA111/); + expect(out).not.toMatch(/BBBB222/); + }, + ); +}); diff --git a/test-matrix/Dockerfile b/test-matrix/Dockerfile new file mode 100644 index 000000000..265dbb599 --- /dev/null +++ b/test-matrix/Dockerfile @@ -0,0 +1,63 @@ +# Parity container for the install/update matrix. +# +# Goal: "green here == green in CI". This image mirrors the GitHub `ubuntu-latest` +# runner (Ubuntu 24.04, Node 20, Python 3, xvfb + Electron's headless libs) and +# runs the SAME test-matrix scripts, so the VSCode lane AND the Linux-only fork +# lanes (Cursor/Windsurf/Kiro/code-server) all execute exactly as they do in CI — +# which a macOS host cannot do natively. +# +# Build context = repo root; excludes via test-matrix/Dockerfile.dockerignore. +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +SHELL ["/bin/bash", "-c"] + +# Base tooling + everything Electron needs to launch headless under xvfb. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl gnupg git unzip jq sudo \ + python3 python3-venv python3-pip \ + libnss3 libnspr4 libatk1.0-0t64 libatk-bridge2.0-0t64 libcups2t64 \ + libdrm2 libgbm1 libgtk-3-0t64 libasound2t64 libxkbcommon0 libxshmfence1 \ + libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libxext6 libx11-6 \ + libpango-1.0-0 libcairo2 libatspi2.0-0t64 libwayland-client0 libsecret-1-0 \ + && rm -rf /var/lib/apt/lists/* + +# xvfb + xauth WITH recommends: the `xvfb-run` launcher script (used by the +# VSCode + fork lanes on Linux) is NOT included under --no-install-recommends. +RUN apt-get update && apt-get install -y xvfb xauth \ + && command -v xvfb-run >/dev/null \ + && rm -rf /var/lib/apt/lists/* + +# Node 20 (matches the workflow's setup-node node-version: "20"). +RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* + +# squashfs-tools + binutils: the fork provisioners extract Cursor's AppImage +# WITHOUT executing it (the x86_64 self-extract runtime returns exit 126 under +# Docker Desktop's QEMU emulation on Apple Silicon). readelf computes the +# squashfs offset; unsquashfs extracts it. No-op on native CI runners, where +# --appimage-extract works and this fallback never runs. +RUN apt-get update && apt-get install -y --no-install-recommends squashfs-tools binutils \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /work + +# Copy the WHOLE repo first, then `npm ci` — exactly like CI (actions/checkout +# then npm ci). The package's `postinstall` (node postInstall.js) reads repo +# files, so a package.json-only copy makes it fail. Host's mac node_modules / +# .vscode-test / venv are excluded via the scoped .dockerignore, so this still +# produces fresh linux deps. (Trades layer caching for CI-identical correctness.) +COPY . . +RUN npm ci --no-audit --no-fund + +# Compile the in-host test suite. +RUN npm run compile + +# pytest (the matrix unit tests) + warm the dbt cache so setup-dbt-env.sh is fast. +RUN python3 -m pip install --break-system-packages --no-cache-dir \ + pytest "dbt-core==1.9.6" "dbt-duckdb==1.9.3" + +# Default: run the whole matrix (all lanes; forks run because this is Linux). +# Pass args through, e.g. `docker run --vsix /vsix/pu.vsix`. +ENTRYPOINT ["bash", "test-matrix/run-local.sh"] diff --git a/test-matrix/Dockerfile.dockerignore b/test-matrix/Dockerfile.dockerignore new file mode 100644 index 000000000..a7c78eb17 --- /dev/null +++ b/test-matrix/Dockerfile.dockerignore @@ -0,0 +1,15 @@ +# Scoped ignore for test-matrix/Dockerfile (BuildKit reads .dockerignore). +# Keep the build context small and platform-correct: never copy the host's +# mac-built node_modules / downloaded editors / venv — they'd be wrong-arch and huge. +node_modules +webview_panels/node_modules +.vscode-test +.matrix-venv +**/result-*.json +**/*-console-*.log +.git +docs +out +coverage +.nyc_output +*.vsix diff --git a/test-matrix/README.md b/test-matrix/README.md new file mode 100644 index 000000000..b1a28cfb6 --- /dev/null +++ b/test-matrix/README.md @@ -0,0 +1,12 @@ +# Install / Update Test Matrix (P1) + +Each cell emits a RESULT_JSON (schema in docs/superpowers/plans/2026-05-30-extension-install-update-matrix-p1.md). + +- `vscode-cell.mjs` — real VSCode/Insiders via @vscode/test-electron (fresh + upgrade) +- `codeserver-cell.sh` — wraps docker-setup/vsix-smoke.sh +- `aggregate.py` — renders the install + update matrices, Slack payload, and the blocking gate exit code + +Run one cell locally: +npm run build && npm run compile +bash test-matrix/setup-dbt-env.sh +node test-matrix/vscode-cell.mjs --mode fresh --target latest --out /tmp/r.json diff --git a/test-matrix/active-versions.py b/test-matrix/active-versions.py new file mode 100644 index 000000000..4edc20840 --- /dev/null +++ b/test-matrix/active-versions.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +"""Query the extension's Azure Application Insights for the live version +distribution (which versions are still running, by distinct install) and pick a +data-driven set of upgrade-from baselines for the test matrix. + +Backends (auto-detected): + - REST API : set APPINSIGHTS_APP_ID + APPINSIGHTS_API_KEY (CI-friendly) + - az CLI : falls back to `az monitor app-insights query` (local, AAD login) + +Writes JSON: + { "generated_at", "app_id", "window_days", "target", + "baselines": [...], "distribution": [ {"version","installs","share"} ... ] } + +NOTE: the output contains per-version install counts (business metrics). Do NOT +commit it to a public repo — it is .gitignored. CI regenerates it from the +secret, or the workflow falls back to a hardcoded baseline list. + +Usage: + python3 test-matrix/active-versions.py --out test-matrix/active-versions.json +""" +from __future__ import annotations + +import argparse +import datetime +import json +import os +import re +import shutil +import subprocess +import sys +import urllib.request + +DEFAULT_APP_ID = "429da6f5-e7b0-40e6-a602-adaa8dcde8b9" +OPENVSX_API = "https://open-vsx.org/api/innoverio/vscode-dbt-power-user" + +# Per-install latest extension version over the window. +KQL = ( + "customEvents " + "| where isnotempty(tostring(customDimensions['common.vscodemachineid'])) " + "| extend m=tostring(customDimensions['common.vscodemachineid']), " + "v=tostring(customDimensions['common.extversion']) " + "| where isnotempty(v) " + "| summarize arg_max(timestamp, v) by m " + "| summarize installs=count() by v " + "| sort by installs desc " + "| take 60" +) + +# Per-install latest (version, os) over the window — same arg_max-per-machine +# logic, additionally bucketed by common.os ('darwin'/'win32'/'linux'). Powers +# the per-version×OS "% of users impacted" figures on the board. Each install is +# counted once at its most-recent (version, os). +KQL_BY_OS = ( + "customEvents " + "| where isnotempty(tostring(customDimensions['common.vscodemachineid'])) " + "| extend m=tostring(customDimensions['common.vscodemachineid']), " + "v=tostring(customDimensions['common.extversion']), " + "os=tostring(customDimensions['common.os']) " + "| where isnotempty(v) and isnotempty(os) " + "| summarize arg_max(timestamp, v, os) by m " + "| summarize installs=count() by v, os " + "| sort by installs desc " + "| take 2000" # high cap: the (version×OS) cardinality is the denominator for + # the impact shares, so truncation would inflate every percentage. 2000 >> the + # realistic count (~100 versions × 3 OSes), so this is effectively "all". +) + +# common.os value -> the board's OS label (matches build-matrix OSES / aggregate OS_ORDER). +OS_LABEL = {"darwin": "macos", "win32": "windows", "linux": "linux"} + +_SEMVER = re.compile(r"^\d+\.\d+\.\d+$") # plain release versions only (no -pre) + + +def _rows_from_table(table) -> list[tuple[str, int]]: + cols = [c["name"] for c in table["columns"]] + vi, ni = cols.index("v"), cols.index("installs") + return [(row[vi], int(row[ni])) for row in table["rows"]] + + +def _query_rest(app_id: str, api_key: str, window_days: int) -> list[tuple[str, int]]: + url = f"https://api.applicationinsights.io/v1/apps/{app_id}/query?timespan=P{window_days}D" + req = urllib.request.Request( + url, + data=json.dumps({"query": KQL}).encode(), + headers={"x-api-key": api_key, "Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=60) as r: + data = json.load(r) + return _rows_from_table(data["tables"][0]) + + +def _query_az(app_id: str, window_days: int) -> list[tuple[str, int]]: + out = subprocess.run( + [ + "az", + "monitor", + "app-insights", + "query", + "--app", + app_id, + "--offset", + f"{window_days}d", + "--analytics-query", + KQL, + "-o", + "json", + ], + capture_output=True, + text=True, + check=True, + ).stdout + return _rows_from_table(json.loads(out)["tables"][0]) + + +def query_rows(app_id: str, window_days: int) -> list[tuple[str, int]] | None: + """Fetch the per-install latest-version rows via whichever backend is + available: the REST API when APPINSIGHTS_API_KEY is set (CI), else the az CLI + when logged in (local). Returns None if neither backend is usable.""" + api_key = os.environ.get("APPINSIGHTS_API_KEY") + if api_key: + return _query_rest(app_id, api_key, window_days) + if shutil.which("az"): + return _query_az(app_id, window_days) + return None + + +def _rows_by_os_from_table(table) -> list[tuple[str, str, int]]: + cols = [c["name"] for c in table["columns"]] + vi, oi, ni = cols.index("v"), cols.index("os"), cols.index("installs") + return [(row[vi], row[oi], int(row[ni])) for row in table["rows"]] + + +def query_rows_by_os( + app_id: str, window_days: int +) -> list[tuple[str, str, int]] | None: + """Per-install latest (version, os) rows via REST (APPINSIGHTS_API_KEY) or az + CLI. Returns None if neither backend is usable, or [] if the query yields + nothing (so callers can degrade to version-only impact gracefully).""" + api_key = os.environ.get("APPINSIGHTS_API_KEY") + try: + if api_key: + url = f"https://api.applicationinsights.io/v1/apps/{app_id}/query?timespan=P{window_days}D" + req = urllib.request.Request( + url, + data=json.dumps({"query": KQL_BY_OS}).encode(), + headers={"x-api-key": api_key, "Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=60) as r: + data = json.load(r) + return _rows_by_os_from_table(data["tables"][0]) + if shutil.which("az"): + out = subprocess.run( + [ + "az", + "monitor", + "app-insights", + "query", + "--app", + app_id, + "--offset", + f"{window_days}d", + "--analytics-query", + KQL_BY_OS, + "-o", + "json", + ], + capture_output=True, + text=True, + check=True, + ).stdout + return _rows_by_os_from_table(json.loads(out)["tables"][0]) + except Exception as e: # noqa: BLE001 - impact data is optional; never crash CI + print(f"::warning::version×OS impact query failed ({e})", file=sys.stderr) + return None + return None + + +def impact_by_version_os( + rows_by_os: list[tuple[str, str, int]], published: set | None = None +) -> dict: + """Collapse (version, os) rows into a nested share map over the FULL running + base (every install counts in the denominator, so shares are honest): + + { "": { + "total_installs": int, "total_share": float, # all OSes + "os": { "": {"installs": int, "share": float} } } } + + 'share' is percent of all installs (version-total and version×OS). os labels + use the board's macos/windows/linux naming. + + `published`: if given, only versions in this set are EMITTED into the map + (junk/fork strings like '1.2.16' that pass the x.y.z regex but aren't real + releases are excluded — otherwise they can be mistaken for the newest target). + The denominator stays the FULL running base (all installs counted), so the + emitted shares remain honest about what fraction of real users they cover.""" + total = sum(n for (_v, _o, n) in rows_by_os) or 1 + vmap: dict[str, dict] = {} + for v, raw_os, n in rows_by_os: + if not _SEMVER.match(v): + continue # junk/fork version strings can't be a tested baseline + if published and v not in published: + continue # not a real published release — keep out of the map + os_label = OS_LABEL.get(raw_os, raw_os) + e = vmap.setdefault(v, {"total_installs": 0, "os": {}}) + e["total_installs"] += n + oe = e["os"].setdefault(os_label, {"installs": 0}) + oe["installs"] += n + for v, e in vmap.items(): + e["total_share"] = round(100 * e["total_installs"] / total, 2) + for oe in e["os"].values(): + oe["share"] = round(100 * oe["installs"] / total, 2) + return vmap + + +def dist_from_rows(rows: list[tuple[str, int]]) -> list[tuple[str, int, float]]: + """Collapse duplicate version rows (App Insights sharding) and attach share %. + Returns (version, installs, share) sorted by installs descending.""" + merged: dict[str, int] = {} + for v, n in rows: + merged[v] = merged.get(v, 0) + n + total = sum(merged.values()) or 1 + return [ + (v, n, round(100 * n / total, 2)) + for v, n in sorted(merged.items(), key=lambda x: x[1], reverse=True) + ] + + +def published_versions() -> set[str]: + """Real installable releases from Open VSX. This is the authoritative list of + published versions (verified complete, ~100+ entries) and correctly excludes + junk/fork version strings (e.g. 1.2.16, 0.34.2003) that appear in telemetry.""" + try: + with urllib.request.urlopen(OPENVSX_API, timeout=30) as r: + data = json.load(r) + return {v for v in data.get("allVersions", {}) if _SEMVER.match(v)} + except Exception as e: # noqa: BLE001 - network optional; degrade gracefully + print( + f"::warning::could not fetch Open VSX versions ({e}); skipping publish filter", + file=sys.stderr, + ) + return set() + + +def _semver_key(v: str) -> tuple[int, int, int]: + a, b, c = v.split(".") + return (int(a), int(b), int(c)) + + +def pick_target(dist, published) -> str: + """The version we upgrade TO: the highest published semver still seen running.""" + cand = [ + v + for (v, _n, _s) in dist + if _SEMVER.match(v) and (not published or v in published) + ] + return max(cand, key=_semver_key, default="") + + +def pick_baselines( + dist, target, min_share, max_baselines, include_oldest_above, published +): + """dist: list of (version, installs, share). Returns a semver-sorted baseline list: + the most-run older *published* versions (common upgrade paths) plus the oldest + version still meaningfully present (big-gap upgrade coverage).""" + eligible = [ + (v, n, s) + for (v, n, s) in dist + if v != target and _SEMVER.match(v) and (not published or v in published) + ] + popular = [ + v + for (v, n, s) in sorted(eligible, key=lambda x: x[1], reverse=True) + if s >= min_share + ][: max(0, max_baselines - 1)] + oldish = sorted( + [v for (v, n, s) in eligible if s >= include_oldest_above], key=_semver_key + ) + chosen = set(popular) + if oldish: + chosen.add(oldish[0]) + return sorted(chosen, key=_semver_key)[:max_baselines] + + +def pick_by_coverage(dist, target, coverage_pct, published, max_baselines): + """Pick the fewest published upgrade-from versions (most-installed first) whose + install share — together with the target's own share — reaches coverage_pct of + the total running base. Returns a semver-sorted baseline list (excludes target). + + Junk/unpublished version strings are skipped (they can't be installed as a + baseline), but their installs still count in the denominator so coverage is + honest about what fraction of the REAL running base we test.""" + total = sum(n for (_v, n, _s) in dist) or 1 + target_installs = sum(n for (v, n, _s) in dist if v == target) + eligible = sorted( + [ + (v, n) + for (v, n, _s) in dist + if v != target and _SEMVER.match(v) and (not published or v in published) + ], + key=lambda x: x[1], + reverse=True, + ) + chosen: list[str] = [] + covered = target_installs + for v, n in eligible: + if 100.0 * covered / total >= coverage_pct or len(chosen) >= max_baselines: + break + chosen.append(v) + covered += n + return sorted(chosen, key=_semver_key) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument( + "--app-id", default=os.environ.get("APPINSIGHTS_APP_ID", DEFAULT_APP_ID) + ) + ap.add_argument("--window-days", type=int, default=30) + ap.add_argument( + "--min-share", + type=float, + default=1.0, + help="min %% share for a popular baseline", + ) + ap.add_argument( + "--include-oldest-above", + type=float, + default=0.5, + help="also include the oldest version whose share is >= this %%", + ) + ap.add_argument("--max-baselines", type=int, default=6) + ap.add_argument( + "--target", + default="", + help="version we upgrade TO; default = highest published semver seen", + ) + ap.add_argument("--out", default="test-matrix/active-versions.json") + ap.add_argument( + "--generated-at", default="", help="ISO timestamp to stamp; default = now (UTC)" + ) + args = ap.parse_args() + + rows = query_rows(args.app_id, args.window_days) + if rows is None: + print( + "::error::no APPINSIGHTS_API_KEY and az CLI not available", file=sys.stderr + ) + return 2 + + dist = dist_from_rows(rows) + total = sum(n for _v, n, _s in dist) or 1 + published = published_versions() + target = args.target or pick_target(dist, published) + baselines = pick_baselines( + dist, + target, + args.min_share, + args.max_baselines, + args.include_oldest_above, + published, + ) + + stamp = args.generated_at or datetime.datetime.now(datetime.timezone.utc).isoformat( + timespec="seconds" + ) + payload = { + "generated_at": stamp, + "app_id": args.app_id, + "window_days": args.window_days, + "target": target, + "baselines": baselines, + "distribution": [{"version": v, "installs": n, "share": s} for v, n, s in dist], + } + os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True) + with open(args.out, "w") as f: + json.dump(payload, f, indent=2) + f.write("\n") + print( + f"target={target} baselines={baselines} ({len(dist)} versions, {total} installs)" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test-matrix/aggregate.py b/test-matrix/aggregate.py new file mode 100644 index 000000000..cfc0035c7 --- /dev/null +++ b/test-matrix/aggregate.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python3 +"""Aggregate per-cell RESULT_JSON files into the install + update matrices, +a Slack payload, and a blocking-gate exit code. + +Usage: + python3 aggregate.py --results-dir --out-dir [--target ] [--trigger pr|schedule|release] +""" +from __future__ import annotations + +import argparse +import glob +import json +import os +import re +import sys + +# Only these runtimes can fail a release. Everything else is informational. +BLOCKING_RUNTIMES = {"vscode"} + +# Stable display order; runtimes not listed are appended alphabetically. +RUNTIME_ORDER = [ + "vscode", + "vscode-insiders", + "cursor", + "windsurf", + "kiro", + "code-server", +] +OS_ORDER = ["linux", "windows", "macos"] + + +def _cell_symbol(cell: dict) -> str: + if cell.get("status") == "skip": + return "⏭️" + if cell.get("status") == "pass": + return "✅" + # failed + return "❌" if cell.get("runtime") in BLOCKING_RUNTIMES else "⚠️" + + +def _runtime_sort_key(rt: str): + return (RUNTIME_ORDER.index(rt) if rt in RUNTIME_ORDER else len(RUNTIME_ORDER), rt) + + +def _os_sort_key(os_name: str): + return (OS_ORDER.index(os_name) if os_name in OS_ORDER else len(OS_ORDER), os_name) + + +def _os_share(impact: dict, version: str, os_name: str): + """Percent of the running base on (version, os), or None if unknown. + `impact` is the build-matrix per-version×OS map.""" + v = impact.get(version) + if not v: + return None + oe = v.get("os", {}).get(os_name) + return oe.get("share") if oe else None + + +def _version_share(impact: dict, version: str): + """Percent of the running base on a version across all OSes, or None.""" + v = impact.get(version) + return v.get("total_share") if v else None + + +def _pct(x) -> str: + return f"{x:.1f}%" if isinstance(x, (int, float)) else "?" + + +def _is_semver(v) -> bool: + return bool(re.match(r"^\d+\.\d+\.\d+$", v)) if isinstance(v, str) else False + + +def _semver_key(v: str): + """Sort key for plain x.y.z versions; non-semver sorts last.""" + try: + a, b, c = v.split(".") + return (0, int(a), int(b), int(c)) + except (ValueError, AttributeError): + return (1, 0, 0, 0) + + +def _newest_impact_version(impact: dict) -> str: + """Highest semver present in the impact map (the de-facto 'latest' a fresh + install lands on), so the install board can show target-version impact even + when result cells label `to` as 'latest'/'pr-build' rather than a semver.""" + sv = [v for v in impact if _is_semver(v)] + return max(sv, key=_semver_key) if sv else "" + + +LEGEND = ( + "**Legend:** ✅ pass · ❌ blocking failure (fails the release gate) · " + "⚠️ non-blocking issue (forks/Insiders — informational) · ⏭️ skipped · — not run. " + "Percentages are the share of the live user base on that version×OS." +) + + +def build_matrices(results: list[dict], impact: dict | None = None) -> dict: + impact = impact or {} + install = [r for r in results if r.get("scenario") == "fresh"] + upgrade = [r for r in results if r.get("scenario") == "upgrade"] + + has_blocking_failure = any( + r.get("status") == "fail" and r.get("runtime") in BLOCKING_RUNTIMES + for r in results + ) + + install_md = _render_install(install, impact) + update_md = _render_update(upgrade, impact) + slack_blocks = _render_slack(results, has_blocking_failure, impact) + + return { + "install_md": install_md, + "update_md": update_md, + "slack_blocks": slack_blocks, + "has_blocking_failure": has_blocking_failure, + } + + +def _render_install(cells: list[dict], impact: dict | None = None) -> str: + impact = impact or {} + runtimes = sorted({c["runtime"] for c in cells}, key=_runtime_sort_key) + oses = sorted({c["os"] for c in cells}, key=_os_sort_key) + by = {(c["runtime"], c["os"]): c for c in cells} + # Target version (what a fresh install lands on). Cells label `to` as a + # semver, or "latest"/"pr-build" in CI — in those cases fall back to the + # newest version in the impact map so the per-OS target share still renders. + raw_target = next( + ( + c.get("to") or c.get("target") + for c in cells + if c.get("to") or c.get("target") + ), + "", + ) + target = raw_target if _is_semver(raw_target) else _newest_impact_version(impact) + + lines = [ + "### Install matrix (fresh install of target)", + "", + "_Scenario: clean install of the latest published build (no prior version)._", + "", + ] + lines.append("| Runtime | " + " | ".join(oses) + " |") + lines.append("|---|" + "---|" * len(oses)) + for rt in runtimes: + row = [rt] + for os_name in oses: + cell = by.get((rt, os_name)) + row.append(_cell_symbol(cell) if cell else "—") + lines.append("| " + " | ".join(row) + " |") + lines.append("") + # Impact context: % of the running base already on the target version, by OS — + # i.e. who a broken fresh install would affect among current users of latest. + if impact and target and impact.get(target): + parts = [] + for os_name in oses: + sh = _os_share(impact, target, os_name) + if sh is not None: + parts.append(f"{os_name} {_pct(sh)}") + if parts: + lines.append( + f"_Users on target `{target}` (share of running base): " + + " · ".join(parts) + + "._" + ) + lines.append("") + return "\n".join(lines) + + +def _cell_text(cell: dict, impact: dict, version: str, os_name: str) -> str: + """A status glyph plus, when known, the version×OS user-impact share.""" + sym = _cell_symbol(cell) + sh = _os_share(impact, version, os_name) + return f"{sym} {_pct(sh)}" if sh is not None else sym + + +def _render_update(cells: list[dict], impact: dict | None = None) -> str: + impact = impact or {} + lines = [ + "### Update matrix — upgrade to latest", + "", + "_Existing install on an older version is updated **directly to latest** " + "(VS Code upgrades in one hop — it does not step through intermediate " + "versions). Each cell: result + share of the live user base on that " + "version×OS = who a broken upgrade hits._", + "", + ] + if not cells: + lines.append("_No upgrade cells in this run._") + lines.append("") + return "\n".join(lines) + + by = {(c["runtime"], c["os"], c.get("from")): c for c in cells} + # Split by runtime CLASS, not baseline count: the blocking runtimes (vscode) + # get the dense version×OS grid even if telemetry picks only one baseline; + # forks/code-server (which only test latest-minus-one) get the compact list so + # the grid isn't a wall of "—". (Splitting on len(baselines) mislabels a + # single-baseline vscode run as "Forks".) + base_by_rt: dict[str, set] = {} + for c in cells: + if c.get("from"): + base_by_rt.setdefault(c["runtime"], set()).add(c["from"]) + grid_rts = sorted( + (rt for rt in base_by_rt if rt in BLOCKING_RUNTIMES), key=_runtime_sort_key + ) + single_rts = sorted( + (rt for rt in base_by_rt if rt not in BLOCKING_RUNTIMES), key=_runtime_sort_key + ) + + # --- Dense grid: blocking runtimes (e.g. vscode), newest baseline first. + for rt in grid_rts: + # Newest real version first; any non-semver string sorts to the END + # (reverse=True alone would push non-semver to the front). + all_b = base_by_rt[rt] + semver_b = sorted( + (v for v in all_b if _is_semver(v)), key=_semver_key, reverse=True + ) + non_semver_b = sorted(v for v in all_b if not _is_semver(v)) + baselines = semver_b + non_semver_b + oses = sorted({c["os"] for c in cells if c["runtime"] == rt}, key=_os_sort_key) + lines.append(f"**{rt}** — upgrade from each version → latest") + lines.append("") + lines.append("| from \\ OS | " + " | ".join(oses) + " |") + lines.append("|:--|" + ":-:|" * len(oses)) + for b in baselines: + row = [f"`{b}`"] + for os_name in oses: + cell = by.get((rt, os_name, b)) + row.append(_cell_text(cell, impact, b, os_name) if cell else "—") + lines.append("| " + " | ".join(row) + " |") + lines.append("") + + # --- Compact list: single-baseline runtimes (forks / code-server). + if single_rts: + lines.append("**Forks & code-server** — upgrade from latest-minus-one → latest") + lines.append("") + lines.append("| Runtime | OS | From | Result | Users on that version×OS |") + lines.append("|:--|:--|:--|:-:|:--|") + srows = sorted( + [c for c in cells if c["runtime"] in single_rts], + key=lambda c: (_runtime_sort_key(c["runtime"]), _os_sort_key(c["os"])), + ) + for c in srows: + sh = _os_share(impact, c.get("from"), c.get("os")) + lines.append( + f"| {c['runtime']} | {c['os']} | `{c.get('from')}` | " + f"{_cell_symbol(c)} | {_pct(sh) if sh is not None else '—'} |" + ) + lines.append("") + + # Worst blocking failures, ranked by user impact (actionable triage list). + fails = [] + for c in cells: + if c.get("status") == "fail" and c.get("runtime") in BLOCKING_RUNTIMES: + sh = _os_share(impact, c.get("from"), c.get("os")) + fails.append((sh if sh is not None else -1.0, c)) + if fails: + fails.sort(key=lambda x: x[0], reverse=True) + lines.append("**⛔ Blocking upgrade failures (most users first):**") + for sh, c in fails: + tag = f"**{_pct(sh)} of users**" if sh >= 0 else "impact unknown" + lines.append( + f"- `{c.get('from')}` → latest on **{c.get('os')}** — {tag}" + + (f" — {c.get('reason')}" if c.get("reason") else "") + ) + lines.append("") + return "\n".join(lines) + + +def _render_slack( + results: list[dict], has_blocking_failure: bool, impact: dict | None = None +) -> list[dict]: + impact = impact or {} + total = len(results) + passed = sum(1 for r in results if r.get("status") == "pass") + failed = [r for r in results if r.get("status") == "fail"] + skipped = [r for r in results if r.get("status") == "skip"] + headline = ( + "❌ Install/Update matrix: BLOCKING failure" + if has_blocking_failure + else ( + "⚠️ Install/Update matrix: non-blocking issues" + if failed + else "✅ Install/Update matrix: all green" + ) + ) + detail = f"{passed}/{total} cells passed" + if failed: + + def _impact_tag(r): + sh = ( + _os_share(impact, r.get("from"), r.get("os")) + if (impact and r.get("from")) + else None + ) + return f" [{_pct(sh)} users]" if sh is not None else "" + + detail += "\nFailures:\n" + "\n".join( + f"• {r.get('runtime','?')}/{r.get('os','?')}/{r.get('scenario','?')}" + + (f" (from {r['from']})" if r.get("from") else "") + + _impact_tag(r) + + f": {r.get('reason') or 'failed'}" + for r in failed + ) + if skipped: + detail += "\nSkipped:\n" + "\n".join( + f"• {r.get('runtime','?')}/{r.get('os','?')}: {r.get('reason') or 'skipped'}" + for r in skipped + ) + return [ + {"type": "section", "text": {"type": "mrkdwn", "text": f"*{headline}*"}}, + {"type": "section", "text": {"type": "mrkdwn", "text": detail}}, + ] + + +def _load_results(results_dir: str): + """Load every RESULT_JSON. Returns (results, errors) where errors is a list + of (path, message) for files that could not be parsed.""" + out = [] + errors = [] + for path in sorted( + glob.glob(os.path.join(results_dir, "**", "*.json"), recursive=True) + ): + try: + with open(path) as f: + out.append(json.load(f)) + except (json.JSONDecodeError, OSError) as e: + errors.append((path, str(e))) + return out, errors + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--results-dir", required=True) + ap.add_argument("--out-dir", required=True) + ap.add_argument("--target", default="") + ap.add_argument("--trigger", default="manual") + ap.add_argument( + "--impact-file", + default="", + help="path to JSON file with the per-version×OS impact map " + "(build-matrix.py `impact=` output). Optional; board omits %% if absent.", + ) + args = ap.parse_args() + + impact = {} + if args.impact_file and os.path.exists(args.impact_file): + try: + with open(args.impact_file) as f: + impact = json.load(f) or {} + except (json.JSONDecodeError, OSError) as e: + print(f"::warning::could not read impact file {args.impact_file}: {e}") + + results, load_errors = _load_results(args.results_dir) + os.makedirs(args.out_dir, exist_ok=True) + for path, msg in load_errors: + print(f"::warning::unreadable result file {path}: {msg}") + + if not results: + # No cell produced a result (e.g. every runner died before writing one). + # We cannot certify the matrix, so block and still emit a visible board. + note = "No result files found — treating as a blocking failure" + print(f"::error::{note}") + board = f"## VSIX Install + Update Matrix\n\n:x: {note}\n" + for name in ("install-matrix.md", "update-matrix.md", "matrix.md"): + with open(os.path.join(args.out_dir, name), "w") as f: + f.write(board) + with open(os.path.join(args.out_dir, "slack.json"), "w") as f: + json.dump( + { + "blocks": [ + { + "type": "section", + "text": {"type": "mrkdwn", "text": f"*❌ {note}*"}, + } + ] + }, + f, + indent=2, + ) + return 1 + + out = build_matrices(results, impact) + total = len(results) + passed = sum(1 for r in results if r.get("status") == "pass") + failed_cells = [r for r in results if r.get("status") == "fail"] + blocking_failed = sum( + 1 for r in failed_cells if r.get("runtime") in BLOCKING_RUNTIMES + ) + nonblocking_failed = len(failed_cells) - blocking_failed + status_line = ( + "❌ **Blocking failure**" + if out["has_blocking_failure"] + else ("⚠️ **Non-blocking issues**" if failed_cells else "✅ **All green**") + ) + # Resolve the title to the actual target VERSION. `--target latest` (the usual + # workflow_dispatch value) is not a version; show the semver a fresh install + # really lands on so the title agrees with the per-version numbers in the body. + resolved = ( + args.target if _is_semver(args.target) else _newest_impact_version(impact) + ) + if resolved and not _is_semver(args.target): + title_target = ( + f"{resolved} ({args.target or 'latest'})" # e.g. "0.61.6 (latest)" + ) + else: + title_target = resolved or args.target or "latest" + # Count line: separate blocking from non-blocking so "N failed" can't be misread + # as N release-blocking failures when they are informational fork/Insiders ⚠️. + counts = f"**{passed}/{total}** cells passed" + if failed_cells: + counts += ( + f" · {blocking_failed} blocking ❌ · {nonblocking_failed} non-blocking ⚠️" + ) + header = ( + f"## VSIX Install + Update Matrix — target `{title_target}` ({args.trigger})\n\n" + f"{status_line} · {counts}\n\n" + LEGEND + "\n\n" + ) + combined = header + out["install_md"] + "\n" + out["update_md"] + with open(os.path.join(args.out_dir, "install-matrix.md"), "w") as f: + f.write(out["install_md"]) + with open(os.path.join(args.out_dir, "update-matrix.md"), "w") as f: + f.write(out["update_md"]) + with open(os.path.join(args.out_dir, "matrix.md"), "w") as f: + f.write(combined) + with open(os.path.join(args.out_dir, "slack.json"), "w") as f: + json.dump({"blocks": out["slack_blocks"]}, f, indent=2) + + print(combined) + if out["has_blocking_failure"]: + print("::error::Blocking-lane cell(s) failed — see matrix above") + return 1 + if load_errors: + # A result file existed but was corrupt — we can't confirm that cell, so block. + print("::error::Some result files were unreadable — cannot certify the matrix") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test-matrix/build-matrix.py b/test-matrix/build-matrix.py new file mode 100644 index 000000000..411254e02 --- /dev/null +++ b/test-matrix/build-matrix.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Compute the install/update matrix `include` array for GitHub Actions. + +Upgrade-from versions are chosen by COVERAGE: the fewest most-installed published +versions whose combined install share (with the target's own share) reaches +TARGET_COVERAGE_PCT of the live running base. Live data comes from App Insights +when APPINSIGHTS_API_KEY is set; otherwise a hardcoded fallback is used (fork PRs +/ telemetry outage) so CI never breaks. + +Every chosen upgrade-from version is tested on ALL THREE OSes (linux/macos/ +windows), plus a fresh-install cell per OS and an Insiders fresh cell on Linux. + +Emits GITHUB_OUTPUT-style lines on stdout: + matrix= + baselines= + source=live|fallback + coverage= +""" +from __future__ import annotations + +import importlib.util +import json +import os +import pathlib +import sys + +# Coverage goal: pick enough upgrade-from versions to test this % of the base. +TARGET_COVERAGE_PCT = 90.0 +# Hard cap so a pathological distribution can't explode CI (3 OSes per baseline). +MAX_BASELINES = 10 + +# Fallback when telemetry is unavailable (fork PRs / unset secret / query error). +# Snapshot of the 2026-05-30 live distribution reaching ~90% coverage; refresh +# occasionally by running `active-versions.py` with the secret. +FALLBACK_TARGET = "0.61.5" +FALLBACK_BASELINES = ["0.60.7", "0.61.0", "0.61.1", "0.61.2", "0.61.3", "0.61.4"] + +OSES = [ + ("ubuntu-latest", "linux", "linux-x64"), + ("macos-latest", "macos", "darwin-arm64"), + ("windows-latest", "windows", "win32-x64"), +] + +# Reuse the tested query/selection logic from active-versions.py (hyphenated +# filename -> load via importlib, same as the unit tests do). +_HERE = pathlib.Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location( + "active_versions", _HERE / "active-versions.py" +) +av = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(av) + + +def live_plan(window_days: int = 30): + """Return (target, baselines, coverage_pct, impact) chosen DYNAMICALLY from + live App Insights — via the REST API when APPINSIGHTS_API_KEY is set (CI) or + the az CLI when logged in (local). Returns None if no backend / query fails / + empty. `impact` is the per-version×OS install-share map (or {} if the impact + query is unavailable — version selection still succeeds without it).""" + app_id = os.environ.get("APPINSIGHTS_APP_ID", av.DEFAULT_APP_ID) + try: + rows = av.query_rows(app_id, window_days) + except Exception as e: # noqa: BLE001 - any failure => fall back, never crash CI + print( + f"::warning::App Insights query failed ({e}); using fallback", + file=sys.stderr, + ) + return None + if not rows: + return None + dist = av.dist_from_rows(rows) + total = sum(n for (_v, n, _s) in dist) or 1 + published = av.published_versions() + target = av.pick_target(dist, published) + baselines = av.pick_by_coverage( + dist, target, TARGET_COVERAGE_PCT, published, MAX_BASELINES + ) + if not target or not baselines: + return None + covered = sum(n for (v, n, _s) in dist if v == target or v in set(baselines)) + # Per-version×OS impact share (best-effort; board degrades to no-% if empty). + try: + rows_by_os = av.query_rows_by_os(app_id, window_days) + # Pass `published` so junk/fork version strings (e.g. 1.2.16) are kept out + # of the impact map and can't be mistaken for the newest target version. + impact = av.impact_by_version_os(rows_by_os, published) if rows_by_os else {} + except Exception as e: # noqa: BLE001 - impact is optional, never block the plan + print( + f"::warning::impact query failed ({e}); board will omit %", file=sys.stderr + ) + impact = {} + return target, baselines, round(100 * covered / total, 1), impact + + +def _cell(os_name, osl, target, vscode, mode, frm): + # "from" is a Python keyword, so build the dict explicitly. + return { + "os": os_name, + "osl": osl, + "target": target, + "vscode": vscode, + "mode": mode, + "from": frm, + } + + +def build_include(baselines: list[str]) -> list[dict]: + """Construct the matrix include list. + + - For EACH OS (linux/macos/windows): one stock-VSCode fresh cell + one + stock-VSCode upgrade cell per chosen upgrade-from version. + - Plus one Insiders fresh cell on Linux (non-blocking early-warning). + """ + inc: list[dict] = [] + for os_name, osl, tgt in OSES: + inc.append(_cell(os_name, osl, tgt, "stable", "fresh", "")) + for b in baselines: + inc.append(_cell(os_name, osl, tgt, "stable", "upgrade", b)) + inc.append(_cell("ubuntu-latest", "linux", "linux-x64", "insiders", "fresh", "")) + return inc + + +def main() -> int: + plan = live_plan() + if plan: + target, baselines, coverage, impact = plan + source, coverage_s = "live", str(coverage) + else: + target, baselines, impact = FALLBACK_TARGET, FALLBACK_BASELINES, {} + source, coverage_s = "fallback", "n/a" + + include = build_include(baselines) + print("matrix=" + json.dumps({"include": include}, separators=(",", ":"))) + print("baselines=" + ",".join(baselines)) + print("source=" + source) + print("coverage=" + coverage_s) + # Per-version×OS install-share map for the board's "% users impacted" column. + # Empty {} on fallback / impact-query failure — the board then omits the %. + print("impact=" + json.dumps(impact, separators=(",", ":"))) + print( + f"target={target} source={source} coverage={coverage_s}% " + f"baselines={baselines} cells={len(include)} impact_versions={len(impact)}", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test-matrix/codeserver-cell.sh b/test-matrix/codeserver-cell.sh new file mode 100755 index 000000000..4fb27587b --- /dev/null +++ b/test-matrix/codeserver-cell.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Runs the existing docker-setup/vsix-smoke.sh and converts its PASS/FAIL into a +# matrix RESULT_JSON. Usage: +# bash test-matrix/codeserver-cell.sh --mode fresh|upgrade [--from ] \ +# [--vsix-file |--target latest] --out +set -uo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +MODE="fresh"; FROM=""; VSIX=""; TARGET="latest"; OUT="/tmp/codeserver.json" +while [ $# -gt 0 ]; do + case "$1" in + --mode) MODE="$2"; shift 2;; + --from) FROM="$2"; shift 2;; + --vsix-file) VSIX="$2"; shift 2;; + --target) TARGET="$2"; shift 2;; + --out) OUT="$2"; shift 2;; + *) shift;; + esac +done + +args=() +[ -n "$VSIX" ] && args+=(--vsix-file "$VSIX") +[ "$MODE" = "upgrade" ] && [ -n "$FROM" ] && args+=(--from-version "$FROM") + +start=$(date +%s) +if VSIX_SMOKE_REPORT=/tmp/cs-smoke.md bash "$REPO_ROOT/docker-setup/vsix-smoke.sh" "${args[@]}"; then + STATUS="pass"; REASON=""; OKS=true +else + STATUS="fail"; REASON="$(tail -3 /tmp/cs-smoke.md 2>/dev/null | tr '\n' ' ' | sed 's/"/'"'"'/g')"; OKS=false +fi +end=$(date +%s) + +python3 - "$OUT" "$MODE" "$FROM" "$TARGET" "$STATUS" "$REASON" "$OKS" $((end-start)) <<'PY' +import json, sys +out, mode, frm, target, status, reason, oks, dur = sys.argv[1:9] +ok = oks == "true" +json.dump({ + "runtime": "code-server", "os": "linux", "scenario": mode, + "from": frm or None, "to": "pr-build" if target != "latest" else "latest", + "install_ok": ok, "deps_resolved": {}, "activation_ok": ok, "dbt_flow_ok": ok, + "status": status, "reason": reason, "duration_s": int(dur), + "log_artifact": "codeserver.json", +}, open(out, "w"), indent=2) +print("wrote", out, status) +PY +[ "$STATUS" = "pass" ] diff --git a/test-matrix/cursor-cell.mjs b/test-matrix/cursor-cell.mjs new file mode 100644 index 000000000..f1db7d1c2 --- /dev/null +++ b/test-matrix/cursor-cell.mjs @@ -0,0 +1,326 @@ +#!/usr/bin/env node +// Driver for one Cursor matrix cell (non-blocking fork lane, Linux only). +// +// Cursor is an Anysphere VSCode fork. @vscode/test-electron CANNOT drive it +// (it downloads/manages Microsoft VSCode), AND Cursor's bundled CLI shim hangs +// on headless `--install-extension` (verified: it wedges the job to timeout). +// So we install WITHOUT the CLI: download each .vsix (deps + baseline from Open +// VSX by id, target from the local build) and UNZIP it straight into a throwaway +// extensions-dir (a .vsix is a zip; installed layout is +// /.-/ holding the `extension/` contents). +// Then LAUNCH the extracted Cursor binary headless against the dbt fixture and +// scan its logs for the activation marker — mirroring the VSCode lane's assert. +// +// Usage: +// node test-matrix/cursor-cell.mjs --bin --mode fresh|upgrade \ +// --target [--from ] --out +import { execFileSync, spawn } from "node:child_process"; +import { + closeSync, existsSync, mkdirSync, mkdtempSync, openSync, readdirSync, + readFileSync, renameSync, rmSync, writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const EXTENSION_ID = "innoverio.vscode-dbt-power-user"; +const OPENVSX = "https://open-vsx.org/api"; +const VSIX_TARGET_PLATFORM = "linux-x64"; // runner arch — must match native deps +// Dependency extensions resolved from Open VSX by id (all verified present). +const DEPS = ["samuelcolvin.jinjahtml", "ms-python.python", "altimateai.vscode-altimate-mcp-server"]; +const INIT_OK = "Initialized dbt project"; +const INIT_FAIL = "Unable to register dbt project"; +const ACTIVATION_TIMEOUT_MS = 90_000; // matches the VSCode lane's in-host wait +// Absolute backstop: the whole cell (download already done by provisioner) must +// finish well under the job timeout even if Cursor wedges. SIGKILLs the node +// process so xvfb-run returns and the job ends with a written RESULT_JSON. +const HARD_DEADLINE_MS = 8 * 60_000; + +function arg(name, def = undefined) { + const i = process.argv.indexOf(`--${name}`); + if (i === -1) return def; + const v = process.argv[i + 1]; + return v && !v.startsWith("--") ? v : true; +} + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// Recursively read every *.log under a dir (Cursor writes LogOutputChannels + +// console output under /logs/**). +function readAllLogs(uddDir) { + const out = []; + const stack = [join(uddDir, "logs")]; + while (stack.length) { + const dir = stack.pop(); + let entries = []; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + continue; + } + for (const e of entries) { + const p = join(dir, e.name); + if (e.isDirectory()) stack.push(p); + else if (e.name.endsWith(".log")) { + try { + out.push(readFileSync(p, "utf8")); + } catch { + /* ignore rotating log */ + } + } + } + } + return out.join("\n"); +} + +// Install a .vsix WITHOUT any editor CLI: unzip it and lay the `extension/` +// contents into /.-/ (VSCode's on-disk format). +function installVsixToDir(vsixPath, extDir) { + const tmp = mkdtempSync(join(tmpdir(), "vsix-x-")); + execFileSync("unzip", ["-q", "-o", vsixPath, "-d", tmp], { stdio: "pipe", timeout: 120_000 }); + const pkg = JSON.parse(readFileSync(join(tmp, "extension", "package.json"), "utf8")); + const folder = `${pkg.publisher}.${pkg.name}-${pkg.version}`; + const dest = join(extDir, folder); + rmSync(dest, { recursive: true, force: true }); + renameSync(join(tmp, "extension"), dest); + rmSync(tmp, { recursive: true, force: true }); + return `${pkg.publisher}.${pkg.name}`; +} + +// Download a URL to a file using curl (handles Open VSX redirects + retries). +// --retry-all-errors is essential: Open VSX /file/ endpoints answer 302 to a CDN, +// and a transient CDN error AFTER the followed redirect is NOT retried by plain +// --retry (which only retries a fixed set of transient conditions). Under QEMU the +// CDN occasionally blips, so without this a single hiccup fails the whole cell. +function curlDownload(url, outFile) { + execFileSync( + "curl", + ["-fSL", "--retry", "5", "--retry-delay", "3", "--retry-all-errors", + "--connect-timeout", "30", "-o", outFile, url], + { stdio: "pipe", timeout: 300_000 }, + ); + return outFile; +} + +// Resolve an Open VSX .vsix download URL for an extension id (+ optional version +// + target platform), via the registry metadata `files.download` field. +function openVsxDownloadUrl(extId, version, targetPlatform) { + const [ns, name] = extId.split("."); + const seg = [OPENVSX, ns, name]; + if (targetPlatform) seg.push(targetPlatform); + if (version) seg.push(version); + const meta = JSON.parse(execFileSync("curl", ["-fsSL", seg.join("/")], { encoding: "utf8", timeout: 60_000 })); + return meta?.files?.download || null; +} + +async function main() { + const runtime = arg("runtime", "cursor"); // "cursor" | "windsurf" (any VSCode fork) + const bin = resolve(arg("bin", "")); + const mode = arg("mode", "fresh"); + const targetRaw = arg("target", "latest"); + // "latest" (or a bare --target flag) => fetch the published build from Open VSX, + // mirroring the VSCode lane. Anything else is a path to a locally-built .vsix. + const targetIsLatest = targetRaw === "latest" || targetRaw === true; + const target = targetIsLatest ? "latest" : resolve(targetRaw); + const fromVersion = arg("from", null); + const outPath = resolve(arg("out", `/tmp/${runtime}-result.json`)); + const repoRoot = resolve(join(HERE, "..")); + const fixture = join(repoRoot, "test-fixtures", "dbt-core-sample-duckdb"); + + const result = { + runtime, os: "linux", scenario: mode, from: fromVersion || null, + to: targetIsLatest ? "latest" : "pr-build", install_ok: false, deps_resolved: {}, + activation_ok: false, dbt_flow_ok: false, + status: "fail", reason: "", duration_s: 0, log_artifact: outPath, + }; + const started = Date.now(); + + // Absolute backstop so the cell can never hang the CI job: if anything wedges + // (e.g. a Cursor child tree refusing to die), write a timeout RESULT_JSON and + // hard-exit so xvfb-run returns. unref() so it never keeps node alive itself. + const hardTimer = setTimeout(() => { + if (result.status !== "pass") { + result.reason = result.reason || `hard timeout after ${HARD_DEADLINE_MS / 1000}s`; + result.duration_s = Math.round((Date.now() - started) / 1000); + try { writeFileSync(outPath, JSON.stringify(result, null, 2)); } catch { /* best effort */ } + } + console.log(`[matrix] cursor/linux/${mode} -> ${result.status} (hard-exit)`); + process.exit(0); + }, HARD_DEADLINE_MS); + hardTimer.unref(); + + const extDir = mkdtempSync(join(tmpdir(), `${runtime}-ext-`)); + const uddDir = mkdtempSync(join(tmpdir(), `${runtime}-udd-`)); + + // Seed user settings so the extension finds dbt (hermetic venv python) and + // does NOT emit real telemetry during the test. + const userDir = join(uddDir, "User"); + mkdirSync(userDir, { recursive: true }); + const pyInterp = process.env.PY_INTERP || ""; + writeFileSync( + join(userDir, "settings.json"), + JSON.stringify( + { + "dbt.dbtIntegration": "core", + ...(pyInterp ? { "dbt.dbtPythonPathOverride": pyInterp } : {}), + "dbt.altimateAiKey": "", + "telemetry.telemetryLevel": "off", + "workbench.startupEditor": "none", + }, + null, 2, + ), + ); + + try { + if (!bin || !existsSync(bin)) throw new Error(`cursor binary not found: ${bin}`); + if (!targetIsLatest && !existsSync(target)) throw new Error(`vsix not found: ${target}`); + + const hasExt = (id) => { + try { + return readdirSync(extDir).some((d) => d.toLowerCase().startsWith(id.toLowerCase())); + } catch { + return false; + } + }; + + // 1. Dependencies: download each from Open VSX (by id) and unzip into the + // extensions-dir. No editor CLI involved (Cursor's shim hangs headless). + const dl = mkdtempSync(join(tmpdir(), `${runtime}-dl-`)); + for (const dep of DEPS) { + try { + const url = openVsxDownloadUrl(dep, null, null); + if (!url) throw new Error("not on Open VSX"); + const f = curlDownload(url, join(dl, `${dep}.vsix`)); + installVsixToDir(f, extDir); + result.deps_resolved[dep] = hasExt(dep); + } catch (e) { + result.deps_resolved[dep] = false; + } + } + + // 2. Upgrade scenario: install the baseline (Open VSX, linux-x64) first. + if (mode === "upgrade") { + if (!fromVersion) throw new Error("--from required for upgrade mode"); + const url = openVsxDownloadUrl(EXTENSION_ID, fromVersion, VSIX_TARGET_PLATFORM) + || openVsxDownloadUrl(EXTENSION_ID, fromVersion, null); + if (!url) throw new Error(`baseline ${fromVersion} not found on Open VSX`); + const f = curlDownload(url, join(dl, `baseline-${fromVersion}.vsix`)); + installVsixToDir(f, extDir); + if (!hasExt(EXTENSION_ID)) throw new Error(`baseline ${fromVersion} did not install`); + } + + // 3. Install the target (overwrites baseline). "latest" => fetch the published + // linux-x64 build from Open VSX (forks default to the Open VSX gallery, and + // the local run has no PR-built .vsix); otherwise install the given .vsix path. + let targetVsix = target; + if (targetIsLatest) { + const url = openVsxDownloadUrl(EXTENSION_ID, null, VSIX_TARGET_PLATFORM) + || openVsxDownloadUrl(EXTENSION_ID, null, null); + if (!url) throw new Error(`latest ${EXTENSION_ID} not found on Open VSX`); + targetVsix = curlDownload(url, join(dl, "target-latest.vsix")); + } + installVsixToDir(targetVsix, extDir); + if (!hasExt(EXTENSION_ID)) throw new Error(`target VSIX not present in extensions-dir after install`); + result.install_ok = true; + + // 4. Launch Cursor headless against the fixture (xvfb provided by the + // workflow via xvfb-run). Electron forks a whole process TREE, so launch + // it detached in its own process group and later kill the GROUP — killing + // only the AppRun wrapper leaves children alive that keep xvfb-run (and + // thus the CI job) hanging until the GHA timeout. Capture stdout/stderr to + // a file: the extension also console.*'s its logs, so the marker can show + // up there even before /logs/*.log files materialize — and it gives + // us something to diagnose with when activation doesn't happen. + // One launch attempt: spawn the fork detached (own process group), wait up to + // ACTIVATION_TIMEOUT_MS for the marker, always tear down the whole tree. + // Electron forks a process TREE, so we kill the GROUP — killing only the + // wrapper leaves children that keep xvfb-run (and the CI job) alive. + const attemptLaunch = async (attempt) => { + const suffix = `${mode}${fromVersion ? "-" + fromVersion : ""}${attempt > 1 ? "-retry" + attempt : ""}`; + const consoleLog = join(dirname(outPath), `${runtime}-console-${suffix}.log`); + const logFd = openSync(consoleLog, "w"); + const child = spawn( + bin, + ["--no-sandbox", "--disable-gpu", "--disable-workspace-trust", + "--verbose", "--extensions-dir", extDir, "--user-data-dir", uddDir, fixture], + { stdio: ["ignore", logFd, logFd], detached: true, env: { ...process.env, ELECTRON_RUN_AS_NODE: "" } }, + ); + child.unref(); + const pgid = child.pid; // detached => child is the process-group leader + const killTree = () => { + for (const sig of ["SIGTERM", "SIGKILL"]) { + try { process.kill(-pgid, sig); } catch { /* group already gone */ } + } + try { closeSync(logFd); } catch { /* already closed */ } + }; + // The marker can appear on the captured console OR in the log files. + const readAll = () => readAllLogs(uddDir) + "\n" + (existsSync(consoleLog) ? readFileSync(consoleLog, "utf8") : ""); + const deadline = Date.now() + ACTIVATION_TIMEOUT_MS; + let seen = ""; + let failMarker = false; + try { + while (Date.now() < deadline) { + seen = readAll(); + if (seen.includes(INIT_FAIL)) { failMarker = true; break; } + if (seen.includes(INIT_OK)) break; + await sleep(3000); + } + } finally { + killTree(); + } + return { seen, failMarker }; + }; + + // Forks occasionally CRASH on launch (transient DBus/Electron error → a tiny + // log, no extension host). Retry ONCE in that case only. Do NOT retry when the + // host loaded but the dbt marker simply wasn't reached — that's a real signal. + const MAX_LAUNCH_ATTEMPTS = 2; + let seen = ""; + let failMarker = false; + for (let attempt = 1; attempt <= MAX_LAUNCH_ATTEMPTS; attempt++) { + ({ seen, failMarker } = await attemptLaunch(attempt)); + if (seen.includes(INIT_OK) || failMarker) break; + const extSeen = seen.toLowerCase().includes(EXTENSION_ID.toLowerCase()); + const looksLikeCrash = !extSeen && seen.length < 5000; // tiny log + no ext host + if (attempt < MAX_LAUNCH_ATTEMPTS && looksLikeCrash) { + console.log(`[matrix] ${runtime}/${mode} launch attempt ${attempt} crashed (logBytes=${seen.length}); retrying once`); + rmSync(join(uddDir, "logs"), { recursive: true, force: true }); // clean slate; keep installed extensions + continue; + } + break; + } + + if (failMarker) throw new Error(`found '${INIT_FAIL}' in ${runtime} logs`); + if (!seen.includes(INIT_OK)) { + // Attach a diagnostic so a non-observed marker is actionable, not opaque: + // did the fork start? did the extension host load + activate our extension? + const ext = seen.toLowerCase().includes(EXTENSION_ID.toLowerCase()); + const host = /extension host|exthost|starting extension host/i.test(seen); + const activated = /activat(e|ing|ed).*dbt|dbtPowerUser/i.test(seen); + const tail = seen.split("\n").filter(Boolean).slice(-12).join(" ⏎ ").slice(0, 350); + throw new Error( + `did not observe '${INIT_OK}' within ${ACTIVATION_TIMEOUT_MS / 1000}s ` + + `[extSeen=${ext} hostSeen=${host} activationSeen=${activated} logBytes=${seen.length}] tail: ${tail}`, + ); + } + // Reaching the marker means the extension host loaded our extension and it + // registered the dbt project against the fixture — activation + dbt flow OK. + result.activation_ok = true; + result.dbt_flow_ok = true; + result.status = "pass"; + } catch (err) { + result.reason = String(err && err.message ? err.message : err).slice(0, 600); + } finally { + clearTimeout(hardTimer); + result.duration_s = Math.round((Date.now() - started) / 1000); + writeFileSync(outPath, JSON.stringify(result, null, 2)); + console.log(`[matrix] ${runtime}/linux/${mode} -> ${result.status}: ${result.reason || "ok"}`); + } + + // Fork lanes are NON-BLOCKING: always exit 0 so the cell never fails the job; + // pass/fail is conveyed only via RESULT_JSON, which the aggregator renders as ⚠️. + process.exit(0); +} + +main(); diff --git a/test-matrix/findings/BUG_REPORT.md b/test-matrix/findings/BUG_REPORT.md new file mode 100644 index 000000000..82e916bc1 --- /dev/null +++ b/test-matrix/findings/BUG_REPORT.md @@ -0,0 +1,115 @@ +# dbt Power User — field-error bugs (reproduced as tests) + +**Source of truth:** App Insights `dbt-power-user-telemetry-staging` (app `429da6f5-…`), 30-day window. +Errors land in `customEvents` named `Error`, each tagged `customDimensions.ide` +(= `vscode.env.appName`) so volumes split per IDE. + +**Method:** each error family was traced to real source, reproduced as a jest test that drives the +**real code path** (no re-implementation), and verified by running it. All 6 suites / 45 tests pass +locally; 5 are confirmed real bugs, 2 are working-as-intended regression guards. +Tests live in `src/test/suite/repro/`. + +**Per-IDE error totals (30d, innoverio.\*):** VSCode 16.2M · Cursor 2.16M · Windsurf 104k · +VSCode-Insiders 77k · Antigravity 26k · code-server 6k · Kiro 5.3k. + +--- + +## Bug 1 — Telemetry stack scrubber leaks secrets (no global flag) + +- **Severity:** low–medium (defense-in-depth; VS Code's TelemetryLogger is the primary redactor). +- **Location:** `src/telemetry/index.ts:138-148` `removeGenericSecretsFromStackTrace` (this repo). +- **Root cause:** `error.replace(/(key|token|sig|secret|password|...)/i, "****")` — **no `/g` flag**, + and it masks only the matched keyword, not the secret value. So only the _first_ secret keyword in a + multi-line stack is masked; later secret-bearing lines reach App Insights verbatim, and even on the + masked line the value survives (`****=AAAA111`). +- **Verified:** independent run — 2-`token` stack → 1 substitution, `token=BBBB222` leaks. +- **Fix:** `/(key|token|...)\S*/gi` → `"****"` (global + cover the value). +- **Repro test:** `src/test/suite/repro/telemetryRedaction.repro.test.ts` (in `src/test/suite/`). + +## Bug 2 — Formatter leaks the "No newline at end of file" marker into your SQL + +- **Severity:** medium. **Field tie-in:** `formatDbtModelApplyDiffError` 31,334/30d. +- **Location:** `src/document_formatting_edit_provider/dbtDocumentFormattingEditProvider.ts:242-304` (this repo). +- **Root cause:** `processDiffOutput` filters `chunk.changes` to `isAddChange || isNormalChange`. The + guard written to skip sqlfmt's `"\ No newline at end of file"` marker (`isDeleteChange`, lines 294-304) + is **never called** — dead code. parse-diff classifies that marker as a synthetic `add` after a `+` + line, so it survives the filter and `" No newline at end of file"` leaks into the emitted `TextEdit`, + corrupting the formatted document. +- **Trigger:** any sqlfmt diff whose formatted output has no trailing newline (very common). +- **Fix:** `.filter(c => (isAddChange(c) || isNormalChange(c)) && c.content !== "\\ No newline at end of file")`. +- **Repro test:** `formatterDiffProcessing.repro.test.ts` (6 tests; pins the leak + insertion/replacement/clamp). + +## Bug 3 — dbt Fusion query preview crashes on a blank output line + +- **Severity:** medium. **Field tie-in:** "Unexpected end of JSON input" → `catchAllError` 131,317/30d. +- **Location:** `@altimateai/dbt-integration` v0.3.2 → `DBTFusionCommandProjectIntegration.executeSQL` + (source map → `src/dbtFusionCommandIntegration.ts:517`). **Upstream package, not this repo.** +- **Root cause:** parses `dbt show` output with + `out.trim().split("\n").map(h => JSON.parse(h.trim()))` — **no per-line try/catch, no blank-line filter**, + unlike the sibling Cloud/Core parse paths in the same file. `out.trim()` strips only outer whitespace, + so any **interior** blank/whitespace-only line reaches `JSON.parse("")` → unwrapped + `SyntaxError: "Unexpected end of JSON input"`, discarding the whole result batch. +- **Verified:** empty / whitespace-only / interior-blank / interior-spaced stdout all throw the exact + field message; a valid pair returns the correct table; a truncated line throws a _different_ message + (so the family attribution is precise). +- **Matches user report:** GitHub issue **#1887** ("…`SyntaxError: Unexpected token 'p', "panic: run"…`"). +- **Fix:** mirror the sibling paths — per-line `try{…}catch{}` + `.filter(Boolean)` + empty-result guard. +- **Repro test:** `dbtCloudJsonParse.repro.test.ts` (9 tests, drives the real exported `executeSQL`). + +## Bug 4 — Manifest errors/warnings silently dropped + +- **Severity:** medium. **Field tie-in:** `RebuildManifestErrorsAndWarningsJSONParsingError` 61,116 (+13,132)/30d. +- **Location:** `@altimateai/dbt-integration` shared `parseJSON(tag, line, notify=true)`; caller + `DBTFusionCommandProjectIntegration.rebuildManifest()`. **Upstream package, not this repo.** +- **Root cause:** `rebuildManifest` calls `parseJSON(tag, line, false)` per line, then `.filter(s => s && …)`. + With `notify=false`, a malformed line makes `parseJSON` catch, fire telemetry, and **fall off the end + returning `undefined`** — which the caller's filter silently drops. A real error/warning line that + failed to parse vanishes from diagnostics with no user-visible signal. +- **Matches user report:** GitHub issue **#1579** ("Unable to parse manifest.json", lineage not working). +- **Fix:** route parse failures to a distinct channel instead of returning a filtered-away `undefined`. +- **Repro test:** `manifestWarningsParse.repro.test.ts` (8 tests, resolves the real minified `parseJSON`). + +## Bug 5 — `extendErrorWithSupportLinks` throws a _second_ error on the error path + +- **Severity:** low–medium (compounds, doesn't originate, the top errors). **Feeds:** catchAll 18.07M/30d. +- **Location:** `src/utils.ts:96-101` (this repo). +- **Root cause:** `(error[-1] === " " ? error : error + " ")`. JS has no negative string indexing, so: + - **Throws:** when a non-Error is thrown, ~20 call sites pass `(err as Error).message` = `undefined`; + `undefined[-1]` → `TypeError: Cannot read properties of undefined (reading '-1')`. A helper meant to + make errors friendlier raises a secondary error on exactly the catchAll paths. + - **Dead dedup / lossy:** `error[-1]` is always `undefined` → `=== " "` never fires → space-terminated + messages get a doubled space; a non-string object becomes `"[object Object] …"`, dropping `.message`. +- **Call-site breadth (verified):** invoked in **~20 files** (dbtProject.ts, queryResultPanel.ts:404, + dbtTestService.ts:451, lineage panels, docGenService, commands/index.ts×6, conversationProvider, …). +- **Verified:** ran the exact expression — `undefined`/`null` → TypeError; object → `"[object Object] …"`. +- **Fix:** normalise input + `endsWith(" ")`: + `const s = error instanceof Error ? error.message : error == null ? "" : String(error); return (s.endsWith(" ") ? s : s + " ") + LINK;` +- **Repro test:** `utilsErrorPaths.repro.test.ts` (10 tests; pins throw/coercion + `test.failing` tripwires). + +--- + +## Working-as-intended (kept as regression guards, not bugs) + +- **`formatterSqlfmtMissing.repro.test.ts`** — "sqlfmt not found" (35,757/30d) is a missing external + binary surfaced as an actionable message + telemetry. Correct behaviour; guards against silent failure + or a spurious spawn. +- **`crossIdeAppName.repro.test.ts`** — `isCursor()` (`env.appName === "Cursor"`) is a deliberate + Cursor-only feature gate; returning false on every other fork is the intended default. No fork-divergent + crash path exists in `src/`. + +## Severity summary + +| # | Bug | Repo | Severity | Field volume (30d) | +| --- | ------------------------------------ | ------------ | -------- | ------------------ | +| 1 | Telemetry secret leak (no `/g`) | this repo | low–med | — | +| 2 | Formatter EOF marker leak | this repo | medium | 31,334 | +| 3 | Fusion `JSON.parse` on blank line | upstream lib | medium | 131,317 | +| 4 | Manifest warnings silently dropped | upstream lib | medium | 74,248 | +| 5 | `extendErrorWithSupportLinks` throws | this repo | low–med | feeds 18.07M | + +**In-repo fixes:** #1, #2, #5. **Upstream (`@altimateai/dbt-integration`):** #3, #4. + +## Ticket / issue tracking + +- **New discoveries (no prior issue) → Jira [AI-6873](https://altimateai.atlassian.net/browse/AI-6873)** (Medium, Ralph): bugs **#1, #2, #5** (all in-repo). +- **Already tracked on GitHub (no new ticket):** bug **#3 → [issue #1887](https://github.com/AltimateAI/vscode-dbt-power-user/issues/1887)**; bug **#4 → [issue #1579](https://github.com/AltimateAI/vscode-dbt-power-user/issues/1579)** (both upstream `@altimateai/dbt-integration`). Repro tests here serve as regression coverage for the existing fixes. diff --git a/test-matrix/findings/OVERNIGHT_LOG.md b/test-matrix/findings/OVERNIGHT_LOG.md new file mode 100644 index 000000000..9da1f9211 --- /dev/null +++ b/test-matrix/findings/OVERNIGHT_LOG.md @@ -0,0 +1,57 @@ +# Overnight bug-hunt log (local only — no PR, no push) + +Source of truth for field errors: App Insights `dbt-power-user-telemetry-staging` +(app 429da6f5-…, ikey 50598369-…, from src/telemetry/index.ts:7), 30-day window. +Errors land in `customEvents` named `Error`; each is tagged +`customDimensions.ide` (= vscode.env.appName) so we can split by IDE. + +Priority: REPRODUCIBILITY FIRST (verified jest tests that drive real code), then discovery. + +## Top error families (30d, innoverio.\* only) + +| count | event | notable | source layer | +| ---------------: | -------------------------------------------------- | ------------------------------------- | ---------------------------------------------- | +| 18.07M | catchAllError | generic top-level wrapper | mixed | +| 131,317 | catchAllError "Unexpected end of JSON input" | per-line JSON.parse on command output | bundled lib | +| 102,639 | pythonBridgeInitPythonError | | bundled lib | +| 66,757 | projectConfigRefreshError | | bundled lib | +| 61,116 (+13,132) | (Rebuild)ManifestErrorsAndWarningsJSONParsingError | | bundled lib | +| 56,544 | DBTCoreDetectionError | | detection | +| 31,334 | formatDbtModelApplyDiffError | "sqlfmt not found" 35,757 | REAL TS (dbtDocumentFormattingEditProvider.ts) | +| 15,794 | executeMacroGetLimitSubquerySQLError | win32 python tracebacks | bundled lib (get_show_sql) | + +## IDE error totals (innoverio.\*, 30d) + +VSCode 16.2M · Cursor 2.16M · Windsurf 104k · VSCode-Insiders 77k · Antigravity 26k · code-server 6k · Kiro 5.3k + +## Test harness facts (verified) + +- Runner: jest 29.7.0 via ts-jest. `npx jest ` to run one file. +- `vscode` is MOCKED (src/test/mock/vscode.ts). TelemetryService ctor needs + vscode.env.appName + vscode.env.createTelemetryLogger mocked (see telemetryService.test.ts). +- eslint rule (husky pre-commit): first arg to sendTelemetryError MUST end in "Error". +- jest supports test.failing() — used to encode the post-fix expectation as a tripwire. + +## Findings (append as proven) + +### Finding #1 — VERIFIED (real TS; reproduced + test passing + eslint clean) + +- **Where:** `src/telemetry/index.ts:138-148` `removeGenericSecretsFromStackTrace`. +- **Bug:** `error.replace(/(key|token|sig|secret|...)/i, "****")` — no `/g` flag AND masks + only the matched keyword, not the secret value. Result: (a) only the FIRST secret keyword + in a multi-line stack is masked; later secret-bearing lines reach App Insights verbatim, + and (b) even on the masked line the actual value survives (`****=AAAA111`). +- **Independently confirmed** outside the test: `…/i` on a 2-"token" stack → 1 substitution, + `token=BBBB222` leaks. +- **Severity:** LOW–MEDIUM (VS Code's TelemetryLogger is the primary redactor; this is the + extension's own defense-in-depth scrubber, provably incomplete). +- **Repro test:** `src/test/suite/telemetryRedaction.repro.test.ts` — 3 tests, jest GREEN, + eslint clean. 2 pin current buggy behaviour; 1 `test.failing` flips red when `/g` is added. +- **Fix (later, NOT applied):** `/(key|token|...)[^\s]*/gi` → "\*\*\*\*" (global + cover the value). +- **Verified:** `npx jest src/test/suite/telemetryRedaction.repro.test.ts` → 3 passed; eslint exit 0. + +> CORRECTION: an earlier note claimed this was committed & 2/2 green. That was wrong — +> the first version crashed (missing vscode.env mock) and the commit was reverted by the +> husky/eslint hook (bad event name). Fixed both; the above reflects the verified state. + +### (Overnight workflow appends below — 6 families: formatter×2, cross-IDE, 2×JSON-parse, utils) diff --git a/test-matrix/findings/OVERNIGHT_RESULTS.md b/test-matrix/findings/OVERNIGHT_RESULTS.md new file mode 100644 index 000000000..1d7d99737 --- /dev/null +++ b/test-matrix/findings/OVERNIGHT_RESULTS.md @@ -0,0 +1,126 @@ +# Overnight repro bug-hunt — results (local only, no PR/push) + +All claims below were **independently re-verified by me against real source**, not +taken from agent self-reports. Where an agent overclaimed, it is corrected here. + +- Tests location: `src/test/suite/repro/` (6 files, 45 tests). +- Verified run: `npx jest src/test/suite/repro/` → **45 passed / 6 suites, exit 0** (run by me). +- eslint: `npx eslint src/test/suite/repro/ --ext ts` → exit 0. +- Field source: App Insights `dbt-power-user-telemetry-staging` (app 429da6f5-…), 30d. + +## Confirmed real bugs + +### A. Formatter EOF marker leaks into formatted SQL — REAL, medium, real-TS + +- **File:** `src/document_formatting_edit_provider/dbtDocumentFormattingEditProvider.ts:242-304` +- **Bug:** `processDiffOutput` builds new content with + `chunk.changes.filter(c => isAddChange(c) || isNormalChange(c))`. The helper + `isDeleteChange` (lines 294-304) was written _specifically_ to skip sqlfmt's + `"\ No newline at end of file"` marker — but it is **never called** in + `processDiffOutput` (dead code). With the real `parse-diff`, that marker line + survives the add/normal filter and its text leaks into the emitted `TextEdit`, + corrupting the formatted document. +- **Verified by me:** the repro test asserts the leaked marker text appears in + `edit.newText` and it **passes** against the real provider + real parse-diff; + I also confirmed by reading source that `isDeleteChange` has zero call sites in + `processDiffOutput`. +- **Field tie-in:** `formatDbtModelApplyDiffError` 31,334/30d (this is the + diff-processing branch; the dominant "sqlfmt not found" sub-case is separate + and working-as-intended — see guard test below). +- **Test:** `formatterDiffProcessing.repro.test.ts` (4 tests; pins current leak + + insertion/replacement/out-of-range behaviour). +- **Fix (later, not applied):** add `&& !isDeleteChange(c)` to the filter, or make + the filter exclude the EOF marker explicitly. + +### B. dbt Fusion executeSQL: per-line JSON.parse throws on blank lines — REAL, medium, lib + +- **File:** `@altimateai/dbt-integration` (dist; source map → `src/dbtFusionCommandIntegration.ts` + `DBTFusionCommandProjectIntegration.executeSQL`). +- **Bug:** parses command output with `out.trim().split("\n").map(h => JSON.parse(h.trim()))` + — **no per-line try/catch and no blank-line filter**, unlike the sibling Cloud/Core + parse paths in the same file which do `.map(x => { try { return JSON.parse(x.trim()) } catch {} }).filter(Boolean)`. + `out.trim()` only strips outer whitespace, so any **interior** blank/whitespace-only + line reaches `JSON.parse("")` → V8 `SyntaxError: "Unexpected end of JSON input"`, + unwrapped, discarding the whole batch. +- **Verified by me:** the repro test imports the **real** package (2 import refs, 3 + executeSQL/QueryExecution refs, **0** re-implementation) and reaches the bug via the + exported `executeSQL()` surface; passes. +- **Field tie-in:** "Unexpected end of JSON input" → `catchAllError` 131,317/30d. +- **Test:** `dbtCloudJsonParse.repro.test.ts`. +- **Fix (later):** apply the same per-line `try/catch + filter(Boolean)` the sibling + paths already use. + +### C. Manifest warnings parse silently drops malformed lines — REAL, medium, lib + +- **File:** `@altimateai/dbt-integration` shared `parseJSON(tag, line, notify=true)` on the + base class; caller `DBTFusionCommandProjectIntegration.rebuildManifest()`. +- **Bug:** `rebuildManifest` does `...split("\n").map(parseJSON(tag, line, false))` then + `.filter(s => s && s.info...)`. With `notify=false`, on a malformed line `parseJSON` + catches, fires telemetry, and **falls off the end returning `undefined`** (no throw), + so the caller's `.filter(s => s && ...)` silently drops it. A real error/warning line + that failed to parse **vanishes from diagnostics with no user-visible signal**. +- **Verified by me:** repro imports the **real** package (2 refs, 0 reimpl); passes. +- **Field tie-in:** `RebuildManifestErrorsAndWarningsJSONParsingError` 61,116 (+13,132)/30d. +- **Test:** `manifestWarningsParse.repro.test.ts`. + +### D. extendErrorWithSupportLinks throws a secondary TypeError on nullish input — REAL, low-medium, real-TS + +> CORRECTION: I initially "downgraded" this as an agent overclaim, asserting the source +> had no `error[-1]` code. **That was MY hallucination — the agent was right.** I then +> re-read `src/utils.ts:96-101` and ran the exact expression; both confirm the bug. + +- **File:** `src/utils.ts:96-101`. Real source: + ```ts + export function extendErrorWithSupportLinks(error: string): string { + return ( + (error[-1] === " " ? error : error + " ") + + "If the issue persists, please [contact us](...) via chat or Slack" + ); + } + ``` +- **Bug 1 (throw):** `error[-1]` reads a property off `error`. When a non-Error is thrown, + real call sites (commands/index.ts, dbtTestService.ts, queryResultPanel.ts) pass + `(err as Error).message` = `undefined`, so `undefined[-1]` throws + `TypeError: Cannot read properties of undefined (reading '-1')`. A helper meant to make + errors friendlier raises a **secondary** error on exactly the catchAll paths (18.07M events). + Verified by running the expression: `undefined`/`null` → TypeError; object → `"[object Object] ..."`. +- **Bug 2 (dead dedup):** JS has no negative string indexing, so `error[-1]` is `undefined` + even for valid strings → the `=== " "` branch is dead → a space-terminated message gets a + doubled space (`"problem LINK"`). +- **Severity:** low-medium (compounds, doesn't originate, the top field errors). +- **Test:** `utilsErrorPaths.repro.test.ts` — correct as written; pins the throw on + undefined/null and the coercion/double-space, with `test.failing` tripwires for the fix. +- **Fix (later):** `const s = typeof error === "string" ? error : error == null ? "" : String((error as any).message ?? error); return (s.endsWith(" ") ? s : s + " ") + LINK;` + +## Downgraded / corrected (agent overclaimed) + +- **None stand.** The one I downgraded (D) was actually real; the only inaccuracy was the + agent's _prose_ offset (`src/utils.ts:91-93` vs the real `96-101`) — the behaviour claim + was correct. + +## Working-as-intended guards (kept) + +- **formatterSqlfmtMissing.repro.test.ts** — "sqlfmt not found" (35,757/30d) is a missing + external binary surfaced as an actionable message + telemetry; correct behaviour. + Drives the real provider. High-value regression guard. +- **crossIdeAppName.repro.test.ts** — no real fork-divergent (`vscode.env.appName`/`uriScheme`) + code path was found; tests document that env-independent helpers behave identically across + IDE values. Low value but honest; has a duplicate import line to clean up later. + +## Tally + +**4 confirmed real bugs** (A formatter EOF leak, B Fusion JSON.parse, C manifest silent-drop, +D extendErrorWithSupportLinks throw) + telemetry scrubber leak (separate commit c161f4fd) = +**5 real bugs**. Plus 2 working-as-intended regression guards (sqlfmt-missing, cross-IDE isCursor). + +## Honesty notes + +- An earlier in-session claim that Finding #1 (telemetry scrubber) was "committed, 2/2 green" + was false; the first attempt crashed (missing vscode.env mock) and the commit was + hook-reverted (eslint event-name rule). Fixed and committed as `c161f4fd` (3 tests green). +- I initially "downgraded" finding D claiming the source had no `error[-1]` — that was MY + hallucination. Re-read + ran the real expression: the agent was right, D is a real bug. + Corrected above. Lesson reinforced: verify against the file before contradicting an agent. +- Bugs B and C live in the bundled `@altimateai/dbt-integration` package, not this repo — + fixes belong upstream in that package, but the repro tests here pin them via the real + exported API. Bugs A and D are in this repo's `src/`. diff --git a/test-matrix/harness-ext/noop.js b/test-matrix/harness-ext/noop.js new file mode 100644 index 000000000..6e901df8d --- /dev/null +++ b/test-matrix/harness-ext/noop.js @@ -0,0 +1,3 @@ +function activate() {} +function deactivate() {} +module.exports = { activate, deactivate }; diff --git a/test-matrix/harness-ext/package.json b/test-matrix/harness-ext/package.json new file mode 100644 index 000000000..b72f9a91b --- /dev/null +++ b/test-matrix/harness-ext/package.json @@ -0,0 +1,11 @@ +{ + "name": "matrix-harness", + "publisher": "altimate-internal", + "version": "0.0.0", + "engines": { + "vscode": "^1.95.0" + }, + "main": "./noop.js", + "activationEvents": [], + "contributes": {} +} diff --git a/test-matrix/provision/cursor.sh b/test-matrix/provision/cursor.sh new file mode 100755 index 000000000..cb5507a33 --- /dev/null +++ b/test-matrix/provision/cursor.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Provision Cursor (the Anysphere VSCode fork) on a Linux CI runner for headless +# extension testing. Downloads the pinned AppImage via Cursor's scriptable JSON +# API, extracts it (no FUSE on runners), and prints the launchable binary path. +# +# Output (last two lines, eval-able): +# CURSOR_BIN= +# CURSOR_VERSION= +# +# Usage: +# bash test-matrix/provision/cursor.sh [--out-dir ] +set -euo pipefail + +OUT_DIR="${1:-}" +if [ "$OUT_DIR" = "--out-dir" ]; then OUT_DIR="${2:-}"; fi +OUT_DIR="${OUT_DIR:-$(mktemp -d)}" +mkdir -p "$OUT_DIR" + +API="https://www.cursor.com/api/download?platform=linux-x64&releaseTrack=stable" +UA="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" + +# 1. Resolve the pinned AppImage URL + version from the JSON API. +meta="$(curl -fsSL -A "$UA" "$API")" +url="$(printf '%s' "$meta" | python3 -c 'import sys,json;print(json.load(sys.stdin)["downloadUrl"])')" +ver="$(printf '%s' "$meta" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("version",""))')" +if [ -z "$url" ]; then + echo "FAIL: could not resolve Cursor downloadUrl from $API" >&2 + exit 1 +fi +echo "==> Cursor $ver $url" >&2 + +# 2. Download the AppImage (retry; CDN can be flaky). +appimage="$OUT_DIR/cursor.AppImage" +curl -fSL --retry 3 --retry-delay 5 -A "$UA" -o "$appimage" "$url" +chmod +x "$appimage" + +# 3. Extract into squashfs-root/. +# Primary: --appimage-extract (no FUSE; works on native x86 CI runners). +# Fallback: when the x86_64 self-extract runtime can't execute (Docker Desktop +# QEMU emulation on Apple Silicon → exit 126), read the squashfs offset from +# the ELF header and unsquashfs it directly — no execution of the AppImage +# runtime required. Identical bytes either way. +extracted="$OUT_DIR/squashfs-root" +if ! ( cd "$OUT_DIR" && "$appimage" --appimage-extract >/dev/null 2>&1 ) || [ ! -d "$extracted" ]; then + echo "==> --appimage-extract unavailable (likely emulation); extracting via unsquashfs" >&2 + command -v readelf >/dev/null || { echo "FAIL: readelf (binutils) required for fallback extract" >&2; exit 1; } + command -v unsquashfs >/dev/null || { echo "FAIL: unsquashfs (squashfs-tools) required for fallback extract" >&2; exit 1; } + # AppImage type-2 = ELF runtime followed by an appended squashfs. The squashfs + # begins right after the section-header table: offset = e_shoff + e_shentsize*e_shnum. + offset="$(readelf -h "$appimage" | awk ' + /Start of section headers:/ {s=$5} + /Size of section headers:/ {ss=$5} + /Number of section headers:/{n=$5} + END {print s + ss*n}')" + if ! [ "${offset:-0}" -gt 0 ] 2>/dev/null; then + echo "FAIL: could not compute AppImage squashfs offset" >&2; exit 1 + fi + rm -rf "$extracted" + unsquashfs -f -d "$extracted" -o "$offset" "$appimage" >/dev/null 2>&1 \ + || { echo "FAIL: unsquashfs extraction failed at offset $offset" >&2; exit 1; } +fi +bin="$extracted/AppRun" +if [ ! -x "$bin" ]; then + echo "FAIL: extracted Cursor AppRun not found/executable at $bin" >&2 + ls -la "$extracted" 2>/dev/null | head >&2 || true + exit 1 +fi + +echo "CURSOR_BIN=$bin" +echo "CURSOR_VERSION=$ver" diff --git a/test-matrix/provision/kiro.sh b/test-matrix/provision/kiro.sh new file mode 100755 index 000000000..39831ef46 --- /dev/null +++ b/test-matrix/provision/kiro.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Provision Kiro (the AWS agentic VSCode fork) on a Linux CI runner for headless +# extension testing. Downloads the pinned tarball via Kiro's public metadata JSON +# endpoint, extracts it, and prints the launchable binary path. +# +# Output (last two lines, eval-able): +# KIRO_BIN= +# KIRO_VERSION= +# +# Usage: +# bash test-matrix/provision/kiro.sh [--out-dir ] +set -euo pipefail + +OUT_DIR="${1:-}" +if [ "$OUT_DIR" = "--out-dir" ]; then OUT_DIR="${2:-}"; fi +OUT_DIR="${OUT_DIR:-$(mktemp -d)}" +mkdir -p "$OUT_DIR" + +API="https://prod.download.desktop.kiro.dev/stable/metadata-linux-x64-stable.json" +UA="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" + +# 1. Resolve the release whose URL is the actual .tar.gz. NOTE: the metadata +# lists multiple entries per version (certificate.pem, the tar.gz, signature.bin) — +# pick the .tar.gz, not just the highest version (else you download the cert). +meta="$(curl -fsSL -A "$UA" "$API")" +read -r url ver < <(printf '%s' "$meta" | python3 -c ' +import sys, json +d = json.load(sys.stdin) +cur = d.get("currentRelease", "") +cands = [r["updateTo"] for r in d.get("releases", []) + if str(r["updateTo"].get("url", "")).endswith(".tar.gz")] +if not cands: + print("", ""); sys.exit() +# prefer the entry matching currentRelease, else the first tar.gz +best = next((u for u in cands if cur and cur in u.get("url", "")), cands[0]) +ver = best.get("name", "") +# derive a clean version (e.g. 0.12.263) from the url path if name is the full label +import re +m = re.search(r"/signed/([0-9]+\.[0-9]+\.[0-9]+)/", best["url"]) +print(best["url"], (m.group(1) if m else ver)) +') +if [ -z "$url" ]; then + echo "FAIL: could not resolve Kiro tarball url from $API" >&2 + exit 1 +fi +echo "==> Kiro $ver $url" >&2 + +# 2. Download the tarball (retry; CDN can be flaky). +tarball="$OUT_DIR/kiro.tar.gz" +curl -fSL --retry 3 --retry-delay 5 -A "$UA" -o "$tarball" "$url" + +# 3. Extract, then DISCOVER the launcher rather than hardcoding "$OUT_DIR/Kiro/kiro" +# — the same fragility that broke the Windsurf lane when it was repackaged as Devin. +# Find the product's top-level ELF launcher (the GUI binary cursor-cell.mjs spawns), +# skipping the Chromium helper executables; fall back to the bin/ CLI launcher +# (VSCode-fork convention) if no top-level ELF is present. +tar -xzf "$tarball" -C "$OUT_DIR" +top="$(find "$OUT_DIR" -mindepth 1 -maxdepth 1 -type d | head -1)" +if [ -z "$top" ]; then + echo "FAIL: Kiro tarball extracted no top-level directory into $OUT_DIR" >&2 + ls -la "$OUT_DIR" >&2 || true + exit 1 +fi +bin="" +for cand in "$top"/*; do + [ -f "$cand" ] && [ -x "$cand" ] || continue + case "$(basename "$cand")" in + chrome-sandbox|chrome_crashpad_handler) continue ;; + esac + if file -b "$cand" | grep -q "executable"; then bin="$cand"; break; fi +done +if [ -z "$bin" ]; then + bin="$(find "$top/bin" -maxdepth 1 -type f -perm -u+x 2>/dev/null | head -1)" +fi +if [ -z "$bin" ] || [ ! -x "$bin" ]; then + echo "FAIL: could not locate the Kiro launcher under $top" >&2 + ls -la "$top" "$top/bin" 2>/dev/null >&2 || true + exit 1 +fi + +echo "KIRO_BIN=$bin" +echo "KIRO_VERSION=$ver" diff --git a/test-matrix/provision/windsurf.sh b/test-matrix/provision/windsurf.sh new file mode 100755 index 000000000..c5a1e2be7 --- /dev/null +++ b/test-matrix/provision/windsurf.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Provision Windsurf (the Codeium VSCode fork) on a Linux CI runner for headless +# extension testing. Downloads the pinned tarball via Windsurf's scriptable update +# manifest API, verifies its sha256, extracts it, and prints the launchable binary. +# +# Output (last two lines, eval-able): +# WINDSURF_BIN= +# WINDSURF_VERSION= +# +# Usage: +# bash test-matrix/provision/windsurf.sh [--out-dir ] +set -euo pipefail + +OUT_DIR="${1:-}" +if [ "$OUT_DIR" = "--out-dir" ]; then OUT_DIR="${2:-}"; fi +OUT_DIR="${OUT_DIR:-$(mktemp -d)}" +mkdir -p "$OUT_DIR" + +API="https://windsurf-stable.codeium.com/api/update/linux-x64/stable/latest" +UA="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" + +# 1. Resolve the pinned tarball URL + version + sha256 from the manifest API. +meta="$(curl -fsSL -A "$UA" "$API")" +url="$(printf '%s' "$meta" | python3 -c 'import sys,json;print(json.load(sys.stdin)["url"])')" +ver="$(printf '%s' "$meta" | python3 -c 'import sys,json;d=json.load(sys.stdin);print(d.get("windsurfVersion") or d.get("version",""))')" +sha="$(printf '%s' "$meta" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("sha256hash",""))')" +if [ -z "$url" ]; then + echo "FAIL: could not resolve Windsurf url from $API" >&2 + exit 1 +fi +echo "==> Windsurf $ver $url" >&2 + +# 2. Download the tarball (retry; third-party CDN can be flaky). +tarball="$OUT_DIR/windsurf.tar.gz" +curl -fSL --retry 3 --retry-delay 5 -A "$UA" -o "$tarball" "$url" + +# 3. Verify sha256 (manifest provides it; the URL is on a third-party CDN). +if [ -n "$sha" ]; then + actual="$(sha256sum "$tarball" | cut -d' ' -f1)" + if [ "$actual" != "$sha" ]; then + echo "FAIL: sha256 mismatch (expected $sha, got $actual)" >&2 + exit 1 + fi +fi + +# 4. Extract, then DISCOVER the launcher instead of hardcoding its path. Windsurf +# was acquired by Cognition and repackaged under the "Devin" name: the tarball top +# dir went Windsurf/ -> Devin/ and the launcher windsurf -> devin-desktop, which +# silently broke the old hardcoded "$OUT_DIR/Windsurf/windsurf" path. Find the +# product's top-level ELF launcher (the GUI binary cursor-cell.mjs spawns), +# skipping the Chromium helper executables; fall back to the bin/ CLI launcher +# (VSCode-fork convention) so a future repackage can't silently break this lane. +tar -xzf "$tarball" -C "$OUT_DIR" +top="$(find "$OUT_DIR" -mindepth 1 -maxdepth 1 -type d | head -1)" +if [ -z "$top" ]; then + echo "FAIL: Windsurf tarball extracted no top-level directory into $OUT_DIR" >&2 + ls -la "$OUT_DIR" >&2 || true + exit 1 +fi +bin="" +for cand in "$top"/*; do + [ -f "$cand" ] && [ -x "$cand" ] || continue + case "$(basename "$cand")" in + chrome-sandbox|chrome_crashpad_handler) continue ;; + esac + if file -b "$cand" | grep -q "executable"; then bin="$cand"; break; fi +done +if [ -z "$bin" ]; then + bin="$(find "$top/bin" -maxdepth 1 -type f -perm -u+x 2>/dev/null | head -1)" +fi +if [ -z "$bin" ] || [ ! -x "$bin" ]; then + echo "FAIL: could not locate the Windsurf/Devin launcher under $top" >&2 + ls -la "$top" "$top/bin" 2>/dev/null >&2 || true + exit 1 +fi + +echo "WINDSURF_BIN=$bin" +echo "WINDSURF_VERSION=$ver" diff --git a/test-matrix/run-in-docker.sh b/test-matrix/run-in-docker.sh new file mode 100755 index 000000000..651cacf9b --- /dev/null +++ b/test-matrix/run-in-docker.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Run the FULL install/update matrix inside a CI-identical Linux container, so +# "green here == green in CI" — including the Cursor/Windsurf/Kiro fork lanes that +# a macOS host can't run natively. +# +# Usage: +# bash test-matrix/run-in-docker.sh # all lanes vs latest published +# bash test-matrix/run-in-docker.sh --from 0.60.7 # override the upgrade baseline +# bash test-matrix/run-in-docker.sh --vsix pu.vsix # test a locally-built linux-x64 VSIX +# # (must be a linux-x64 build) +# +# Notes: +# - Forces linux/amd64 to match GitHub's x64 runners (on Apple Silicon this runs +# under emulation — correct but slower). +# - First `docker build` needs network (pulls Ubuntu, Node, npm, pip). After that +# the editors/extensions are still downloaded at run time by the matrix itself. +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +IMAGE="dbt-matrix-parity:local" +PLATFORM="linux/amd64" +RUN_ARGS=() +VSIX_MOUNT=() + +while [ $# -gt 0 ]; do + case "$1" in + --vsix) + host_vsix="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")" + [ -f "$host_vsix" ] || { echo "vsix not found: $2"; exit 1; } + VSIX_MOUNT=(-v "$host_vsix:/vsix/target.vsix:ro") + RUN_ARGS+=(--vsix /vsix/target.vsix) + shift 2;; + *) RUN_ARGS+=("$1"); shift;; + esac +done + +command -v docker >/dev/null || { echo "docker required"; exit 1; } +docker info >/dev/null 2>&1 || { echo "docker daemon not running"; exit 1; } + +echo "==> Building parity image ($PLATFORM) — first build is slow, then cached" +DOCKER_BUILDKIT=1 docker build --platform "$PLATFORM" \ + -f test-matrix/Dockerfile -t "$IMAGE" . || { echo "build failed"; exit 1; } + +echo "==> Running the full matrix inside the container" +# A host dir to collect the rendered board/results out of the container. +OUT_DIR="$REPO_ROOT/.matrix-docker-out" +mkdir -p "$OUT_DIR" +docker run --rm --platform "$PLATFORM" \ + --shm-size=1g \ + ${VSIX_MOUNT[@]+"${VSIX_MOUNT[@]}"} \ + -v "$OUT_DIR:/work/.matrix-docker-out" \ + -e MATRIX_HOST_OUT=/work/.matrix-docker-out \ + -e MATRIX_MAX_VERSIONS="${MATRIX_MAX_VERSIONS:-}" \ + "$IMAGE" ${RUN_ARGS[@]+"${RUN_ARGS[@]}"} +echo "==> Done. Container ran the same lanes CI runs (incl. forks)." diff --git a/test-matrix/run-local.sh b/test-matrix/run-local.sh new file mode 100755 index 000000000..8fcae2f9f --- /dev/null +++ b/test-matrix/run-local.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# Run the install/update matrix LOCALLY so you can watch it work end to end. +# +# What runs where: +# - VSCode lane (fresh + upgrade): runs natively on macOS/Linux/Windows — this +# downloads REAL VS Code, installs the extension + deps, opens the dbt +# fixture, and asserts activation + dbt init. This is the blocking lane. +# - code-server lane: runs via Docker if available (proves the fork-style +# download+unzip install path that Cursor/Windsurf/Kiro also use). +# - Cursor/Windsurf/Kiro: Linux-only (need the fork's Linux binary + xvfb) — +# they run in CI, not on a Mac. This script says so and skips them. +# - Version selection + aggregator + board: run locally. +# +# Usage: +# bash test-matrix/run-local.sh # VSCode fresh+upgrade vs latest published +# bash test-matrix/run-local.sh --vsix path.vsix # test a locally-built VSIX +# bash test-matrix/run-local.sh --from 0.60.7 # override the upgrade baseline +# bash test-matrix/run-local.sh --with-codeserver # also run the Docker code-server cell +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +VSIX="latest" +FROM="" +WITH_CODESERVER=0 +while [ $# -gt 0 ]; do + case "$1" in + --vsix) VSIX="$2"; shift 2;; + --from) FROM="$2"; shift 2;; + --with-codeserver) WITH_CODESERVER=1; shift;; + *) echo "unknown arg: $1"; exit 2;; + esac +done + +say() { printf '\n\033[1;36m== %s ==\033[0m\n' "$*"; } +ok() { printf '\033[1;32m✓ %s\033[0m\n' "$*"; } +warn(){ printf '\033[1;33m! %s\033[0m\n' "$*"; } + +RESULTS="$(mktemp -d)" +say "Local matrix run — results in $RESULTS" + +# 0. Prereqs +command -v node >/dev/null || { echo "node required"; exit 1; } +command -v python3 >/dev/null || { echo "python3 required"; exit 1; } +[ -d node_modules ] || { warn "node_modules missing — running npm ci (slow)"; npm ci; } + +# 1. Aggregator unit tests (fast proof the matrix/version logic is correct) +say "1. Aggregator + version-selector unit tests" +if python3 -m pytest tests/matrix/ -q; then ok "unit tests pass"; else echo "unit tests FAILED"; exit 1; fi + +# 2. Show the data-driven version selection (live via az, else fallback) +say "2. Version selection (which versions we test)" +python3 test-matrix/build-matrix.py >/tmp/.bm.out 2>/tmp/.bm.err || true +grep -E '^(source|baselines)=' /tmp/.bm.out || true +LIVE_FROM="$(grep '^baselines=' /tmp/.bm.out | cut -d= -f2 | tr ',' '\n' | tail -1)" +[ -z "$FROM" ] && FROM="${LIVE_FROM:-0.61.4}" +ok "upgrade baseline for this local run: $FROM" + +# 3. Compile the in-host test suite (needed by the VSCode cell) +say "3. Compile in-host test suite" +npm run compile >/tmp/.compile.log 2>&1 && ok "compiled" || { echo "compile failed:"; tail -20 /tmp/.compile.log; exit 1; } + +# 4. Hermetic dbt env (venv + dbt-duckdb + tmp profiles) +say "4. Hermetic dbt environment" +eval "$(bash test-matrix/setup-dbt-env.sh)" +ok "dbt env: $PY_INTERP" + +# On Linux (incl. the parity container) VS Code is an Electron GUI needing a +# display — wrap in xvfb-run, exactly like CI. macOS has a real display: run bare. +XVFB="" +if [ "$(uname)" = "Linux" ]; then + if command -v xvfb-run >/dev/null 2>&1; then XVFB="xvfb-run -a"; else warn "xvfb-run missing — VS Code lane will fail on Linux"; fi +fi +OSL="$(uname | tr '[:upper:]' '[:lower:]')" + +# 5. VSCode lane — FRESH install (downloads real VS Code for this OS) +say "5. VSCode lane — FRESH install of '$VSIX'" +$XVFB node test-matrix/vscode-cell.mjs --mode fresh --target "$VSIX" \ + --out "$RESULTS/result-$OSL-stable-fresh-.json" || true + +# 6. VSCode lane — UPGRADE from $FROM +say "6. VSCode lane — UPGRADE from $FROM -> '$VSIX'" +$XVFB node test-matrix/vscode-cell.mjs --mode upgrade --from "$FROM" --target "$VSIX" \ + --out "$RESULTS/result-$OSL-stable-upgrade-$FROM.json" || true + +# 7. Optional: code-server lane via Docker (the fork-style install path) +if [ "$WITH_CODESERVER" = "1" ]; then + if command -v docker >/dev/null && docker info >/dev/null 2>&1; then + say "7. code-server lane (Docker)" + if [ "$VSIX" = "latest" ]; then + bash test-matrix/codeserver-cell.sh --mode fresh --target latest --out "$RESULTS/result-codeserver-fresh.json" || true + else + bash test-matrix/codeserver-cell.sh --mode fresh --vsix-file "$VSIX" --out "$RESULTS/result-codeserver-fresh.json" || true + fi + else + warn "Docker not running — skipping code-server lane" + fi +fi + +# 8. Fork lanes (Cursor / Windsurf / Kiro) — Linux only (need the fork's Linux +# binary + xvfb). They RUN here when on Linux (e.g. inside the parity Docker +# container), and are skipped with a note on macOS. +say "8. Fork lanes (Cursor / Windsurf / Kiro)" +if [ "$(uname)" = "Linux" ] && command -v xvfb-run >/dev/null 2>&1; then + TARGET_ARG=(--target "$VSIX"); [ "$VSIX" != "latest" ] || TARGET_ARG=(--target latest) + for fork in cursor windsurf kiro; do + prov="test-matrix/provision/$fork.sh" + [ -f "$prov" ] || { warn "$fork: no provisioner, skipping"; continue; } + say " $fork — provisioning + fresh install" + if ! prov_out="$(bash "$prov" 2>"/tmp/prov-$fork.err")" || [ -z "$prov_out" ]; then + warn "$fork: provision failed, skipping"; sed 's/^/ /' "/tmp/prov-$fork.err" 2>/dev/null | tail -4; continue + fi + eval "$prov_out" + BINVAR="$(echo "$fork" | tr '[:lower:]' '[:upper:]')_BIN"; BIN="${!BINVAR:-}" + if [ -z "$BIN" ] || [ ! -x "$BIN" ]; then warn "$fork: binary not found, skipping"; continue; fi + # fresh + xvfb-run -a node test-matrix/cursor-cell.mjs --runtime "$fork" --bin "$BIN" \ + --mode fresh --target "$VSIX" --out "$RESULTS/result-$fork-fresh.json" || true + # upgrade (only meaningful for a published target; skip for a local custom vsix + # since the baseline still comes from Open VSX which is independent of $VSIX) + xvfb-run -a node test-matrix/cursor-cell.mjs --runtime "$fork" --bin "$BIN" \ + --mode upgrade --from "$FROM" --target "$VSIX" --out "$RESULTS/result-$fork-upgrade-$FROM.json" || true + done +else + warn "Not on Linux (or no xvfb) — fork lanes skipped. Run inside the parity container" + warn "(bash test-matrix/run-in-docker.sh) to exercise Cursor/Windsurf/Kiro exactly as CI does." +fi + +# 9. Aggregate -> board +say "9. Aggregate into the install/update board" +# aggregate.py already prints the combined board to stdout; don't cat it again. +python3 test-matrix/aggregate.py --results-dir "$RESULTS" --out-dir "$RESULTS/agg" --target "$VSIX" --trigger local || warn "no board produced" +echo +say "Per-cell results" +for f in "$RESULTS"/result-*.json; do + [ -f "$f" ] && python3 -c "import json,sys;d=json.load(open('$f'));print(f\" {d['runtime']}/{d['os']}/{d['scenario']}\"+((' from '+str(d['from'])) if d.get('from') else '')+f\": {d['status']} ({d.get('reason','') or 'ok'})\")" +done +echo +# When run inside the parity container, copy the board + results out to the host. +if [ -n "${MATRIX_HOST_OUT:-}" ] && [ -d "$MATRIX_HOST_OUT" ]; then + cp -f "$RESULTS"/result-*.json "$MATRIX_HOST_OUT"/ 2>/dev/null || true + cp -rf "$RESULTS/agg" "$MATRIX_HOST_OUT"/ 2>/dev/null || true + ok "Board + results copied to host: .matrix-docker-out/" +fi +ok "Local run complete. Results dir: $RESULTS" diff --git a/test-matrix/setup-dbt-env.sh b/test-matrix/setup-dbt-env.sh new file mode 100755 index 000000000..8c9044548 --- /dev/null +++ b/test-matrix/setup-dbt-env.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Creates a hermetic dbt env for the dbt-core-sample-duckdb fixture and prints, +# on the LAST line, the python interpreter path (for dbt.dbtPythonPathOverride). +# Also writes a profiles dir with a tmp duckdb path and exports DBT_PROFILES_DIR. +# +# eval "$(bash test-matrix/setup-dbt-env.sh)" # exports PY_INTERP + DBT_PROFILES_DIR +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FIXTURE="$REPO_ROOT/test-fixtures/dbt-core-sample-duckdb" +VENV="${MATRIX_VENV:-$REPO_ROOT/.matrix-venv}" +PROFILES_DIR="${MATRIX_PROFILES_DIR:-$(mktemp -d)}" + +python3 -m venv "$VENV" +# shellcheck disable=SC1091 +"$VENV/bin/pip" install --quiet --upgrade pip +"$VENV/bin/pip" install --quiet "dbt-core==1.9.6" "dbt-duckdb==1.9.3" + +# Hermetic profiles.yml: same profile name as the fixture, duckdb at a tmp path. +cat > "$PROFILES_DIR/profiles.yml" </dev/null 2>&1 || true ) + +# Emit shell-eval-able exports. +echo "export PY_INTERP='$VENV/bin/python'" +echo "export DBT_PROFILES_DIR='$PROFILES_DIR'" diff --git a/test-matrix/vscode-cell.mjs b/test-matrix/vscode-cell.mjs new file mode 100644 index 000000000..e3f4a9ba9 --- /dev/null +++ b/test-matrix/vscode-cell.mjs @@ -0,0 +1,177 @@ +#!/usr/bin/env node +// Driver for one VSCode/Insiders matrix cell. Downloads a real editor, installs +// the extension (+ dependencies) via the editor CLI, launches it headless against +// the dbt-core-sample-duckdb fixture, and writes a RESULT_JSON. +// +// Usage: +// node test-matrix/vscode-cell.mjs --mode fresh|upgrade --target \ +// [--from ] [--vscode-version stable|insiders|x.y.z] --out +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + downloadAndUnzipVSCode, + resolveCliArgsFromVSCodeExecutablePath, + runTests, +} from "@vscode/test-electron"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const EXTENSION_ID = "innoverio.vscode-dbt-power-user"; +const DEPS = ["samuelcolvin.jinjahtml", "ms-python.python", "altimateai.vscode-altimate-mcp-server"]; + +function arg(name, def = undefined) { + const i = process.argv.indexOf(`--${name}`); + if (i === -1) return def; + const v = process.argv[i + 1]; + return v && !v.startsWith("--") ? v : true; +} + +function osLabel() { + if (process.platform === "darwin") return "macos"; + if (process.platform === "win32") return "windows"; + return "linux"; +} + +async function main() { + const mode = arg("mode", "fresh"); + const target = arg("target", "latest"); // vsix path or "latest" + const fromVersion = arg("from", null); + const vscodeVersion = arg("vscode-version", "stable"); // stable | insiders | x.y.z + const outPath = resolve(arg("out", "/tmp/result.json")); + const repoRoot = resolve(join(HERE, "..")); + const fixture = join(repoRoot, "test-fixtures", "dbt-core-sample-duckdb"); + const runtime = vscodeVersion === "insiders" ? "vscode-insiders" : "vscode"; + + const result = { + runtime, os: osLabel(), scenario: mode, from: fromVersion || null, + to: target === "latest" ? "latest" : "pr-build", install_ok: false, + deps_resolved: {}, activation_ok: false, dbt_flow_ok: false, + status: "fail", reason: "", duration_s: 0, log_artifact: outPath, + }; + const started = Date.now(); + + const extDir = mkdtempSync(join(tmpdir(), "matrix-ext-")); + const uddDir = mkdtempSync(join(tmpdir(), "matrix-udd-")); + + // Pre-seed user settings so the extension finds dbt (via the hermetic venv's + // python) and so the test run does NOT emit real telemetry to App Insights. + const userDir = join(uddDir, "User"); + mkdirSync(userDir, { recursive: true }); + const pyInterp = process.env.PY_INTERP || ""; + writeFileSync( + join(userDir, "settings.json"), + JSON.stringify( + { + "dbt.dbtIntegration": "core", + ...(pyInterp ? { "dbt.dbtPythonPathOverride": pyInterp } : {}), + "dbt.altimateAiKey": "", + "telemetry.telemetryLevel": "off", + "redhat.telemetry.enabled": false, + "workbench.startupEditor": "none", + }, + null, + 2, + ), + ); + + try { + // 1. Download the editor + resolve its CLI. + const exe = await downloadAndUnzipVSCode(vscodeVersion); + const [cli, ...baseArgs] = resolveCliArgsFromVSCodeExecutablePath(exe); + // On Windows the resolved CLI is a `.cmd` (code.cmd); execFileSync cannot run + // a batch file without a shell, so enable shell there (per @vscode/test-electron). + const cliRun = (extraArgs) => + execFileSync(cli, [...baseArgs, "--extensions-dir", extDir, "--user-data-dir", uddDir, ...extraArgs], + { stdio: "pipe", encoding: "utf8", shell: process.platform === "win32" }); + + // Marketplace --install-extension is network-flaky (it can transiently resolve + // an empty version, esp. for older platform-specific builds). Retry a few times + // so a single hiccup doesn't red-flag the blocking gate. Verify the install + // actually landed (the CLI can exit 0 yet install nothing). + const sleepSync = (ms) => { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); }; + const installFromMarketplace = (idAtVersion, label) => { + let lastErr; + for (let attempt = 1; attempt <= 3; attempt++) { + try { + cliRun(["--install-extension", idAtVersion, "--force"]); + const listed = cliRun(["--list-extensions", "--show-versions"]); + if (listed.toLowerCase().includes(EXTENSION_ID.toLowerCase())) return; + lastErr = new Error(`installed but ${EXTENSION_ID} absent from --list-extensions`); + } catch (e) { + lastErr = e; + } + if (attempt < 3) sleepSync(5000 * attempt); + } + throw new Error(`${label} install failed after 3 attempts: ${lastErr && lastErr.message}`); + }; + + // 2. Install dependencies (real VSCode resolves ms-python.python from the MS marketplace). + for (const dep of DEPS) { + try { + cliRun(["--install-extension", dep, "--force"]); + result.deps_resolved[dep] = true; + } catch { + result.deps_resolved[dep] = false; + } + } + + // 3. Upgrade scenario: install the baseline first (with retry — flaky network). + if (mode === "upgrade") { + if (!fromVersion) throw new Error("--from required for upgrade mode"); + installFromMarketplace(`${EXTENSION_ID}@${fromVersion}`, `baseline ${fromVersion}`); + } + + // 4. Install the target (a built .vsix path, or latest from marketplace). + const targetArg = target === "latest" ? EXTENSION_ID : resolve(target); + if (target !== "latest" && !existsSync(targetArg)) throw new Error(`vsix not found: ${targetArg}`); + cliRun(["--install-extension", targetArg, "--force"]); + + // 5. Verify the target is present. + const listed = cliRun(["--list-extensions", "--show-versions"]); + if (!listed.toLowerCase().includes(EXTENSION_ID.toLowerCase())) { + throw new Error(`extension not present after install:\n${listed}`); + } + result.install_ok = true; + + // 6. Launch headless against the fixture; the in-host suite asserts activation + dbt init. + await runTests({ + vscodeExecutablePath: exe, + extensionDevelopmentPath: join(repoRoot, "test-matrix", "harness-ext"), + extensionTestsPath: join(repoRoot, "out", "test", "matrix", "index"), + launchArgs: [ + "--extensions-dir", extDir, + "--user-data-dir", uddDir, + "--log", "trace", + "--disable-workspace-trust", + fixture, + ], + extensionTestsEnv: { + ...process.env, + MATRIX_UDD: uddDir, + DBT_PROFILES_DIR: process.env.DBT_PROFILES_DIR ?? "", + }, + }); + + // runTests resolves only if all in-host tests passed. + result.activation_ok = true; + result.dbt_flow_ok = true; + result.status = "pass"; + } catch (err) { + // Capture the CLI's stderr/stdout too — execFileSync errors otherwise hide + // the actual reason (e.g. dependency resolution failures). install_ok / + // activation_ok already encode which phase failed. + const base = String(err && err.message ? err.message : err); + const std = `${(err && err.stderr) || ""}${(err && err.stdout) || ""}`.trim(); + result.reason = (std ? `${base} | ${std}` : base).slice(0, 600); + } finally { + result.duration_s = Math.round((Date.now() - started) / 1000); + writeFileSync(outPath, JSON.stringify(result, null, 2)); + console.log(`[matrix] ${runtime}/${result.os}/${mode} -> ${result.status}: ${result.reason || "ok"}`); + } + + process.exit(result.status === "pass" ? 0 : 1); +} + +main(); diff --git a/tests/matrix/__init__.py b/tests/matrix/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/matrix/fixtures/.gitkeep b/tests/matrix/fixtures/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/matrix/test_active_versions.py b/tests/matrix/test_active_versions.py new file mode 100644 index 000000000..b87263a0d --- /dev/null +++ b/tests/matrix/test_active_versions.py @@ -0,0 +1,93 @@ +import importlib.util +import pathlib + +_spec = importlib.util.spec_from_file_location( + "active_versions", + pathlib.Path(__file__).resolve().parents[2] / "test-matrix" / "active-versions.py", +) +av = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(av) + +# A realistic live distribution incl. a junk/fork version (1.2.16) and a +# pre-release tag, mirroring what App Insights actually returns. +_DIST = [ + ("0.61.5", 19685, 41.3), ("0.61.4", 9904, 20.8), ("0.61.2", 4803, 10.1), + ("0.60.7", 3565, 7.5), ("0.61.3", 3226, 6.8), ("0.61.0", 938, 1.97), + ("0.61.1", 789, 1.66), ("0.55.5", 562, 1.18), ("0.49.0", 100, 0.21), + ("1.2.16", 89, 0.19), ("0.34.2003", 88, 0.18), +] +# Open VSX publishes the real releases; junk/fork versions are absent. +_PUBLISHED = {"0.61.5", "0.61.4", "0.61.3", "0.61.2", "0.61.1", "0.61.0", + "0.60.7", "0.55.5", "0.49.0"} + + +def test_semver_key_orders_numerically(): + assert av._semver_key("0.9.0") < av._semver_key("0.55.5") < av._semver_key("0.61.4") + + +def test_pick_target_is_highest_published_semver_not_junk(): + # 1.2.16 has a higher numeric semver than 0.61.5 but is NOT a real release, + # so the target must resolve to 0.61.5 (publish-filtered). + assert av.pick_target(_DIST, _PUBLISHED) == "0.61.5" + + +def test_pick_baselines_popular_plus_oldest(): + b = av.pick_baselines(_DIST, target="0.61.5", min_share=1.0, max_baselines=6, + include_oldest_above=0.5, published=_PUBLISHED) + assert "0.61.5" not in b # never upgrade-from the target itself + assert {"0.61.4", "0.61.2", "0.60.7", "0.61.3", "0.61.0"} <= set(b) # top-run older versions + assert "0.55.5" in b # oldest >=0.5% kept for big-gap coverage + assert "1.2.16" not in b and "0.34.2003" not in b # junk excluded by publish filter + assert "0.49.0" not in b # below include-oldest-above threshold + assert len(b) <= 6 + assert b == sorted(b, key=av._semver_key) # output is semver-sorted + + +def test_pick_baselines_excludes_unpublished_and_prerelease(): + dist = [("0.61.5", 100, 50.0), ("0.61.4", 50, 25.0), + ("1.2.16", 30, 15.0), ("0.61.0-pre", 20, 10.0)] + published = {"0.61.5", "0.61.4"} + b = av.pick_baselines(dist, target="0.61.5", min_share=1.0, max_baselines=6, + include_oldest_above=0.5, published=published) + assert b == ["0.61.4"] + + +def test_pick_by_coverage_reaches_target_with_fewest_versions(): + # By INSTALLS over total=43749: target 0.61.5 = 45.0%, then greedily add the + # most-installed published versions until cumulative >= 90%: + # +0.61.4 -> 67.6%, +0.61.2 -> 78.6%, +0.60.7 -> 86.8%, +0.61.3 -> 94.1% (stop) + b = av.pick_by_coverage(_DIST, target="0.61.5", coverage_pct=90.0, + published=_PUBLISHED, max_baselines=10) + assert b == ["0.60.7", "0.61.2", "0.61.3", "0.61.4"] # semver-sorted, exact set + assert "0.61.5" not in b # target is never an upgrade-from + assert "0.61.0" not in b and "0.55.5" not in b # not needed to reach 90% + # it must actually cross the threshold + total = sum(n for _v, n, _s in _DIST) + covered = sum(n for v, n, _s in _DIST if v == "0.61.5" or v in set(b)) + assert 100 * covered / total >= 90.0 + + +def test_pick_by_coverage_excludes_junk_from_baselines(): + b = av.pick_by_coverage(_DIST, target="0.61.5", coverage_pct=99.9, + published=_PUBLISHED, max_baselines=20) + assert "1.2.16" not in b and "0.34.2003" not in b # unpublished -> never a baseline + + +def test_pick_by_coverage_respects_max_baselines(): + b = av.pick_by_coverage(_DIST, target="0.61.5", coverage_pct=99.9, + published=_PUBLISHED, max_baselines=2) + assert len(b) == 2 + + +def test_pick_baselines_respects_max(): + # Realistic shape: newer versions have MORE installs than older ones, so the + # oldest version is not also the most-popular. With max_baselines=3 the result + # is the 2 most-installed older versions plus the oldest eligible one. + dist = [(f"0.60.{i}", 100 + 10 * i, 5.0) for i in range(10)] # 0.60.9 most installed + published = {v for v, _, _ in dist} + b = av.pick_baselines(dist, target="0.61.5", min_share=1.0, max_baselines=3, + include_oldest_above=0.5, published=published) + assert len(b) == 3 + assert "0.60.9" in b and "0.60.8" in b # two most-installed + assert "0.60.0" in b # oldest eligible, for big-gap coverage + assert b == sorted(b, key=av._semver_key) diff --git a/tests/matrix/test_aggregate.py b/tests/matrix/test_aggregate.py new file mode 100644 index 000000000..bbe9fcb7b --- /dev/null +++ b/tests/matrix/test_aggregate.py @@ -0,0 +1,138 @@ +import importlib.util +import json +import pathlib +import subprocess +import sys + +_spec = importlib.util.spec_from_file_location( + "aggregate", pathlib.Path(__file__).resolve().parents[2] / "test-matrix" / "aggregate.py" +) +aggregate = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(aggregate) + + +def _cell(**kw): + base = dict( + runtime="vscode", os="linux", scenario="fresh", **{"from": None}, + to="0.61.5", install_ok=True, deps_resolved={}, activation_ok=True, + dbt_flow_ok=True, status="pass", reason="", duration_s=10, log_artifact="x.log", + ) + base.update(kw) + return base + + +def test_blocking_failure_when_vscode_fails(): + results = [_cell(status="fail", activation_ok=False, reason="no activate")] + out = aggregate.build_matrices(results) + assert out["has_blocking_failure"] is True + + +def test_no_blocking_failure_when_only_codeserver_fails(): + results = [ + _cell(), + _cell(runtime="code-server", status="fail", dbt_flow_ok=False, reason="boom"), + ] + out = aggregate.build_matrices(results) + assert out["has_blocking_failure"] is False + + +def test_insiders_is_non_blocking(): + results = [_cell(runtime="vscode-insiders", status="fail", reason="upstream churn")] + out = aggregate.build_matrices(results) + assert out["has_blocking_failure"] is False + + +def test_install_matrix_has_a_row_per_runtime_os(): + results = [ + _cell(os="linux"), _cell(os="windows"), _cell(os="macos"), + _cell(runtime="code-server", os="linux"), + ] + md = aggregate.build_matrices(results)["install_md"] + assert "vscode" in md and "code-server" in md + assert "linux" in md and "windows" in md and "macos" in md + assert "✅" in md + + +def test_update_matrix_groups_by_baseline(): + results = [ + _cell(scenario="upgrade", **{"from": "0.61.4"}), + _cell(scenario="upgrade", **{"from": "0.55.5"}, status="fail", dbt_flow_ok=False), + ] + md = aggregate.build_matrices(results)["update_md"] + assert "0.61.4" in md and "0.55.5" in md + assert "✅" in md and "❌" in md + + +def test_fork_failure_renders_warning_not_cross(): + # In P1 there are no forks, but a non-blocking runtime failure must render as ⚠️ + results = [_cell(runtime="code-server", status="fail", reason="x")] + md = aggregate.build_matrices(results)["install_md"] + assert "⚠️" in md + + +def test_skip_cell_renders_skip_symbol(): + results = [_cell(status="skip", reason="not applicable")] + md = aggregate.build_matrices(results)["install_md"] + assert "⏭️" in md + + +def test_cli_writes_files_and_exit_code(tmp_path): + rdir = tmp_path / "results" + rdir.mkdir() + (rdir / "a.json").write_text(json.dumps(_cell())) + (rdir / "b.json").write_text( + json.dumps( + _cell(runtime="vscode", os="windows", status="fail", activation_ok=False, reason="x") + ) + ) + odir = tmp_path / "out" + root = pathlib.Path(__file__).resolve().parents[2] + proc = subprocess.run( + [sys.executable, str(root / "test-matrix" / "aggregate.py"), + "--results-dir", str(rdir), "--out-dir", str(odir), "--target", "0.61.5"], + capture_output=True, text=True, + ) + assert proc.returncode == 1 # a blocking vscode cell failed + assert (odir / "matrix.md").exists() + assert (odir / "slack.json").exists() + slack = json.loads((odir / "slack.json").read_text()) + assert "blocks" in slack and len(slack["blocks"]) == 2 + + +def _run_cli(rdir, odir): + root = pathlib.Path(__file__).resolve().parents[2] + return subprocess.run( + [sys.executable, str(root / "test-matrix" / "aggregate.py"), + "--results-dir", str(rdir), "--out-dir", str(odir)], + capture_output=True, text=True, + ) + + +def test_cli_empty_results_blocks(tmp_path): + rdir = tmp_path / "results" + rdir.mkdir() + odir = tmp_path / "out" + proc = _run_cli(rdir, odir) + assert proc.returncode == 1 # no results == cannot certify == block + assert (odir / "matrix.md").exists() + assert (odir / "slack.json").exists() + + +def test_cli_malformed_result_blocks_even_with_a_passing_cell(tmp_path): + rdir = tmp_path / "results" + rdir.mkdir() + (rdir / "good.json").write_text(json.dumps(_cell())) # a passing vscode cell + (rdir / "bad.json").write_text("{ this is not valid json") + odir = tmp_path / "out" + proc = _run_cli(rdir, odir) + assert proc.returncode == 1 # corrupt file blocks despite the good cell passing + + +def test_update_matrix_separates_os_rows(): + results = [ + _cell(scenario="upgrade", os="linux", **{"from": "0.61.4"}), + _cell(scenario="upgrade", os="windows", **{"from": "0.61.4"}, status="fail", dbt_flow_ok=False), + ] + md = aggregate.build_matrices(results)["update_md"] + assert "vscode (linux)" in md and "vscode (windows)" in md + assert "✅" in md and "❌" in md diff --git a/tests/matrix/test_build_matrix.py b/tests/matrix/test_build_matrix.py new file mode 100644 index 000000000..c231df977 --- /dev/null +++ b/tests/matrix/test_build_matrix.py @@ -0,0 +1,66 @@ +import importlib.util +import pathlib + +_spec = importlib.util.spec_from_file_location( + "build_matrix", + pathlib.Path(__file__).resolve().parents[2] / "test-matrix" / "build-matrix.py", +) +bm = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(bm) + + +def _by(inc, osl, mode): + return [c for c in inc if c["osl"] == osl and c["mode"] == mode] + + +def test_every_baseline_tested_on_all_three_oses(): + baselines = ["0.60.7", "0.61.2", "0.61.4"] + inc = bm.build_include(baselines) + for osl in ("linux", "macos", "windows"): + ups = sorted(c["from"] for c in _by(inc, osl, "upgrade")) + assert ups == sorted(baselines), f"{osl} upgrade-from versions must be all baselines" + + +def test_each_os_has_exactly_one_fresh_stable_cell(): + inc = bm.build_include(["0.61.4"]) + for osl in ("linux", "macos", "windows"): + fresh = [c for c in inc if c["osl"] == osl and c["vscode"] == "stable" and c["mode"] == "fresh"] + assert len(fresh) == 1 + + +def test_insiders_fresh_linux_only(): + inc = bm.build_include(["0.61.4"]) + ins = [c for c in inc if c["vscode"] == "insiders"] + assert len(ins) == 1 + assert ins[0]["mode"] == "fresh" and ins[0]["osl"] == "linux" + + +def test_targets_match_os(): + inc = bm.build_include(["0.61.4", "0.60.7"]) + want = {"linux": "linux-x64", "macos": "darwin-arm64", "windows": "win32-x64"} + for c in inc: + assert c["target"] == want[c["osl"]] + + +def test_cell_count_is_three_oses_times_baselines_plus_fresh_plus_insiders(): + baselines = ["0.60.7", "0.61.0", "0.61.1", "0.61.2", "0.61.3", "0.61.4"] + inc = bm.build_include(baselines) + # 3 OSes * (1 fresh + N upgrades) + 1 insiders + assert len(inc) == 3 * (1 + len(baselines)) + 1 # = 3*7 + 1 = 22 + + +def test_empty_baselines_keeps_fresh_cells_no_upgrades(): + inc = bm.build_include([]) + assert _by(inc, "linux", "upgrade") == [] + assert _by(inc, "macos", "upgrade") == [] + assert _by(inc, "windows", "upgrade") == [] + assert len([c for c in inc if c["mode"] == "fresh"]) == 4 # 3 stable + insiders + + +def test_fallback_is_valid_semver_ascending(): + import re + for v in bm.FALLBACK_BASELINES: + assert re.match(r"^\d+\.\d+\.\d+$", v) + assert bm.FALLBACK_BASELINES == sorted( + bm.FALLBACK_BASELINES, key=lambda x: tuple(int(p) for p in x.split(".")) + )