diff --git a/.github/workflows/harness-quickstart.yml b/.github/workflows/harness-quickstart.yml index 354f176a..c25a1cdc 100644 --- a/.github/workflows/harness-quickstart.yml +++ b/.github/workflows/harness-quickstart.yml @@ -8,40 +8,135 @@ on: channel: description: Installer channel to validate type: choice - options: [main, next] - default: main + options: [latest, next] + default: latest permissions: contents: read concurrency: - group: harness-quickstart-${{ inputs.channel || 'main' }} + group: harness-quickstart-${{ inputs.channel || github.event_name }} cancel-in-progress: true jobs: validate: - name: harness quickstart / latest registry (${{ inputs.channel || 'main' }}) + name: harness quickstart / published registry (${{ matrix.channel }}) + strategy: + fail-fast: false + matrix: + channel: >- + ${{ + github.event_name == 'schedule' + && fromJSON('["latest","next"]') + || fromJSON(format('["{0}"]', inputs.channel)) + }} runs-on: ubuntu-latest - timeout-minutes: 25 + # Covers registry timeouts plus terminal rendering and finalization. + timeout-minutes: 35 steps: + - name: Notify Slack that validation started + id: slack_start + continue-on-error: true + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + SLACK_CHANNEL: worker-releases + III_CHANNEL: ${{ matrix.channel }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + text="⏳ *Harness quickstart* started · \`${III_CHANNEL}\` · <${RUN_URL}|view run>" + payload=$(jq -n \ + --arg channel "$SLACK_CHANNEL" \ + --arg text "$text" \ + '{channel: $channel, text: $text}') + response=$(curl -sS --retry 3 --retry-all-errors \ + -X POST https://slack.com/api/chat.postMessage \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + -H "Content-Type: application/json; charset=utf-8" \ + -d "$payload") + if ! jq -e '.ok == true' <<<"$response" >/dev/null 2>&1; then + error=$(jq -r '.error // "invalid_response"' <<<"$response" 2>/dev/null || printf 'invalid_response') + echo "::warning::Slack start notification failed: $error" + exit 1 + fi + message_ts=$(jq -er '.ts' <<<"$response") + channel_id=$(jq -er '.channel' <<<"$response") + echo "ts=$message_ts" >>"$GITHUB_OUTPUT" + echo "channel_id=$channel_id" >>"$GITHUB_OUTPUT" + - uses: actions/checkout@v5 + - name: Install VHS for terminal recording + continue-on-error: true + env: + VHS_VERSION: "0.11.0" + TTYD_VERSION: "1.7.7" + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends ffmpeg + curl -fsSL --retry 3 \ + "https://github.com/charmbracelet/vhs/releases/download/v${VHS_VERSION}/vhs_${VHS_VERSION}_Linux_x86_64.tar.gz" \ + -o /tmp/vhs.tar.gz + tar -xzf /tmp/vhs.tar.gz -C /tmp + sudo install -m 755 "/tmp/vhs_${VHS_VERSION}_Linux_x86_64/vhs" /usr/local/bin/vhs + curl -fsSL --retry 3 \ + "https://github.com/tsl0922/ttyd/releases/download/${TTYD_VERSION}/ttyd.x86_64" \ + -o /tmp/ttyd + sudo install -m 755 /tmp/ttyd /usr/local/bin/ttyd + vhs --version + ttyd --version + ffprobe -v quiet -version | sed -n '1p' + vhs validate harness/tests/quickstart/quickstart.tape + - name: Run quickstart validator + id: quickstart env: HARNESS_QUICKSTART_ARTIFACTS_DIR: ${{ github.workspace }}/target/harness-quickstart - III_CHANNEL: ${{ inputs.channel || 'main' }} + HARNESS_QUICKSTART_TRACE: "1" + III_CHANNEL: ${{ matrix.channel }} ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }} - run: harness/tests/quickstart/run-ci.sh + run: | + set -euo pipefail + + # VHS records the real validator operations. The tape waits for a + # unique shell prompt instead of matching transient terminal output. + # Pass/fail is read from result.json because VHS cannot propagate the + # typed command's exit code. + if command -v vhs >/dev/null 2>&1; then + if vhs harness/tests/quickstart/quickstart.tape; then + if ! ffprobe -v error \ + -select_streams v:0 \ + -show_entries stream=codec_type \ + -of csv=p=0 \ + target/harness-quickstart/quickstart.mp4 | + grep -qx video; then + echo "::warning::VHS produced an invalid terminal recording" + rm -f target/harness-quickstart/quickstart.mp4 + fi + else + echo "::warning::VHS recording failed" + rm -f target/harness-quickstart/quickstart.mp4 + if [[ ! -f target/harness-quickstart/result.json ]]; then + echo "::warning::No validator result was produced; rerunning without recording" + harness/tests/quickstart/run-ci.sh + fi + fi + else + echo "::warning::VHS is unavailable; running without recording" + harness/tests/quickstart/run-ci.sh + fi + + jq -e '.status == "passed"' target/harness-quickstart/result.json >/dev/null || { + reason=$(jq -r '.failure_reason // empty' target/harness-quickstart/result.json 2>/dev/null || true) + echo "::error::quickstart validator failed${reason:+: $reason}" + exit 1 + } - name: Show quickstart logs if: always() run: | shopt -s nullglob - if [[ -f target/harness-quickstart/EVIDENCE.md ]]; then - echo "::group::EVIDENCE.md" - cat target/harness-quickstart/EVIDENCE.md - echo "::endgroup::" - fi logs=(target/harness-quickstart/logs/*.log) if ((${#logs[@]} == 0)); then echo "No quickstart logs were produced." @@ -57,48 +152,185 @@ jobs: if: always() run: | { - if [[ -f target/harness-quickstart/EVIDENCE.md ]]; then - cat target/harness-quickstart/EVIDENCE.md - elif [[ -f target/harness-quickstart/result.json ]]; then + if [[ -f target/harness-quickstart/result.json ]]; then echo "### Harness quickstart" echo jq -r ' - "| Status | CLI | Duration |", - "| --- | --- | ---: |", - "| \(.status) | \(.cli_version) | \(.elapsed_ms) ms |", - (if .failure_reason != "" then "", "Failure: \(.failure_reason)" else empty end) + "| Status | Channel | CLI | GLM canary | Duration |", + "| --- | --- | --- | --- | ---: |", + "| \(.status) | \(.channel) | \(.cli_version) | \(.message_check) | \(.elapsed_ms) ms |", + (if .failure_reason != "" then "", "**Failure:** \(.failure_reason)" else empty end) ' target/harness-quickstart/result.json else echo "### Harness quickstart" echo echo "No quickstart result was produced." fi + if [[ -f target/harness-quickstart/quickstart.mp4 ]]; then + echo + echo "Terminal recording: \`quickstart.mp4\` in the \`harness-quickstart-${{ matrix.channel }}\` artifact." + fi } >> "$GITHUB_STEP_SUMMARY" - - name: Upload quickstart result + - name: Upload quickstart artifacts if: always() uses: actions/upload-artifact@v6 with: - name: harness-quickstart-result - path: | - target/harness-quickstart/result.json - target/harness-quickstart/EVIDENCE.md - target/harness-quickstart/timings.tsv - target/harness-quickstart/cli-version.txt - target/harness-quickstart/console-send.json - target/harness-quickstart/console-status.json - target/harness-quickstart/config.yaml - target/harness-quickstart/iii.lock + name: harness-quickstart-${{ matrix.channel }} + path: target/harness-quickstart/ retention-days: 7 if-no-files-found: ignore - - name: Upload quickstart logs - if: always() - uses: actions/upload-artifact@v6 - with: - name: harness-quickstart-logs - path: | - target/harness-quickstart/logs/ - target/harness-quickstart/console.html - retention-days: 7 - if-no-files-found: ignore + - name: Finalize Slack notification + if: ${{ always() && steps.slack_start.outputs.ts != '' && steps.slack_start.outputs.channel_id != '' }} + continue-on-error: true + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + SLACK_CHANNEL_ID: ${{ steps.slack_start.outputs.channel_id }} + SLACK_MESSAGE_TS: ${{ steps.slack_start.outputs.ts }} + III_CHANNEL: ${{ matrix.channel }} + QUICKSTART_OUTCOME: ${{ steps.quickstart.outcome }} + JOB_STATUS: ${{ job.status }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} + COMMIT_SHA: ${{ github.sha }} + run: | + set -uo pipefail + + slack_call() { + local method=$1 payload=$2 response error + if ! response=$(curl -sS --retry 3 --retry-all-errors \ + -X POST "https://slack.com/api/${method}" \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + -H "Content-Type: application/json; charset=utf-8" \ + -d "$payload"); then + echo "::warning::Slack ${method} request failed" + return 1 + fi + if ! jq -e '.ok == true' <<<"$response" >/dev/null 2>&1; then + error=$(jq -r '.error // "invalid_response"' <<<"$response" 2>/dev/null || printf 'invalid_response') + echo "::warning::Slack ${method} failed: $error" + return 1 + fi + } + + result_file=target/harness-quickstart/result.json + if [[ -f "$result_file" ]]; then + result_status=$(jq -r '.status // "failed"' "$result_file") + cli_version=$(jq -r '.cli_version // "unknown"' "$result_file") + duration=$(jq -r '((.elapsed_ms // 0) / 1000 | floor | tostring) + "s"' "$result_file") + failure_reason=$(jq -r '.failure_reason // ""' "$result_file") + message_check=$(jq -r '.message_check // "unknown"' "$result_file") + else + result_status=failed + cli_version=unknown + duration=unknown + message_check=unknown + failure_reason="No result.json was produced (validator outcome: ${QUICKSTART_OUTCOME})." + fi + + short_sha=${COMMIT_SHA:0:7} + if [[ "$JOB_STATUS" == "success" && "$result_status" == "passed" ]]; then + icon=✅ + label=succeeded + detail=$(printf "*Status:* succeeded\n*Installer channel:* \`%s\`\n*CLI:* \`%s\`\n*GLM canary:* \`%s\`\n*Duration:* %s\n*Commit:* <%s|\`%s\`>\n<%s|Open GitHub Actions run>" \ + "$III_CHANNEL" "$cli_version" "$message_check" "$duration" \ + "$COMMIT_URL" "$short_sha" "$RUN_URL") + else + icon=❌ + label=failed + if [[ -z "$failure_reason" ]]; then + failure_reason="Workflow status: ${JOB_STATUS} (validator outcome: ${QUICKSTART_OUTCOME})." + fi + detail=$(printf "*Status:* failed\n*Installer channel:* \`%s\`\n*CLI:* \`%s\`\n*GLM canary:* \`%s\`\n*Duration:* %s\n*Failure:* %s\n*Commit:* <%s|\`%s\`>\n<%s|Open GitHub Actions run>" \ + "$III_CHANNEL" "$cli_version" "$message_check" "$duration" \ + "${failure_reason:-unknown failure}" \ + "$COMMIT_URL" "$short_sha" "$RUN_URL") + fi + + root_text="${icon} *Harness quickstart* ${label} · \`${III_CHANNEL}\` · GLM ${message_check} · ${duration} · <${RUN_URL}|view run>" + update_payload=$( + # shellcheck disable=SC2016 + jq -n \ + --arg channel "$SLACK_CHANNEL_ID" \ + --arg ts "$SLACK_MESSAGE_TS" \ + --arg text "$root_text" \ + '{channel: $channel, ts: $ts, text: $text}' + ) + thread_payload=$( + # shellcheck disable=SC2016 + jq -n \ + --arg channel "$SLACK_CHANNEL_ID" \ + --arg ts "$SLACK_MESSAGE_TS" \ + --arg text "$detail" \ + '{channel: $channel, thread_ts: $ts, text: $text}' + ) + + failed=0 + slack_call chat.update "$update_payload" || failed=1 + slack_call chat.postMessage "$thread_payload" || failed=1 + exit "$failed" + + - name: Upload terminal recording to Slack thread + if: ${{ always() && steps.slack_start.outputs.ts != '' && steps.slack_start.outputs.channel_id != '' }} + continue-on-error: true + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + SLACK_CHANNEL_ID: ${{ steps.slack_start.outputs.channel_id }} + SLACK_MESSAGE_TS: ${{ steps.slack_start.outputs.ts }} + run: | + set -uo pipefail + + recording=target/harness-quickstart/quickstart.mp4 + if [[ ! -s "$recording" ]]; then + echo "No terminal recording was produced; skipping the Slack upload." + exit 0 + fi + filename=$(basename "$recording") + length=$(wc -c <"$recording" | tr -d '[:space:]') + + # Slack external upload flow: reserve a URL, send the bytes, then + # attach the file to the run's thread. Needs the files:write scope. + ticket=$(curl -sS --retry 3 --retry-all-errors \ + -X POST https://slack.com/api/files.getUploadURLExternal \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + --data-urlencode "filename=${filename}" \ + --data-urlencode "length=${length}") + if ! jq -e '.ok == true' <<<"$ticket" >/dev/null 2>&1; then + error=$(jq -r '.error // "invalid_response"' <<<"$ticket" 2>/dev/null || printf 'invalid_response') + echo "::warning::Slack files.getUploadURLExternal failed: $error (is the files:write scope granted?)" + exit 1 + fi + upload_url=$(jq -er '.upload_url' <<<"$ticket") + file_id=$(jq -er '.file_id' <<<"$ticket") + + if ! curl -sS --fail --retry 3 --retry-all-errors \ + -X POST "$upload_url" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @"$recording" >/dev/null; then + echo "::warning::Slack recording upload failed" + exit 1 + fi + + complete_payload=$(jq -n \ + --arg channel "$SLACK_CHANNEL_ID" \ + --arg ts "$SLACK_MESSAGE_TS" \ + --arg id "$file_id" \ + --arg title "Harness quickstart terminal recording.mp4" \ + '{ + channel_id: $channel, + thread_ts: $ts, + initial_comment: "🎬 Terminal recording of this run", + files: [{id: $id, title: $title}] + }') + response=$(curl -sS --retry 3 --retry-all-errors \ + -X POST https://slack.com/api/files.completeUploadExternal \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + -H "Content-Type: application/json; charset=utf-8" \ + -d "$complete_payload") + if ! jq -e '.ok == true' <<<"$response" >/dev/null 2>&1; then + error=$(jq -r '.error // "invalid_response"' <<<"$response" 2>/dev/null || printf 'invalid_response') + echo "::warning::Slack files.completeUploadExternal failed: $error" + exit 1 + fi + echo "Uploaded ${filename} (${length} bytes) to the Slack thread." diff --git a/harness/tests/quickstart/README.md b/harness/tests/quickstart/README.md index 1d3decf2..c84c774e 100644 --- a/harness/tests/quickstart/README.md +++ b/harness/tests/quickstart/README.md @@ -1,8 +1,7 @@ # Harness quickstart validator -This validator exercises the published installation path. It runs in an -isolated temporary home and project directory, installs the `iii` CLI, starts -a clean engine, and follows the documented commands: +This check exercises the published installation path in an isolated temporary +home and project: ```bash printf 'workers: []\n' > config.yaml @@ -10,20 +9,17 @@ iii -c config.yaml iii worker add harness console ``` -The check waits for the engine and the core harness/Console function surface, -calls `console::status`, and fetches the Console HTTP root. It also verifies -that `config.yaml` and `iii.lock` were produced by the registry install. +It verifies that: -When `ZAI_API_KEY` is set, the validator goes one step further: it adds the -Z.AI provider (`iii worker add provider-zai`), then sends a real message -through the Console's `/ws` proxy — the same WebSocket path the browser SPA -uses — via `console_send.py`, and asserts the turn completes with a non-empty -assistant reply (default model `glm-5.2`, overridable with -`HARNESS_QUICKSTART_MODEL`/`HARNESS_QUICKSTART_PROVIDER`). Without the key the -live check is skipped and recorded as such in `result.json` and `EVIDENCE.md`. +- the published installer provides a working `iii` CLI; +- a clean engine starts; +- the core harness and Console functions register; +- `console::status` and the Console HTTP root respond; and +- `config.yaml` and `iii.lock` contain the installed workers. -Set `III_CHANNEL=next` to validate the `next` installer channel instead of -`main`; in CI this is exposed as the `channel` input on manual dispatches. +When `ZAI_API_KEY` is set, it also installs `provider-zai`, resolves +`zai/glm-5.2`, sends a real Harness message through the Console `/ws` proxy, +waits for the turn to complete, and requires a non-empty assistant reply. Run it locally with: @@ -31,17 +27,35 @@ Run it locally with: make -C harness quickstart-validate ``` -The local machine needs `curl` and `jq`; CI provides both on the standard -Ubuntu runner. - -The run narrates every step with timestamped log lines and `[ok]` assertions, -records per-stage durations in `timings.tsv` (rendered as a `[timing breakdown]` -at the end and embedded in `result.json`), and always writes an `EVIDENCE.md` -digest — status, CLI version, timing, and the tail of every log — into the -artifacts directory, on success and on failure alike. In CI the evidence file -becomes the job step summary and the raw logs are printed in collapsible -groups, so a run can be audited without downloading anything. - -The CI workflow runs this check nightly and manually against the `latest` -registry artifacts. It intentionally does not test model generation; provider -credentials and model quality are covered by the Harness E2E workflow. +The machine needs `curl` and `jq`; the GLM canary additionally needs +`python3` with `venv` support. The default engine and Console ports (`49134` +and `3113`) must be available. The default installer channel is `latest`; set +`III_CHANNEL=next` to validate `next`. `HARNESS_QUICKSTART_MODEL` overrides the +default GLM model. +Set `HARNESS_QUICKSTART_TRACE=1` to print only the important external commands +(`iii worker add`, `iii trigger`, installer, engine, and GLM send) and save the +sanitized list as `commands.log`. Polling attempts, assignments, cleanup, and +other shell internals are omitted. Secret values such as `ZAI_API_KEY` are +replaced with `[REDACTED]`. + +The nightly/manual CI workflow preserves `result.json`, the generated project +files, Console responses, raw logs, and the command trace. In CI a +[VHS](https://github.com/charmbracelet/vhs) tape records the real validator +operations and renders `quickstart.mp4` (the same recording pattern as +`iii-hq/templates`). The tape waits for a unique shell prompt so completion +does not depend on matching scrolling output. VHS cannot propagate the typed +command's exit code, so the workflow reads pass/fail from `result.json`; +recording failures do not repeat a completed live GLM request or override its +result. Each run also creates one +`#worker-releases` Slack message, updates it with the final status, posts the +result details in its thread, and uploads the terminal recording to the same +thread (pass or fail; requires the `files:write` bot scope). This uses the +organization-level `SLACK_BOT_TOKEN`; the bot must be invited to the channel. +Notification errors are reported as workflow warnings without blocking +validation. + +The nightly schedule runs both `latest` and `next` as independent matrix jobs. +Manual runs select one of those channels through the workflow input. + +Without `ZAI_API_KEY`, the live canary is recorded as `skipped`. Behavioral +quality remains covered by the Harness E2E workflows. diff --git a/harness/tests/quickstart/console_send.py b/harness/tests/quickstart/console_send.py index f52be828..f980b63e 100755 --- a/harness/tests/quickstart/console_send.py +++ b/harness/tests/quickstart/console_send.py @@ -1,17 +1,5 @@ #!/usr/bin/env python3 -"""Send a message through the Console's /ws proxy and wait for the reply. - -Speaks the engine WebSocket protocol (invokefunction/invocationresult) -through the Console proxy — the same path the browser SPA uses — so a pass -proves the Console-to-engine-to-provider chain end to end: - - 1. harness::send {message, model, provider} - 2. poll harness::status until the turn completes (fail on failed/cancelled) - 3. session::messages -> last non-empty assistant text - -Prints {"session_id", "turn_id", "reply"} as JSON on stdout; progress and -errors go to stderr. Requires the `websockets` package. -""" +"""Send one Harness message through the Console WebSocket proxy.""" import argparse import asyncio @@ -23,7 +11,7 @@ import websockets -def eprint(message: str) -> None: +def progress(message: str) -> None: print(f"[console-send] {message}", file=sys.stderr, flush=True) @@ -45,7 +33,6 @@ async def invoke(ws, function_id: str, data: dict, timeout: float = 30.0): if remaining <= 0: raise TimeoutError(f"{function_id}: no invocationresult within {timeout}s") frame = json.loads(await asyncio.wait_for(ws.recv(), remaining)) - # The proxy is transparent, so unrelated engine frames may interleave. if ( frame.get("type") != "invocationresult" or frame.get("invocation_id") != invocation_id @@ -56,18 +43,6 @@ async def invoke(ws, function_id: str, data: dict, timeout: float = 30.0): return frame.get("result") -def last_assistant_text(messages: list) -> str: - texts = [] - for item in messages: - message = item.get("message") or {} - if message.get("role") != "assistant": - continue - for part in message.get("content") or []: - if part.get("type") == "text" and (part.get("text") or "").strip(): - texts.append(part["text"].strip()) - return texts[-1] if texts else "" - - async def transcript(ws, session_id: str) -> list: messages, cursor = [], None while True: @@ -84,56 +59,67 @@ async def transcript(ws, session_id: str) -> list: messages.extend(page.get("messages") or []) next_cursor = page.get("next_cursor") if not next_cursor or next_cursor == cursor: - break + return messages cursor = next_cursor - return messages + + +def last_assistant_text(messages: list) -> str: + texts = [] + for item in messages: + message = item.get("message") or {} + if message.get("role") != "assistant": + continue + for part in message.get("content") or []: + if part.get("type") == "text" and (part.get("text") or "").strip(): + texts.append(part["text"].strip()) + return texts[-1] if texts else "" async def run(args) -> None: - async with websockets.connect(args.url, max_size=None) as ws: - send = await invoke( + async with websockets.connect(args.url, max_size=None, open_timeout=30) as ws: + sent = await invoke( ws, "harness::send", { "message": args.prompt, "model": args.model, "provider": args.provider, - "session": {"title": "Harness quickstart validation"}, + "session": {"title": "Harness quickstart GLM canary"}, "options": { - "max_turns": 2, + "max_turns": 1, "max_output_tokens": 1024, "max_total_tokens": 16384, }, }, ) - if not send or not send.get("accepted"): - raise RuntimeError(f"harness::send was not accepted: {json.dumps(send)}") - session_id = send["session_id"] - turn_id = send.get("turn_id") - eprint(f"turn {turn_id} accepted on session {session_id}") + if not sent or not sent.get("accepted"): + raise RuntimeError(f"harness::send was not accepted: {json.dumps(sent)}") + + session_id = sent["session_id"] + turn_id = sent.get("turn_id") + progress(f"turn {turn_id} accepted on session {session_id}") deadline = time.monotonic() + args.timeout - last_status = "unknown" while True: status = await invoke(ws, "harness::status", {"session_id": session_id}) - last_status = status.get("status") - if last_status in ("failed", "cancelled"): + state = (status or {}).get("status", "unknown") + if state in ("failed", "cancelled"): error = status.get("result_error") or "no error was reported" - raise RuntimeError(f"turn ended as {last_status}: {error}") - if last_status == "completed" and not status.get("expects_wake"): - eprint("turn completed") + raise RuntimeError(f"turn ended as {state}: {error}") + if state == "completed" and not status.get("expects_wake"): break - if time.monotonic() > deadline: + if time.monotonic() >= deadline: raise TimeoutError( f"turn did not complete within {args.timeout}s " - f"(last status: {last_status})" + f"(last status: {state})" ) await asyncio.sleep(2) reply = last_assistant_text(await transcript(ws, session_id)) if not reply: - raise RuntimeError("no non-empty assistant reply in the session transcript") - eprint(f"assistant replied ({len(reply)} chars)") + raise RuntimeError("session transcript has no non-empty assistant reply") + + progress(f"assistant replied ({len(reply)} chars)") json.dump( {"session_id": session_id, "turn_id": turn_id, "reply": reply}, sys.stdout, @@ -143,16 +129,16 @@ async def run(args) -> None: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--url", required=True, help="Console proxy, e.g. ws://127.0.0.1:3113/ws") + parser.add_argument("--url", required=True) parser.add_argument("--model", required=True) parser.add_argument("--provider", required=True) parser.add_argument("--prompt", required=True) - parser.add_argument("--timeout", type=int, default=240, help="turn completion timeout in seconds") + parser.add_argument("--timeout", type=int, default=240) try: asyncio.run(run(parser.parse_args())) - except Exception as error: # surface a single clean line for the validator - eprint(f"FAIL: {error}") - sys.exit(1) + except Exception as error: + progress(f"FAIL: {error}") + raise SystemExit(1) from error if __name__ == "__main__": diff --git a/harness/tests/quickstart/quickstart.tape b/harness/tests/quickstart/quickstart.tape new file mode 100644 index 00000000..a5b97f99 --- /dev/null +++ b/harness/tests/quickstart/quickstart.tape @@ -0,0 +1,26 @@ +# Records the real quickstart validator operations. A unique shell prompt makes +# completion deterministic without matching output that may leave the viewport. +Output target/harness-quickstart/quickstart.mp4 + +Set Shell bash +Set Width 1200 +Set Height 760 +Set FontSize 20 +Set FontFamily "DejaVu Sans Mono" +Set Theme "Catppuccin Mocha" +Set Padding 18 +Set Framerate 15 +Set PlaybackSpeed 1.5 +Set WaitTimeout 1200s + +Hide +Type "export PS1='__HARNESS_QUICKSTART_DONE__> '" +Enter +Type "clear" +Enter +Show + +Type "harness/tests/quickstart/run-ci.sh" +Enter +Wait /__HARNESS_QUICKSTART_DONE__>/ +Sleep 2s diff --git a/harness/tests/quickstart/run-ci.sh b/harness/tests/quickstart/run-ci.sh index f22ec210..ab1b0781 100755 --- a/harness/tests/quickstart/run-ci.sh +++ b/harness/tests/quickstart/run-ci.sh @@ -1,47 +1,74 @@ #!/usr/bin/env bash set -Eeuo pipefail -# Validate the documented install path against the published registry. This -# intentionally does not start a provider or make a model request: the goal is -# to prove that a fresh project can install and boot the harness stack. -# -# Output contract (mirrors iii-hq/quickstart-validator): -# - every step logs a timestamped narrative line plus [ok] assertions, so -# the CI job log shows live what happened and in which order; -# - each stage appends a row to $artifact_dir/timings.tsv and the run ends -# with an aligned [timing breakdown]; -# - a self-contained $artifact_dir/EVIDENCE.md digest (status, versions, -# timing, log tails) is always written, even when the run fails. +# Validate the published quickstart in an isolated home and project: +# install iii, boot an empty engine, add harness + console, and probe the +# resulting function and HTTP surfaces. script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) repo_root=$(cd -- "$script_dir/../../.." && pwd) artifact_dir=${HARNESS_QUICKSTART_ARTIFACTS_DIR:-"$repo_root/target/harness-quickstart"} install_url=${III_INSTALL_URL:-https://install.iii.dev/iii/main/install.sh} -channel=${III_CHANNEL:-main} -engine_port=${HARNESS_QUICKSTART_ENGINE_PORT:-49134} +channel=${III_CHANNEL:-latest} +engine_port=49134 wait_seconds=${HARNESS_QUICKSTART_WAIT_SECONDS:-180} add_timeout_seconds=${HARNESS_QUICKSTART_ADD_TIMEOUT_SECONDS:-600} -tail_lines=${HARNESS_QUICKSTART_EVIDENCE_TAIL_LINES:-80} -# Live message check through the Console: runs only when ZAI_API_KEY is set. -send_model=${HARNESS_QUICKSTART_MODEL:-glm-5.2} -send_provider=${HARNESS_QUICKSTART_PROVIDER:-zai} -send_prompt=${HARNESS_QUICKSTART_PROMPT:-"Reply with a one-sentence confirmation that you received this message."} turn_timeout_seconds=${HARNESS_QUICKSTART_TURN_TIMEOUT_SECONDS:-240} +send_model=${HARNESS_QUICKSTART_MODEL:-glm-5.2} +send_provider=zai +send_prompt=${HARNESS_QUICKSTART_PROMPT:-"Reply with exactly: harness quickstart ok"} +trace_enabled=${HARNESS_QUICKSTART_TRACE:-0} +has_zai_key=false +[[ -n "${ZAI_API_KEY:-}" ]] && has_zai_key=true + +case "$trace_enabled" in + 0 | 1) ;; + *) + echo "HARNESS_QUICKSTART_TRACE must be '0' or '1' (got: $trace_enabled)" >&2 + exit 2 + ;; +esac case "$channel" in - main | next) ;; + latest | next) ;; *) - echo "III_CHANNEL must be 'main' or 'next' (got: $channel)" >&2 + echo "III_CHANNEL must be 'latest' or 'next' (got: $channel)" >&2 exit 2 ;; esac -[[ "$engine_port" =~ ^[0-9]+$ ]] || { - echo "HARNESS_QUICKSTART_ENGINE_PORT must be numeric" >&2 +for command_name in curl jq; do + command -v "$command_name" >/dev/null 2>&1 || { + echo "$command_name is required" >&2 + exit 2 + } +done + +[[ "$wait_seconds" =~ ^[0-9]+$ ]] || { + echo "HARNESS_QUICKSTART_WAIT_SECONDS must be a positive integer" >&2 exit 2 } -[[ "$wait_seconds" =~ ^[0-9]+$ ]] || { - echo "HARNESS_QUICKSTART_WAIT_SECONDS must be numeric" >&2 +wait_seconds=$((10#$wait_seconds)) +((wait_seconds > 0)) || { + echo "HARNESS_QUICKSTART_WAIT_SECONDS must be a positive integer" >&2 + exit 2 +} +[[ "$add_timeout_seconds" =~ ^[0-9]+$ ]] || { + echo "HARNESS_QUICKSTART_ADD_TIMEOUT_SECONDS must be a positive integer" >&2 + exit 2 +} +add_timeout_seconds=$((10#$add_timeout_seconds)) +((add_timeout_seconds > 0)) || { + echo "HARNESS_QUICKSTART_ADD_TIMEOUT_SECONDS must be a positive integer" >&2 + exit 2 +} +[[ "$turn_timeout_seconds" =~ ^[0-9]+$ ]] || { + echo "HARNESS_QUICKSTART_TURN_TIMEOUT_SECONDS must be a positive integer" >&2 + exit 2 +} +turn_timeout_seconds=$((10#$turn_timeout_seconds)) +((turn_timeout_seconds > 0)) || { + echo "HARNESS_QUICKSTART_TURN_TIMEOUT_SECONDS must be a positive integer" >&2 exit 2 } @@ -49,212 +76,102 @@ mkdir -p "$artifact_dir" artifact_dir=$(cd "$artifact_dir" && pwd) run_root=$(mktemp -d "${TMPDIR:-/tmp}/harness-quickstart.XXXXXX") -home_dir="$run_root/home" project_dir="$run_root/project" +quickstart_home="$run_root/home" log_dir="$artifact_dir/logs" -timings_file="$artifact_dir/timings.tsv" -evidence_file="$artifact_dir/EVIDENCE.md" -mkdir -p "$home_dir" "$project_dir" "$log_dir" -# Reset per-run outputs so local reruns don't accumulate stale rows. -: >"$timings_file" -rm -f "$evidence_file" +mkdir -p "$project_dir" "$quickstart_home" "$log_dir" +rm -f \ + "$artifact_dir/result.json" \ + "$artifact_dir/cli-version.txt" \ + "$artifact_dir/console-status.json" \ + "$artifact_dir/console.html" \ + "$artifact_dir/config.yaml" \ + "$artifact_dir/iii.lock" \ + "$artifact_dir/EVIDENCE.md" \ + "$artifact_dir/timings.tsv" \ + "$artifact_dir/console-send.json" \ + "$artifact_dir/commands.log" \ + "$log_dir"/*.log + +trace_log="$artifact_dir/commands.log" +if [[ "$trace_enabled" == 1 ]]; then + : >"$trace_log" +fi -export HOME="$home_dir" -export XDG_CONFIG_HOME="$home_dir/.config" -export PATH="$HOME/.local/bin:$HOME/.iii/bin:$PATH" +export HOME="$quickstart_home" +export XDG_CONFIG_HOME="$quickstart_home/.config" +export PATH="$quickstart_home/.local/bin:$quickstart_home/.iii/bin:$PATH" engine_pid="" failure_reason="" cli_version="unknown" message_check="skipped" +started_at_seconds=$SECONDS log() { printf '\n[%s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2 } -ok() { - printf ' [ok] %s\n' "$*" >&2 -} +log_command() { + [[ "$trace_enabled" == 1 ]] || return 0 -now_ms() { - local value - value=$(date +%s%3N 2>/dev/null || true) - if [[ "$value" =~ ^[0-9]+$ ]]; then - printf '%s\n' "$value" - else - python3 -c 'import time; print(time.time_ns() // 1_000_000)' + local rendered="$*" + if [[ -n "${ZAI_API_KEY:-}" ]]; then + rendered=${rendered//"$ZAI_API_KEY"/"[REDACTED]"} fi + printf '\n $ %s\n' "$rendered" >&2 + printf '$ %s\n' "$rendered" >>"$trace_log" } -started_at_ms=$(now_ms) - -# --- per-stage timing ------------------------------------------------------ -stage_name="" -stage_start_epoch="" - -stage_start() { - stage_name=$1 - stage_start_epoch=$(date +%s) -} - -stage_end() { - [[ -n "$stage_name" ]] || return 0 - local duration=$(($(date +%s) - stage_start_epoch)) - if [[ ! -s "$timings_file" ]]; then - printf 'stage\tduration_seconds\n' >"$timings_file" - fi - printf '%s\t%s\n' "$stage_name" "$duration" >>"$timings_file" - log "[timing] $stage_name: ${duration}s" - stage_name="" - stage_start_epoch="" +ok() { + printf ' [ok] %s\n' "$*" >&2 } -print_timing_breakdown() { - [[ -s "$timings_file" ]] || return 0 - local data_lines - data_lines=$(tail -n +2 "$timings_file" | wc -l | tr -d '[:space:]') - [[ "${data_lines:-0}" -gt 0 ]] || return 0 - awk -F'\t' ' - NR == 1 { next } - { - n++ - stages[n] = $1 - durations[n] = $2 + 0 - total += $2 - if (length($1) > maxname) maxname = length($1) - } - END { - if (maxname < 6) maxname = 6 - width = maxname + 10 - print "[timing breakdown]" - for (i = 1; i <= n; i++) { - dots = "" - for (j = 0; j < width - length(stages[i]) - 1; j++) dots = dots "." - printf " %s %s %3ds\n", stages[i], dots, durations[i] - } - sep = "" - for (j = 0; j < width + 6; j++) sep = sep "-" - printf " %s\n", sep - dots = "" - for (j = 0; j < width - 6; j++) dots = dots "." - printf " %s %s %3ds\n", "Total", dots, total - } - ' "$timings_file" >&2 +die() { + failure_reason=$1 + printf '\n[FAIL] %s\n' "$failure_reason" >&2 + return 1 } write_result() { local status=$1 - local finished_at_ms elapsed_ms timings_json - finished_at_ms=$(now_ms) - elapsed_ms=$((finished_at_ms - started_at_ms)) - - if command -v jq >/dev/null 2>&1; then - timings_json='[]' - if [[ -s "$timings_file" ]]; then - timings_json=$(tail -n +2 "$timings_file" | jq -Rn \ - '[inputs | split("\t") | {stage: .[0], duration_seconds: (.[1] | tonumber)}]' \ - 2>/dev/null) || timings_json='[]' - fi - jq -n \ - --arg status "$status" \ - --arg reason "${failure_reason:-}" \ - --arg cli_version "$cli_version" \ - --arg install_url "$install_url" \ - --arg channel "$channel" \ - --arg message_check "$message_check" \ - --argjson elapsed_ms "$elapsed_ms" \ - --argjson engine_port "$engine_port" \ - --argjson timings "$timings_json" \ - '{status: $status, failure_reason: $reason, cli_version: $cli_version, - install_url: $install_url, channel: $channel, - message_check: $message_check, elapsed_ms: $elapsed_ms, - engine_port: $engine_port, timings: $timings}' \ - >"$artifact_dir/result.json" - else - printf '{"status":"%s","failure_reason":"%s"}\n' \ - "$status" "${failure_reason:-}" >"$artifact_dir/result.json" - fi -} - -# Aggregate everything we know into one markdown digest. Best-effort: runs in -# the cleanup trap, so it must never fail the script. -collect_evidence() { - local status=$1 - local run_url="" - if [[ -n "${GITHUB_SERVER_URL:-}" && -n "${GITHUB_REPOSITORY:-}" && -n "${GITHUB_RUN_ID:-}" ]]; then - run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" - fi - { - printf '# Harness quickstart -- evidence\n\n' - printf -- '- **Status:** %s\n' "$status" - printf -- '- **Channel:** %s\n' "$channel" - printf -- '- **CLI:** %s\n' "$cli_version" - printf -- '- **Install URL:** %s\n' "$install_url" - printf -- '- **Message check:** %s\n' "$message_check" - printf -- '- **Timestamp (UTC):** %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" - printf -- '- **Commit:** %s\n' "${GITHUB_SHA:-local}" - [[ -n "$run_url" ]] && printf -- '- **CI run:** <%s>\n' "$run_url" - printf '\n' - - if [[ -n "$failure_reason" ]]; then - printf '## Failure\n\n```\n%s\n```\n\n' "$failure_reason" - fi - - if [[ -f "$artifact_dir/console-send.json" ]]; then - printf '## Console message check (%s via %s)\n\n' "$send_model" "$send_provider" - printf '**Prompt:** %s\n\n' "$send_prompt" - printf '**Reply:**\n\n```\n' - jq -r '.reply' "$artifact_dir/console-send.json" 2>/dev/null - printf '```\n\n' - fi - - if [[ -s "$timings_file" ]]; then - printf '## Timing breakdown\n\n```\n' - print_timing_breakdown 2>&1 - printf '```\n\n' - fi - - local f name - for f in "$log_dir"/*.log; do - [[ -f "$f" ]] || continue - name=$(basename "$f") - # Poll chatter and raw trigger output add noise without evidence value; - # both files still ship in the logs artifact. - case "$name" in - discovery.log | console-status.log) continue ;; - esac - printf '## %s (last %s lines)\n\n```\n' "$name" "$tail_lines" - tail -n "$tail_lines" "$f" 2>/dev/null - printf '```\n\n' - done - - if [[ -f "$artifact_dir/console-status.json" ]]; then - printf '## console::status\n\n```json\n' - cat "$artifact_dir/console-status.json" - printf '```\n\n' - fi - - if [[ -f "$artifact_dir/config.yaml" ]]; then - printf '## config.yaml\n\n```yaml\n' - cat "$artifact_dir/config.yaml" - printf '```\n' - fi - } >"$evidence_file" 2>/dev/null - echo "[collect-evidence] wrote $evidence_file" >&2 + jq -n \ + --arg status "$status" \ + --arg reason "$failure_reason" \ + --arg cli_version "$cli_version" \ + --arg install_url "$install_url" \ + --arg channel "$channel" \ + --arg message_check "$message_check" \ + --arg message_model "$send_model" \ + --arg message_provider "$send_provider" \ + --argjson elapsed_ms "$(((SECONDS - started_at_seconds) * 1000))" \ + --argjson engine_port "$engine_port" \ + '{ + status: $status, + failure_reason: $reason, + cli_version: $cli_version, + install_url: $install_url, + channel: $channel, + message_check: $message_check, + message_model: $message_model, + message_provider: $message_provider, + elapsed_ms: $elapsed_ms, + engine_port: $engine_port + }' >"$artifact_dir/result.json" } stop_engine() { - if [[ -n "$engine_pid" ]] && kill -0 "$engine_pid" 2>/dev/null; then - # The engine is started in its own process group so workers started by the - # CLI cannot survive the validator after the job exits. - kill -- "-$engine_pid" 2>/dev/null || kill "$engine_pid" 2>/dev/null || true - for _ in {1..20}; do - kill -0 "$engine_pid" 2>/dev/null || break - sleep 0.1 - done - kill -KILL -- "-$engine_pid" 2>/dev/null || kill -KILL "$engine_pid" 2>/dev/null || true - wait "$engine_pid" 2>/dev/null || true - fi + [[ -n "$engine_pid" ]] && kill -0 "$engine_pid" 2>/dev/null || return 0 + + # The engine gets its own process group so registry workers cannot outlive + # the validator. + kill -- "-$engine_pid" 2>/dev/null || kill "$engine_pid" 2>/dev/null || true + for _ in {1..20}; do + kill -0 "$engine_pid" 2>/dev/null || break + sleep 0.1 + done + kill -KILL -- "-$engine_pid" 2>/dev/null || kill -KILL "$engine_pid" 2>/dev/null || true + wait "$engine_pid" 2>/dev/null || true } cleanup() { @@ -262,27 +179,15 @@ cleanup() { trap - EXIT INT TERM ERR set +e - # Close a stage interrupted by a failure so its partial duration still lands - # in the breakdown. - stage_end - stop_engine - - if [[ -f "$project_dir/config.yaml" ]]; then - cp "$project_dir/config.yaml" "$artifact_dir/config.yaml" - fi - if [[ -f "$project_dir/iii.lock" ]]; then - cp "$project_dir/iii.lock" "$artifact_dir/iii.lock" - fi - - print_timing_breakdown + for output in config.yaml iii.lock; do + [[ -f "$project_dir/$output" ]] && cp "$project_dir/$output" "$artifact_dir/$output" + done if ((status == 0)); then write_result passed - collect_evidence PASS else write_result failed - collect_evidence "FAIL (exit $status)" echo "quickstart validator failed: ${failure_reason:-exit $status}" >&2 fi @@ -292,35 +197,25 @@ cleanup() { on_error() { local status=$? line=$1 - if [[ -z "$failure_reason" ]]; then - failure_reason="command failed at line $line (exit $status)" - fi + [[ -n "$failure_reason" ]] || failure_reason="command failed at line $line (exit $status)" return "$status" } trap 'on_error "$LINENO"' ERR -trap cleanup EXIT INT TERM - -die() { - failure_reason=$1 - printf '\n[FAIL] %s\n' "$1" >&2 - return 1 -} - -command -v curl >/dev/null 2>&1 || die "curl is required" -command -v jq >/dev/null 2>&1 || die "jq is required" +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM wait_for_engine() { - local response i - log "waiting for engine on port $engine_port (up to ${wait_seconds}s)" - for ((i = 0; i < wait_seconds; i++)); do - if ! kill -0 "$engine_pid" 2>/dev/null; then - die "engine exited before becoming ready" - fi + local response attempt + log "Waiting for engine on port $engine_port (up to ${wait_seconds}s)" + log_command "iii trigger engine::workers::list --port $engine_port --json '{}'" + for ((attempt = 0; attempt < wait_seconds; attempt++)); do + kill -0 "$engine_pid" 2>/dev/null || die "engine exited before becoming ready" response=$("$iii_bin" trigger engine::workers::list --port "$engine_port" \ --json '{}' 2>>"$log_dir/discovery.log" || true) if jq -e '.workers != null' <<<"$response" >/dev/null 2>&1; then - ok "engine answered engine::workers::list after ${i}s" + ok "engine ready after ${attempt}s" return 0 fi sleep 1 @@ -329,62 +224,78 @@ wait_for_engine() { } wait_for_functions() { - local response missing function_id i - local required_functions=( - harness::send - harness::status - queue::define - session::messages - context::assemble - router::models::get - console::status - ) - - log "waiting for ${#required_functions[@]} required functions (up to ${wait_seconds}s)" - for ((i = 0; i < wait_seconds; i++)); do + local response missing attempt + local required='[ + "harness::send", + "harness::status", + "queue::define", + "session::messages", + "context::assemble", + "router::models::get", + "console::status" + ]' + + log "Waiting for the harness and Console function surface (up to ${wait_seconds}s)" + log_command "iii trigger engine::functions::list --port $engine_port --json '{\"include_internal\":true}'" + for ((attempt = 0; attempt < wait_seconds; attempt++)); do response=$("$iii_bin" trigger engine::functions::list --port "$engine_port" \ --json '{"include_internal":true}' 2>>"$log_dir/discovery.log" || true) - missing="" - for function_id in "${required_functions[@]}"; do - if ! jq -e --arg id "$function_id" \ - '(.functions // []) | any(.[]; .function_id == $id)' \ - <<<"$response" >/dev/null 2>&1; then - missing+=" $function_id" - fi - done + missing=$(jq -r --argjson required "$required" ' + (.functions // [] | map(.function_id)) as $available + | ($required - $available) + | join(" ") + ' <<<"$response" 2>/dev/null || printf 'function discovery failed') if [[ -z "$missing" ]]; then - ok "all required functions registered after ${i}s" + ok "all required functions registered after ${attempt}s" return 0 fi - if ((i > 0 && i % 15 == 0)); then - log "still waiting; missing:${missing}" + if ((attempt > 0 && attempt % 15 == 0)); then + log "Still waiting for: $missing" fi sleep 1 done - die "required functions did not register:${missing:- unknown}" + die "required functions did not register: $missing" } -# wait_for_function : poll engine::functions::list until the -# function registers. wait_for_function() { - local function_id=$1 response i - log "waiting for function $function_id (up to ${wait_seconds}s)" - for ((i = 0; i < wait_seconds; i++)); do + local function_id=$1 response attempt + log "Waiting for $function_id (up to ${wait_seconds}s)" + log_command "iii trigger engine::functions::list --port $engine_port --json '{\"include_internal\":true}'" + for ((attempt = 0; attempt < wait_seconds; attempt++)); do response=$("$iii_bin" trigger engine::functions::list --port "$engine_port" \ --json '{"include_internal":true}' 2>>"$log_dir/discovery.log" || true) if jq -e --arg id "$function_id" \ - '(.functions // []) | any(.[]; .function_id == $id)' \ + 'any(.functions[]?; .function_id == $id)' \ + <<<"$response" >/dev/null 2>&1; then + ok "$function_id registered after ${attempt}s" + return 0 + fi + sleep 1 + done + die "$function_id did not register within ${wait_seconds}s" +} + +wait_for_model() { + local request response attempt + request=$(jq -cn --arg provider "$send_provider" --arg id "$send_model" \ + '{provider: $provider, id: $id}') + log "Waiting for model $send_provider/$send_model (up to ${wait_seconds}s)" + log_command "iii trigger router::models::get --port $engine_port --json '{\"provider\":\"$send_provider\",\"id\":\"$send_model\"}'" + for ((attempt = 0; attempt < wait_seconds; attempt++)); do + response=$("$iii_bin" trigger router::models::get --port "$engine_port" \ + --json "$request" 2>>"$log_dir/discovery.log" || true) + if jq -e --arg id "$send_model" '.model.id == $id' \ <<<"$response" >/dev/null 2>&1; then - ok "function $function_id registered after ${i}s" + ok "model $send_provider/$send_model resolved after ${attempt}s" return 0 fi sleep 1 done - die "function $function_id did not register within ${wait_seconds}s" + die "model $send_provider/$send_model did not resolve within ${wait_seconds}s" } -# run_add : `iii worker add` with a hard timeout. -run_add() { +run_worker_add() { + log_command "iii worker add $*" if command -v timeout >/dev/null 2>&1; then timeout --signal=TERM --kill-after=15s "$add_timeout_seconds" \ "$iii_bin" worker add "$@" @@ -394,132 +305,101 @@ run_add() { } start_engine() { + local command=("$iii_bin" -c "$project_dir/config.yaml" --no-update-check) + log_command "iii -c config.yaml --no-update-check" if command -v setsid >/dev/null 2>&1; then - setsid "$iii_bin" -c "$project_dir/config.yaml" --no-update-check \ - >"$log_dir/engine.log" 2>&1 & + setsid "${command[@]}" >"$log_dir/engine.log" 2>&1 & else - "$iii_bin" -c "$project_dir/config.yaml" --no-update-check \ - >"$log_dir/engine.log" 2>&1 & + "${command[@]}" >"$log_dir/engine.log" 2>&1 & fi engine_pid=$! } -mkdir -p "$project_dir" cd "$project_dir" -# --------------------------------------------------------------------------- -# Step 1: install the CLI from the published installer -# --------------------------------------------------------------------------- -stage_start install_cli -log "Step 1: install the iii CLI from $install_url (channel=$channel)" +log "Step 1/6: Install iii from $install_url (channel=$channel)" +log_command "curl -fsSL $install_url -o install.sh" curl -fsSL --retry 3 --retry-connrefused --retry-delay 5 \ "$install_url" -o "$run_root/install.sh" if [[ "$channel" == "next" ]]; then + log_command "sh install.sh --next" sh "$run_root/install.sh" --next 2>&1 | tee "$log_dir/install.log" else + log_command "sh install.sh" sh "$run_root/install.sh" 2>&1 | tee "$log_dir/install.log" fi - iii_bin=$(command -v iii || true) [[ -n "$iii_bin" && -x "$iii_bin" ]] || die "iii CLI was not installed" +log_command "iii --version" cli_version=$("$iii_bin" --version 2>&1) printf '%s\n' "$cli_version" >"$artifact_dir/cli-version.txt" -ok "installed $cli_version at $iii_bin" -stage_end - -# --------------------------------------------------------------------------- -# Step 2: start a clean engine -# --------------------------------------------------------------------------- -stage_start start_engine -log "Step 2: start the engine with an empty config" +ok "installed $cli_version" + +log "Step 2/6: Start an empty engine" printf 'workers: []\n' >config.yaml start_engine -ok "engine started (pid $engine_pid)" wait_for_engine -stage_end - -# --------------------------------------------------------------------------- -# Step 3: the command this pipeline exists to guarantee -# --------------------------------------------------------------------------- -stage_start worker_add_harness_console -log "Step 3: iii worker add harness console" -run_add harness console 2>&1 | tee "$log_dir/worker-add.log" -ok "worker add exited 0" -stage_end - -# --------------------------------------------------------------------------- -# Step 4: wait for the harness/Console function surface -# --------------------------------------------------------------------------- -stage_start wait_for_functions -log "Step 4: wait for the harness/Console function surface" + +log "Step 3/6: Add harness and Console" +run_worker_add harness console 2>&1 | tee "$log_dir/worker-add.log" +ok "iii worker add harness console exited successfully" + +log "Step 4/6: Verify registered functions" wait_for_functions -stage_end -# --------------------------------------------------------------------------- -# Step 5: Console answers over HTTP -# --------------------------------------------------------------------------- -stage_start console_check -log "Step 5: query console::status and fetch the Console root" +log "Step 5/6: Verify the Console HTTP surface" +log_command "iii trigger console::status --port $engine_port --json '{}'" console_status=$("$iii_bin" trigger console::status --port "$engine_port" \ --json '{}' 2>"$log_dir/console-status.log") printf '%s\n' "$console_status" >"$artifact_dir/console-status.json" console_port=$(jq -er '.http_port | select(type == "number")' <<<"$console_status") \ || die "console::status did not return a numeric http_port" -ok "console reports http_port=$console_port" - -curl -fsS --retry 10 --retry-delay 1 \ +log_command "curl -fsS http://127.0.0.1:$console_port/" +curl -fsS --retry 10 --retry-all-errors --retry-delay 1 \ "http://127.0.0.1:$console_port/" -o "$artifact_dir/console.html" -ok "fetched Console root ($(wc -c <"$artifact_dir/console.html" | tr -d ' ') bytes)" -stage_end - -# --------------------------------------------------------------------------- -# Step 6: the install left the documented files behind -# --------------------------------------------------------------------------- -stage_start verify_files -log "Step 6: verify config.yaml and iii.lock" -[[ -s config.yaml ]] || die "worker add did not write config.yaml" -[[ -s iii.lock ]] || die "worker add did not write iii.lock" -grep -Eiq 'harness' config.yaml || die "config.yaml does not contain harness" -grep -Eiq 'console' config.yaml || die "config.yaml does not contain console" -grep -Eiq 'harness' iii.lock || die "iii.lock does not contain harness" -grep -Eiq 'console' iii.lock || die "iii.lock does not contain console" +ok "Console answered on port $console_port" + +log "Step 6/6: Verify generated project files" +for output in config.yaml iii.lock; do + [[ -s "$output" ]] || die "worker add did not write $output" + grep -Eiq 'harness' "$output" || die "$output does not contain harness" + grep -Eiq 'console' "$output" || die "$output does not contain console" +done ok "config.yaml and iii.lock reference harness + console" -stage_end - -# --------------------------------------------------------------------------- -# Steps 7-8: add the Z.AI provider and prove a real model reply through the -# Console. Gated on ZAI_API_KEY: the engine (started in Step 2) inherited -# this shell's environment, so the provider worker it spawns sees the key. -# --------------------------------------------------------------------------- -if [[ -n "${ZAI_API_KEY:-}" ]]; then - stage_start worker_add_provider_zai - log "Step 7: iii worker add provider-zai" - run_add provider-zai 2>&1 | tee "$log_dir/provider-add.log" - wait_for_function provider::zai::stream - stage_end - stage_start console_message_check - log "Step 8: send a message through the Console /ws proxy (model=$send_model provider=$send_provider)" +if [[ "$has_zai_key" == true ]]; then + command -v python3 >/dev/null 2>&1 || die "python3 is required for the GLM canary" message_check="failed" + + log "Step 7/8: Add provider-zai and resolve $send_provider/$send_model" + run_worker_add provider-zai 2>&1 | tee "$log_dir/provider-add.log" + wait_for_function provider::zai::stream + wait_for_model + + log "Step 8/8: Send a real message through the Console /ws proxy" python3 -m venv "$run_root/venv" >>"$log_dir/console-send.log" 2>&1 \ || die "python3 -m venv failed (see console-send.log)" - "$run_root/venv/bin/pip" install --quiet websockets >>"$log_dir/console-send.log" 2>&1 \ + "$run_root/venv/bin/pip" install --quiet --disable-pip-version-check websockets \ + >>"$log_dir/console-send.log" 2>&1 \ || die "pip install websockets failed (see console-send.log)" + log_command "python console_send.py --model $send_model --provider $send_provider --timeout $turn_timeout_seconds" if ! "$run_root/venv/bin/python" "$script_dir/console_send.py" \ --url "ws://127.0.0.1:$console_port/ws" \ - --model "$send_model" --provider "$send_provider" \ - --prompt "$send_prompt" --timeout "$turn_timeout_seconds" \ + --model "$send_model" \ + --provider "$send_provider" \ + --prompt "$send_prompt" \ + --timeout "$turn_timeout_seconds" \ >"$artifact_dir/console-send.json" \ 2> >(tee -a "$log_dir/console-send.log" >&2); then - die "console message send failed (see console-send.log)" + die "GLM canary failed (see console-send.log)" fi - reply=$(jq -er '.reply | select(length > 0)' "$artifact_dir/console-send.json") \ - || die "console send did not produce a non-empty assistant reply" - ok "assistant replied through the Console (${#reply} chars)" + reply=$(jq -er '.reply | select(type == "string" and length > 0)' \ + "$artifact_dir/console-send.json") \ + || die "GLM canary did not produce a non-empty assistant reply" message_check="passed" - stage_end + ok "GLM replied through the Console (${#reply} chars)" else - log "Steps 7-8: ZAI_API_KEY is not set; skipping provider add + Console message check" + log "GLM canary skipped: ZAI_API_KEY is not set" fi log "ALL QUICKSTART ASSERTIONS PASSED ($cli_version, channel=$channel, message_check=$message_check)"