diff --git a/.github/workflows/harness-quickstart.yml b/.github/workflows/harness-quickstart.yml index 6e0285d1..354f176a 100644 --- a/.github/workflows/harness-quickstart.yml +++ b/.github/workflows/harness-quickstart.yml @@ -4,25 +4,33 @@ on: schedule: - cron: "17 5 * * *" workflow_dispatch: + inputs: + channel: + description: Installer channel to validate + type: choice + options: [main, next] + default: main permissions: contents: read concurrency: - group: harness-quickstart + group: harness-quickstart-${{ inputs.channel || 'main' }} cancel-in-progress: true jobs: validate: - name: harness quickstart / latest registry + name: harness quickstart / latest registry (${{ inputs.channel || 'main' }}) runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 25 steps: - uses: actions/checkout@v5 - name: Run quickstart validator env: HARNESS_QUICKSTART_ARTIFACTS_DIR: ${{ github.workspace }}/target/harness-quickstart + III_CHANNEL: ${{ inputs.channel || 'main' }} + ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }} run: harness/tests/quickstart/run-ci.sh - name: Show quickstart logs @@ -77,6 +85,7 @@ jobs: target/harness-quickstart/EVIDENCE.md target/harness-quickstart/timings.tsv target/harness-quickstart/cli-version.txt + target/harness-quickstart/console-send.json target/harness-quickstart/console-status.json target/harness-quickstart/config.yaml target/harness-quickstart/iii.lock diff --git a/harness/tests/e2e/README.md b/harness/tests/e2e/README.md index 997d47bb..e871d906 100644 --- a/harness/tests/e2e/README.md +++ b/harness/tests/e2e/README.md @@ -44,7 +44,7 @@ scenario through one global `allow: ["*"]` policy: accuracy, remediation, and clarity. - `reactive_automation`: orchestrates three parallel database writers, trigger-spawned aggregate reactors, and a single finalizer. The CI stack uses - SQLite, so the scenario proves bounded `database::row-change` discovery and + SQLite, so the scenario proves bounded `database::row-changed` discovery and the namespaced `state`-trigger fallback. The evaluator queries the resulting tables, verifies trigger provenance from session metadata, and checks that all run-owned triggers were removed. diff --git a/harness/tests/e2e/run-ci.sh b/harness/tests/e2e/run-ci.sh index 32ba6219..cb54404f 100755 --- a/harness/tests/e2e/run-ci.sh +++ b/harness/tests/e2e/run-ci.sh @@ -137,7 +137,7 @@ start_process database \ --url "$iii_url" \ --config "$database_config" wait_for_function database::query -wait_for_trigger_type database::row-change +wait_for_trigger_type database::row-changed start_process state \ "$repo_root/state/target/release/state" \ diff --git a/harness/tests/e2e/src/scenarios/reactive_automation/prompt.rs b/harness/tests/e2e/src/scenarios/reactive_automation/prompt.rs index fe5f78da..57695041 100644 --- a/harness/tests/e2e/src/scenarios/reactive_automation/prompt.rs +++ b/harness/tests/e2e/src/scenarios/reactive_automation/prompt.rs @@ -21,7 +21,7 @@ Namespace every other resource and session with `{run_label}`. Required execution: 1. Before spawning writers, list the available trigger types and probe - `database::row-change` against `{orders}`. Allow at most 30 seconds for the probe, remove its + `database::row-changed` against `{orders}`. Allow at most 30 seconds for the probe, remove its test row afterward, and record the result. 2. Before spawning writers, register a namespaced `state` trigger targeting `harness::react`. diff --git a/harness/tests/e2e/src/scenarios/reactive_automation/queries.rs b/harness/tests/e2e/src/scenarios/reactive_automation/queries.rs index 3a2bd071..1a25bbb5 100644 --- a/harness/tests/e2e/src/scenarios/reactive_automation/queries.rs +++ b/harness/tests/e2e/src/scenarios/reactive_automation/queries.rs @@ -56,7 +56,7 @@ pub(super) async fn collect( row_change_probe: calls.iter().position(|call| { call.function_id == "engine::register_trigger" && call.arguments.get("trigger_type").and_then(Value::as_str) - == Some("database::row-change") + == Some("database::row-changed") }), state_reaction: calls.iter().position(|call| { call.function_id == "engine::register_trigger" diff --git a/harness/tests/quickstart/README.md b/harness/tests/quickstart/README.md index fa87e3de..1d3decf2 100644 --- a/harness/tests/quickstart/README.md +++ b/harness/tests/quickstart/README.md @@ -1,9 +1,8 @@ # Harness quickstart validator -This validator exercises the published installation path without provider -credentials or model calls. It runs in an isolated temporary home and project -directory, installs the `iii` CLI, starts a clean engine, and follows the -documented commands: +This validator exercises the published installation path. It runs in an +isolated temporary home and project directory, installs the `iii` CLI, starts +a clean engine, and follows the documented commands: ```bash printf 'workers: []\n' > config.yaml @@ -15,6 +14,17 @@ The check waits for the engine and the core harness/Console function surface, calls `console::status`, and fetches the Console HTTP root. It also verifies that `config.yaml` and `iii.lock` were produced by the registry install. +When `ZAI_API_KEY` is set, the validator goes one step further: it adds the +Z.AI provider (`iii worker add provider-zai`), then sends a real message +through the Console's `/ws` proxy — the same WebSocket path the browser SPA +uses — via `console_send.py`, and asserts the turn completes with a non-empty +assistant reply (default model `glm-5.2`, overridable with +`HARNESS_QUICKSTART_MODEL`/`HARNESS_QUICKSTART_PROVIDER`). Without the key the +live check is skipped and recorded as such in `result.json` and `EVIDENCE.md`. + +Set `III_CHANNEL=next` to validate the `next` installer channel instead of +`main`; in CI this is exposed as the `channel` input on manual dispatches. + Run it locally with: ```bash diff --git a/harness/tests/quickstart/console_send.py b/harness/tests/quickstart/console_send.py new file mode 100755 index 00000000..f52be828 --- /dev/null +++ b/harness/tests/quickstart/console_send.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Send a message through the Console's /ws proxy and wait for the reply. + +Speaks the engine WebSocket protocol (invokefunction/invocationresult) +through the Console proxy — the same path the browser SPA uses — so a pass +proves the Console-to-engine-to-provider chain end to end: + + 1. harness::send {message, model, provider} + 2. poll harness::status until the turn completes (fail on failed/cancelled) + 3. session::messages -> last non-empty assistant text + +Prints {"session_id", "turn_id", "reply"} as JSON on stdout; progress and +errors go to stderr. Requires the `websockets` package. +""" + +import argparse +import asyncio +import json +import sys +import time +import uuid + +import websockets + + +def eprint(message: str) -> None: + print(f"[console-send] {message}", file=sys.stderr, flush=True) + + +async def invoke(ws, function_id: str, data: dict, timeout: float = 30.0): + invocation_id = str(uuid.uuid4()) + await ws.send( + json.dumps( + { + "type": "invokefunction", + "invocation_id": invocation_id, + "function_id": function_id, + "data": data, + } + ) + ) + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"{function_id}: no invocationresult within {timeout}s") + frame = json.loads(await asyncio.wait_for(ws.recv(), remaining)) + # The proxy is transparent, so unrelated engine frames may interleave. + if ( + frame.get("type") != "invocationresult" + or frame.get("invocation_id") != invocation_id + ): + continue + if frame.get("error"): + raise RuntimeError(f"{function_id} failed: {json.dumps(frame['error'])}") + return frame.get("result") + + +def last_assistant_text(messages: list) -> str: + texts = [] + for item in messages: + message = item.get("message") or {} + if message.get("role") != "assistant": + continue + for part in message.get("content") or []: + if part.get("type") == "text" and (part.get("text") or "").strip(): + texts.append(part["text"].strip()) + return texts[-1] if texts else "" + + +async def transcript(ws, session_id: str) -> list: + messages, cursor = [], None + while True: + page = await invoke( + ws, + "session::messages", + { + "session_id": session_id, + "limit": 500, + "cursor": cursor, + "include_custom": True, + }, + ) + messages.extend(page.get("messages") or []) + next_cursor = page.get("next_cursor") + if not next_cursor or next_cursor == cursor: + break + cursor = next_cursor + return messages + + +async def run(args) -> None: + async with websockets.connect(args.url, max_size=None) as ws: + send = await invoke( + ws, + "harness::send", + { + "message": args.prompt, + "model": args.model, + "provider": args.provider, + "session": {"title": "Harness quickstart validation"}, + "options": { + "max_turns": 2, + "max_output_tokens": 1024, + "max_total_tokens": 16384, + }, + }, + ) + if not send or not send.get("accepted"): + raise RuntimeError(f"harness::send was not accepted: {json.dumps(send)}") + session_id = send["session_id"] + turn_id = send.get("turn_id") + eprint(f"turn {turn_id} accepted on session {session_id}") + + deadline = time.monotonic() + args.timeout + last_status = "unknown" + while True: + status = await invoke(ws, "harness::status", {"session_id": session_id}) + last_status = status.get("status") + if last_status in ("failed", "cancelled"): + error = status.get("result_error") or "no error was reported" + raise RuntimeError(f"turn ended as {last_status}: {error}") + if last_status == "completed" and not status.get("expects_wake"): + eprint("turn completed") + break + if time.monotonic() > deadline: + raise TimeoutError( + f"turn did not complete within {args.timeout}s " + f"(last status: {last_status})" + ) + await asyncio.sleep(2) + + reply = last_assistant_text(await transcript(ws, session_id)) + if not reply: + raise RuntimeError("no non-empty assistant reply in the session transcript") + eprint(f"assistant replied ({len(reply)} chars)") + json.dump( + {"session_id": session_id, "turn_id": turn_id, "reply": reply}, + sys.stdout, + ) + print(flush=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", required=True, help="Console proxy, e.g. ws://127.0.0.1:3113/ws") + parser.add_argument("--model", required=True) + parser.add_argument("--provider", required=True) + parser.add_argument("--prompt", required=True) + parser.add_argument("--timeout", type=int, default=240, help="turn completion timeout in seconds") + try: + asyncio.run(run(parser.parse_args())) + except Exception as error: # surface a single clean line for the validator + eprint(f"FAIL: {error}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/harness/tests/quickstart/run-ci.sh b/harness/tests/quickstart/run-ci.sh index cb9d7a69..f22ec210 100755 --- a/harness/tests/quickstart/run-ci.sh +++ b/harness/tests/quickstart/run-ci.sh @@ -17,10 +17,24 @@ script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) repo_root=$(cd -- "$script_dir/../../.." && pwd) artifact_dir=${HARNESS_QUICKSTART_ARTIFACTS_DIR:-"$repo_root/target/harness-quickstart"} install_url=${III_INSTALL_URL:-https://install.iii.dev/iii/main/install.sh} +channel=${III_CHANNEL:-main} engine_port=${HARNESS_QUICKSTART_ENGINE_PORT:-49134} wait_seconds=${HARNESS_QUICKSTART_WAIT_SECONDS:-180} add_timeout_seconds=${HARNESS_QUICKSTART_ADD_TIMEOUT_SECONDS:-600} tail_lines=${HARNESS_QUICKSTART_EVIDENCE_TAIL_LINES:-80} +# Live message check through the Console: runs only when ZAI_API_KEY is set. +send_model=${HARNESS_QUICKSTART_MODEL:-glm-5.2} +send_provider=${HARNESS_QUICKSTART_PROVIDER:-zai} +send_prompt=${HARNESS_QUICKSTART_PROMPT:-"Reply with a one-sentence confirmation that you received this message."} +turn_timeout_seconds=${HARNESS_QUICKSTART_TURN_TIMEOUT_SECONDS:-240} + +case "$channel" in + main | next) ;; + *) + echo "III_CHANNEL must be 'main' or 'next' (got: $channel)" >&2 + exit 2 + ;; +esac [[ "$engine_port" =~ ^[0-9]+$ ]] || { echo "HARNESS_QUICKSTART_ENGINE_PORT must be numeric" >&2 @@ -52,6 +66,7 @@ export PATH="$HOME/.local/bin:$HOME/.iii/bin:$PATH" engine_pid="" failure_reason="" cli_version="unknown" +message_check="skipped" log() { printf '\n[%s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2 @@ -145,11 +160,14 @@ write_result() { --arg reason "${failure_reason:-}" \ --arg cli_version "$cli_version" \ --arg install_url "$install_url" \ + --arg channel "$channel" \ + --arg message_check "$message_check" \ --argjson elapsed_ms "$elapsed_ms" \ --argjson engine_port "$engine_port" \ --argjson timings "$timings_json" \ '{status: $status, failure_reason: $reason, cli_version: $cli_version, - install_url: $install_url, elapsed_ms: $elapsed_ms, + install_url: $install_url, channel: $channel, + message_check: $message_check, elapsed_ms: $elapsed_ms, engine_port: $engine_port, timings: $timings}' \ >"$artifact_dir/result.json" else @@ -169,8 +187,10 @@ collect_evidence() { { printf '# Harness quickstart -- evidence\n\n' printf -- '- **Status:** %s\n' "$status" + printf -- '- **Channel:** %s\n' "$channel" printf -- '- **CLI:** %s\n' "$cli_version" printf -- '- **Install URL:** %s\n' "$install_url" + printf -- '- **Message check:** %s\n' "$message_check" printf -- '- **Timestamp (UTC):** %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf -- '- **Commit:** %s\n' "${GITHUB_SHA:-local}" [[ -n "$run_url" ]] && printf -- '- **CI run:** <%s>\n' "$run_url" @@ -180,6 +200,14 @@ collect_evidence() { printf '## Failure\n\n```\n%s\n```\n\n' "$failure_reason" fi + if [[ -f "$artifact_dir/console-send.json" ]]; then + printf '## Console message check (%s via %s)\n\n' "$send_model" "$send_provider" + printf '**Prompt:** %s\n\n' "$send_prompt" + printf '**Reply:**\n\n```\n' + jq -r '.reply' "$artifact_dir/console-send.json" 2>/dev/null + printf '```\n\n' + fi + if [[ -s "$timings_file" ]]; then printf '## Timing breakdown\n\n```\n' print_timing_breakdown 2>&1 @@ -190,6 +218,11 @@ collect_evidence() { for f in "$log_dir"/*.log; do [[ -f "$f" ]] || continue name=$(basename "$f") + # Poll chatter and raw trigger output add noise without evidence value; + # both files still ship in the logs artifact. + case "$name" in + discovery.log | console-status.log) continue ;; + esac printf '## %s (last %s lines)\n\n```\n' "$name" "$tail_lines" tail -n "$tail_lines" "$f" 2>/dev/null printf '```\n\n' @@ -204,12 +237,6 @@ collect_evidence() { if [[ -f "$artifact_dir/config.yaml" ]]; then printf '## config.yaml\n\n```yaml\n' cat "$artifact_dir/config.yaml" - printf '```\n\n' - fi - - if [[ -f "$artifact_dir/iii.lock" ]]; then - printf '## iii.lock\n\n```\n' - cat "$artifact_dir/iii.lock" printf '```\n' fi } >"$evidence_file" 2>/dev/null @@ -337,12 +364,32 @@ wait_for_functions() { die "required functions did not register:${missing:- unknown}" } +# wait_for_function : poll engine::functions::list until the +# function registers. +wait_for_function() { + local function_id=$1 response i + log "waiting for function $function_id (up to ${wait_seconds}s)" + for ((i = 0; i < wait_seconds; i++)); do + response=$("$iii_bin" trigger engine::functions::list --port "$engine_port" \ + --json '{"include_internal":true}' 2>>"$log_dir/discovery.log" || true) + if jq -e --arg id "$function_id" \ + '(.functions // []) | any(.[]; .function_id == $id)' \ + <<<"$response" >/dev/null 2>&1; then + ok "function $function_id registered after ${i}s" + return 0 + fi + sleep 1 + done + die "function $function_id did not register within ${wait_seconds}s" +} + +# run_add : `iii worker add` with a hard timeout. run_add() { if command -v timeout >/dev/null 2>&1; then timeout --signal=TERM --kill-after=15s "$add_timeout_seconds" \ - "$iii_bin" worker add harness console + "$iii_bin" worker add "$@" else - "$iii_bin" worker add harness console + "$iii_bin" worker add "$@" fi } @@ -364,10 +411,14 @@ cd "$project_dir" # Step 1: install the CLI from the published installer # --------------------------------------------------------------------------- stage_start install_cli -log "Step 1: install the iii CLI from $install_url" +log "Step 1: install the iii CLI from $install_url (channel=$channel)" curl -fsSL --retry 3 --retry-connrefused --retry-delay 5 \ "$install_url" -o "$run_root/install.sh" -sh "$run_root/install.sh" 2>&1 | tee "$log_dir/install.log" +if [[ "$channel" == "next" ]]; then + sh "$run_root/install.sh" --next 2>&1 | tee "$log_dir/install.log" +else + sh "$run_root/install.sh" 2>&1 | tee "$log_dir/install.log" +fi iii_bin=$(command -v iii || true) [[ -n "$iii_bin" && -x "$iii_bin" ]] || die "iii CLI was not installed" @@ -392,7 +443,7 @@ stage_end # --------------------------------------------------------------------------- stage_start worker_add_harness_console log "Step 3: iii worker add harness console" -run_add 2>&1 | tee "$log_dir/worker-add.log" +run_add harness console 2>&1 | tee "$log_dir/worker-add.log" ok "worker add exited 0" stage_end @@ -435,4 +486,40 @@ grep -Eiq 'console' iii.lock || die "iii.lock does not contain console" ok "config.yaml and iii.lock reference harness + console" stage_end -log "ALL QUICKSTART ASSERTIONS PASSED ($cli_version)" +# --------------------------------------------------------------------------- +# Steps 7-8: add the Z.AI provider and prove a real model reply through the +# Console. Gated on ZAI_API_KEY: the engine (started in Step 2) inherited +# this shell's environment, so the provider worker it spawns sees the key. +# --------------------------------------------------------------------------- +if [[ -n "${ZAI_API_KEY:-}" ]]; then + stage_start worker_add_provider_zai + log "Step 7: iii worker add provider-zai" + run_add provider-zai 2>&1 | tee "$log_dir/provider-add.log" + wait_for_function provider::zai::stream + stage_end + + stage_start console_message_check + log "Step 8: send a message through the Console /ws proxy (model=$send_model provider=$send_provider)" + message_check="failed" + python3 -m venv "$run_root/venv" >>"$log_dir/console-send.log" 2>&1 \ + || die "python3 -m venv failed (see console-send.log)" + "$run_root/venv/bin/pip" install --quiet websockets >>"$log_dir/console-send.log" 2>&1 \ + || die "pip install websockets failed (see console-send.log)" + if ! "$run_root/venv/bin/python" "$script_dir/console_send.py" \ + --url "ws://127.0.0.1:$console_port/ws" \ + --model "$send_model" --provider "$send_provider" \ + --prompt "$send_prompt" --timeout "$turn_timeout_seconds" \ + >"$artifact_dir/console-send.json" \ + 2> >(tee -a "$log_dir/console-send.log" >&2); then + die "console message send failed (see console-send.log)" + fi + reply=$(jq -er '.reply | select(length > 0)' "$artifact_dir/console-send.json") \ + || die "console send did not produce a non-empty assistant reply" + ok "assistant replied through the Console (${#reply} chars)" + message_check="passed" + stage_end +else + log "Steps 7-8: ZAI_API_KEY is not set; skipping provider add + Console message check" +fi + +log "ALL QUICKSTART ASSERTIONS PASSED ($cli_version, channel=$channel, message_check=$message_check)"