diff --git a/loops/experiment/README.md b/loops/experiment/README.md new file mode 100644 index 0000000..835a529 --- /dev/null +++ b/loops/experiment/README.md @@ -0,0 +1,54 @@ +# experiment-orchestrator loop + +A **long-horizon, multi-step** loop that drives an experiment from hypothesis to +decision — the flagship example of loopy's `longrun` primitives (durable state, +human-approval gates, resumable plans). + +It is **not** a single-shot `Loop`; it advances a resumable **plan** via +`runPlan`. Call `advanceExperiment(...)` repeatedly (e.g. on a schedule); each +call resumes from the last pause and is a no-op once the plan completes. + +## The plan + +``` +design → approve-design (gate) → launch → bake (wait) → readout → decide (gate) +``` + +| Step | What happens | Pause? | +|------|--------------|--------| +| **design** | AI designer turns the hypothesis into an `ExperimentDesign` (variants, metric, guardrails, sample size, duration) | — | +| **approve-design** | Human approval gate before spend | **blocked** until decided | +| **launch** | Create the experiment on the platform; record launch time | — | +| **bake** | Wait until results are available (long-horizon) | **waiting** until results land | +| **readout** | Render a readout from results | — | +| **decide** | Ship/kill/iterate — human gate; respects the recommendation | **blocked** until decided | + +A rejected design short-circuits the plan to `finalDecision: "rejected"` without +launching. + +## Boundaries (injected, testable) + +- `services.designer` — the AI design step (`createExperimentDesigner` over the + OpenRouter provider). +- `services.platform` — the experimentation platform (`launch` + `results`); + adapt to Statsig/Eppo/GrowthBook/LaunchDarkly. +- A `StateStore` (memory or file) carries plan progress + experiment registry + across runs; gates are created from it. + +## Usage + +```ts +import { advanceExperiment } from "loopy/loops/experiment"; +import { createFileStateStore } from "loopy"; + +const store = createFileStateStore(".loopy/state"); +const result = await advanceExperiment("exp-cta", hypothesis, { designer, platform }, store); +// result.status: "blocked" | "waiting" | "completed"; result.memory carries +// design, experimentKey, results, readout, finalDecision as they are produced. +``` + +## Status + +Programmatic for now. A CLI verb to advance plans on a schedule +(`loopy advance `) is a planned follow-up; it needs a persistent state +location and a hypothesis registry. diff --git a/loops/experiment/hooks/types.ts b/loops/experiment/hooks/types.ts new file mode 100644 index 0000000..301bbf3 --- /dev/null +++ b/loops/experiment/hooks/types.ts @@ -0,0 +1,38 @@ +/** A testable hypothesis for an experiment. */ +export interface Hypothesis { + id: string; + statement: string; + /** the primary success metric */ + metric: string; + /** metrics that must not regress */ + guardrailMetrics?: string[]; + /** the surface/area the experiment runs on */ + surface?: string; +} + +/** A concrete experiment design produced from a hypothesis. */ +export interface ExperimentDesign { + variants: string[]; + metric: string; + guardrailMetrics: string[]; + minSampleSize: number; + durationDays: number; +} + +/** The outcome of a baked experiment. */ +export interface ExperimentResults { + significant: boolean; + metricDelta: number; + guardrailBreached: boolean; + recommendation: "ship" | "kill" | "iterate"; +} + +/** The AI step (driven by `playbook.md`): design an experiment from a hypothesis. */ +export type ExperimentDesigner = (hypothesis: Hypothesis) => Promise; + +/** The experimentation-platform boundary (Statsig/Eppo/GrowthBook/LaunchDarkly, etc.). */ +export interface ExperimentPlatform { + launch(hypothesis: Hypothesis, design: ExperimentDesign): Promise<{ experimentKey: string }>; + /** results once the experiment has baked, or null while still running */ + results(experimentKey: string): Promise; +} diff --git a/loops/experiment/index.ts b/loops/experiment/index.ts new file mode 100644 index 0000000..f2cf60a --- /dev/null +++ b/loops/experiment/index.ts @@ -0,0 +1,168 @@ +import { + createGate, + runPlan, + type Gate, + type PlanRunResult, + type StateStore, + type Step, + type StepOutcome, +} from "../../src/core/index.js"; +import type { + ExperimentDesign, + ExperimentDesigner, + ExperimentPlatform, + ExperimentResults, + Hypothesis, +} from "./hooks/types.js"; + +export type { + ExperimentDesign, + ExperimentDesigner, + ExperimentPlatform, + ExperimentResults, + Hypothesis, +} from "./hooks/types.js"; + +export interface ExperimentServices { + designer: ExperimentDesigner; + platform: ExperimentPlatform; +} + +interface PlanInput { + planId: string; + hypothesis: Hypothesis; + services: ExperimentServices; + gate: Gate; +} + +/** Typed read of a value previously merged into plan memory. */ +function mem(memory: Readonly>, key: string): T | undefined { + return memory[key] as T | undefined; +} + +const DAY_MS = 24 * 60 * 60 * 1000; + +/** + * The experiment lifecycle as a resumable plan: + * design → approve-design (gate) → launch → bake (wait) → readout → decide (gate) + * + * Each step is safe to re-run; the plan persists progress via the StateStore and + * resumes from where it paused (a human gate or a long-horizon bake). + */ +export function experimentSteps(): Step[] { + return [ + { + name: "design", + async run({ input }): Promise { + const design = await input.services.designer(input.hypothesis); + return { status: "done", data: { design } }; + }, + }, + { + name: "approve-design", + async run({ input, memory }): Promise { + const design = mem(memory, "design"); + const gateId = `${input.planId}:design`; + const req = await input.gate.require( + gateId, + `Approve experiment design for "${input.hypothesis.statement}" ` + + `(${design?.durationDays ?? "?"}d, metric ${design?.metric ?? "?"}).`, + ); + if (req.status === "pending") return { status: "blocked", gateId }; + if (req.status === "rejected") { + return { status: "done", data: { rejected: true, finalDecision: "rejected" } }; + } + return { status: "done" }; + }, + }, + { + name: "launch", + async run({ input, memory, now }): Promise { + if (mem(memory, "rejected")) return { status: "done" }; + const design = mem(memory, "design"); + if (!design) return { status: "done", data: { error: "missing design" } }; + const { experimentKey } = await input.services.platform.launch(input.hypothesis, design); + return { status: "done", data: { experimentKey, launchedAtIso: now.toISOString() } }; + }, + }, + { + name: "bake", + async run({ input, memory, now }): Promise { + if (mem(memory, "rejected")) return { status: "done" }; + const experimentKey = mem(memory, "experimentKey"); + if (!experimentKey) return { status: "done", data: { error: "missing experimentKey" } }; + + // The step owns the wait→done transition: complete as soon as results are + // available, otherwise keep waiting until the projected bake deadline. + const results = await input.services.platform.results(experimentKey); + if (results) return { status: "done", data: { results } }; + + const design = mem(memory, "design"); + const launchedAtIso = mem(memory, "launchedAtIso"); + const launchedAt = launchedAtIso ? Date.parse(launchedAtIso) : now.getTime(); + const untilIso = new Date(launchedAt + (design?.durationDays ?? 14) * DAY_MS).toISOString(); + return { status: "waiting", untilIso }; + }, + }, + { + name: "readout", + async run({ input, memory }): Promise { + if (mem(memory, "rejected")) return { status: "done" }; + const results = mem(memory, "results"); + return { status: "done", data: { readout: renderReadout(input.hypothesis, results) } }; + }, + }, + { + name: "decide", + async run({ input, memory }): Promise { + if (mem(memory, "rejected")) { + return { status: "done", data: { finalDecision: "rejected" } }; + } + const results = mem(memory, "results"); + const gateId = `${input.planId}:decision`; + const req = await input.gate.require( + gateId, + `Ship decision for "${input.hypothesis.statement}": ` + + `recommended ${results?.recommendation ?? "?"} ` + + `(delta ${results?.metricDelta ?? "?"}, guardrail breached: ${results?.guardrailBreached ?? "?"}).`, + ); + if (req.status === "pending") return { status: "blocked", gateId }; + if (req.status === "rejected") return { status: "done", data: { finalDecision: "hold" } }; + return { status: "done", data: { finalDecision: results?.recommendation ?? "iterate" } }; + }, + }, + ]; +} + +/** Render a human-readable experiment readout. */ +export function renderReadout(hypothesis: Hypothesis, results?: ExperimentResults): string { + if (!results) return `## Experiment readout: ${hypothesis.statement}\n\n_No results available._`; + return [ + `## Experiment readout: ${hypothesis.statement}`, + "", + `- Metric: \`${hypothesis.metric}\` — delta **${results.metricDelta}** ` + + `(${results.significant ? "significant" : "not significant"})`, + `- Guardrails breached: **${results.guardrailBreached ? "yes" : "no"}**`, + `- Recommendation: **${results.recommendation}**`, + "", + "_Generated by loopy experiment-orchestrator. Final ship/kill is human-gated._", + ].join("\n"); +} + +/** + * Advance one experiment's lifecycle by (re)running its plan. Call repeatedly + * (e.g. on a schedule); it resumes from the last gate/bake pause and is a no-op + * once completed. Returns the plan result, whose `memory` carries `design`, + * `experimentKey`, `results`, `readout`, and `finalDecision` as they are produced. + */ +export async function advanceExperiment( + planId: string, + hypothesis: Hypothesis, + services: ExperimentServices, + store: StateStore, + options?: { now?: () => Date }, +): Promise { + const gate = createGate(store, options); + const input: PlanInput = { planId, hypothesis, services, gate }; + return runPlan(planId, experimentSteps(), input, store, options); +} diff --git a/loops/experiment/playbook.md b/loops/experiment/playbook.md new file mode 100644 index 0000000..656754b --- /dev/null +++ b/loops/experiment/playbook.md @@ -0,0 +1,27 @@ +# Experiment Designer Playbook + +This playbook drives the **design** step of the experiment-orchestrator loop — +the AI step that turns a hypothesis into a concrete A/B test design. The loop +framework handles approval gates, launch, baking, readout, and the ship +decision; this document tells the designer what to produce. + +The designer is supplied as `services.designer` (see `createExperimentDesigner`). +It receives a `Hypothesis` and returns an `ExperimentDesign`. + +## Instructions + +1. **Pick variants.** Usually `["control", ""]`; more only if justified. +2. **Confirm the primary metric** from the hypothesis; restate it precisely. +3. **Always include guardrail metrics** — the things that must not regress + (latency, error rate, revenue, churn proxy). Never ship without them. +4. **Size the test realistically** — a `minSampleSize` and `durationDays` that + can plausibly detect the expected effect. Prefer a 1–2 week bake. +5. **Return ONLY JSON**: `{ variants, metric, guardrailMetrics, minSampleSize, durationDays }`. + +## Guardrails (enforced by the loop, not optional) + +- Design is reviewed at a **human approval gate** before launch. +- The final ship/kill is a **human gate**, never automatic — the loop only + recommends based on results. +- Decisions are made on baked results (significance + guardrails), not on + intraday noise. diff --git a/openspec/archive/add-experiment-orchestrator/proposal.md b/openspec/archive/add-experiment-orchestrator/proposal.md new file mode 100644 index 0000000..1c9bd12 --- /dev/null +++ b/openspec/archive/add-experiment-orchestrator/proposal.md @@ -0,0 +1,70 @@ +# Proposal: Experiment Lifecycle Orchestrator + +**Change ID:** `add-experiment-orchestrator` +**Created:** 2026-06-22 +**Status:** Implementation Complete +**Completed:** 2026-06-22 + +--- + +## Problem Statement + +The `longrun` primitives (durable state, approval gates, resumable plans) shipped +with no loop consuming them. The Experiment Lifecycle Orchestrator — the +top-differentiation product-level loop from the research — is the flagship that +exercises all of them and proves the long-horizon model end-to-end. + +## Proposed Solution + +Add `loops/experiment/`: a resumable plan that drives an experiment +`design → approve-design (gate) → launch → bake (wait) → readout → decide (gate)` +via `runPlan`, persisting progress in a `StateStore` and pausing at human gates +and the long-horizon bake. Boundaries are injected: an AI **designer** +(`createExperimentDesigner` over the OpenRouter provider) and an +**ExperimentPlatform** (`launch` + `results`). Advanced by calling +`advanceExperiment(...)` repeatedly; resumes from the last pause; no-op once +complete. A rejected design short-circuits without launching. + +## Scope + +### In Scope +- `loops/experiment/` (types, plan/steps, `advanceExperiment`, readout, README, playbook) +- `createExperimentDesigner` AI adapter +- Full-lifecycle unit tests (gates, bake wait, completion, rejection) with fakes + memory store + +### Out of Scope +- A CLI verb to advance plans on a schedule (`loopy advance`) — follow-up (needs a persistent state location + hypothesis registry) +- Concrete platform connectors (Statsig/Eppo/GrowthBook) — injected boundary +- Auto-ship (decision is always human-gated) + +## Impact Analysis + +| Component | Change | Details | +|-----------|--------|---------| +| Core | No | consumes `longrun` as-is | +| Loops | Yes | new `loops/experiment/` (plan-based, not single-shot) | +| AI provider | Yes | `createExperimentDesigner` | + +## Success Criteria + +- [x] Blocks at the design gate; resumes on approval. +- [x] Waits during bake; completes the bake step once results land. +- [x] Blocks at the decision gate; completes with the recommended decision on approval. +- [x] Rejected design short-circuits to `finalDecision: "rejected"` without launching. +- [x] State + progress persist and resume across separate `advanceExperiment` calls. + +## Risks & Mitigations + +| Risk | Prob | Impact | Mitigation | +|------|------|--------|------------| +| Acting on noisy/early results | Med | High | Bake completes only when platform reports results; decision is human-gated | +| Runaway auto-ship | Low | High | No auto-ship — explicit decision gate | +| Stuck plans | Low | Med | Steps own wait→done; resumable; rejection short-circuit | + +--- + +## Archive Information + +**Archived:** 2026-06-22 +**Outcome:** Successfully implemented +**Verification:** typecheck + lint + 111 tests + build all passing diff --git a/openspec/archive/add-experiment-orchestrator/specs/experiment-orchestrator_delta.md b/openspec/archive/add-experiment-orchestrator/specs/experiment-orchestrator_delta.md new file mode 100644 index 0000000..ab727b6 --- /dev/null +++ b/openspec/archive/add-experiment-orchestrator/specs/experiment-orchestrator_delta.md @@ -0,0 +1,41 @@ +# Delta: Experiment Lifecycle Orchestrator + +**Change ID:** `add-experiment-orchestrator` +**Affects:** `loops/experiment/`, AI provider + +--- + +## ADDED + +### Requirement: Resumable Experiment Lifecycle + +The orchestrator advances an experiment through design → approve → launch → bake +→ readout → decide as a resumable plan, pausing at human gates and the bake wait. + +#### Scenario: Design approval gate +- GIVEN a new hypothesis +- WHEN the plan is advanced +- THEN it produces a design and blocks awaiting design approval, resuming on approval + +#### Scenario: Long-horizon bake +- GIVEN an approved, launched experiment with no results yet +- WHEN the plan is advanced +- THEN it waits, and a later advance completes the bake once results are available + +#### Scenario: Human ship decision +- GIVEN baked results +- WHEN the plan is advanced +- THEN it produces a readout and blocks at the decision gate, completing with the recommended decision on approval + +#### Scenario: Rejected design short-circuits +- GIVEN the design approval is rejected +- WHEN the plan is advanced +- THEN it completes with finalDecision "rejected" and never launches + +## MODIFIED + +(None) + +## REMOVED + +(None) diff --git a/openspec/archive/add-experiment-orchestrator/tasks.md b/openspec/archive/add-experiment-orchestrator/tasks.md new file mode 100644 index 0000000..3e71fa0 --- /dev/null +++ b/openspec/archive/add-experiment-orchestrator/tasks.md @@ -0,0 +1,25 @@ +# Implementation Tasks: Experiment Lifecycle Orchestrator + +**Change ID:** `add-experiment-orchestrator` +**Status:** Implementation Complete + +--- + +## Phase 1: Orchestrator +- [x] 1.1 Types/boundaries (`hooks/types.ts`: Hypothesis, Design, Results, Designer, Platform) +- [x] 1.2 Plan steps + `advanceExperiment` over `runPlan` + gates + state +- [x] 1.3 Readout renderer +- [x] 1.4 README + playbook + +## Phase 2: AI + tests +- [x] 2.1 `createExperimentDesigner` AI adapter +- [x] 2.2 Full-lifecycle tests (gates, bake wait, completion, rejection) +- [x] 2.3 Package export + +**Quality Gate:** typecheck + lint + 111 tests + build — PASSED + +--- + +## Completion Checklist +- [x] All phases complete and validated +- [x] Ready for `/openspec-archive` diff --git a/openspec/roadmap.md b/openspec/roadmap.md index d98eb53..70d1b9a 100644 --- a/openspec/roadmap.md +++ b/openspec/roadmap.md @@ -46,11 +46,15 @@ compliance). These need new framework primitives — durable state/memory, resumable multi-step orchestration, long-horizon scheduling, human-approval gates, and non-PR output adapters. -**Delivered so far:** the **KB-Gap Self-Heal** loop (first product-level wedge, -shipped) and the **long-horizon primitives** (durable state, approval gates, -resumable plans) in `src/core/longrun/`. Remaining wedge candidates: -Experiment Lifecycle Orchestrator and Metric Anomaly Narrator (which can now be -built on `runPlan` + gates + state). +**Delivered so far:** the **long-horizon primitives** (durable state, approval +gates, resumable plans) in `src/core/longrun/`; the **KB-Gap Self-Heal** loop +(first product-level wedge, turnkey); and the **Experiment Lifecycle +Orchestrator** ([`specs/experiment-orchestrator.md`](specs/experiment-orchestrator.md)) +— the flagship long-horizon loop on `runPlan` + gates + state (design → approve → +launch → bake → readout → decide). Remaining candidates on the same primitives: +Metric Anomaly → Root-Cause Narrator, Incident Action-Item/Recurrence, Churn → +Intervention. Cross-cutting follow-ups: non-PR output adapters (issue tracker, +experimentation platform, dashboards) and a `loopy advance ` CLI verb. ## Backlog (catalogued, not yet proposed) diff --git a/openspec/specs/experiment-orchestrator.md b/openspec/specs/experiment-orchestrator.md new file mode 100644 index 0000000..60ce17b --- /dev/null +++ b/openspec/specs/experiment-orchestrator.md @@ -0,0 +1,33 @@ +# Experiment Lifecycle Orchestrator Specification + +> Source of truth. Established by change `add-experiment-orchestrator` (2026-06-22). +> The first long-horizon loop built on the `longrun` primitives (state, gates, +> resumable plans). + +## Requirements + +### Requirement: Resumable Experiment Lifecycle + +The orchestrator advances an experiment through design → approve → launch → bake +→ readout → decide as a resumable plan, persisting progress and pausing at human +gates and the bake wait. + +#### Scenario: Design approval gate +- GIVEN a new hypothesis +- WHEN the plan is advanced +- THEN it produces a design and blocks awaiting design approval, resuming on approval + +#### Scenario: Long-horizon bake +- GIVEN an approved, launched experiment with no results yet +- WHEN the plan is advanced +- THEN it waits, and a later advance completes the bake once results are available + +#### Scenario: Human ship decision +- GIVEN baked results +- WHEN the plan is advanced +- THEN it produces a readout and blocks at the decision gate, completing with the recommended decision on approval + +#### Scenario: Rejected design short-circuits +- GIVEN the design approval is rejected +- WHEN the plan is advanced +- THEN it completes with finalDecision "rejected" and never launches diff --git a/package.json b/package.json index 5fc4cf1..c69ef94 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,10 @@ "./loops/kb-gap": { "types": "./dist/loops/kb-gap/index.d.ts", "default": "./dist/loops/kb-gap/index.js" + }, + "./loops/experiment": { + "types": "./dist/loops/experiment/index.d.ts", + "default": "./dist/loops/experiment/index.js" } }, "files": [ diff --git a/src/ai/providers.ts b/src/ai/providers.ts index 11ec586..81ef1b4 100644 --- a/src/ai/providers.ts +++ b/src/ai/providers.ts @@ -9,6 +9,11 @@ import type { Reviewer, ReviewResult } from "../../loops/pr-review/index.js"; import type { CoverageGap } from "../../loops/test-coverage/index.js"; import type { TestGenerator } from "../../loops/test-coverage/index.js"; import type { ArticleWriter, KbGap } from "../../loops/kb-gap/index.js"; +import type { + ExperimentDesign, + ExperimentDesigner, + Hypothesis, +} from "../../loops/experiment/index.js"; /** AI-backed doc writer for the auto-docs loop. */ export function createDocWriter(client: AiClient): DocWriter { @@ -99,6 +104,25 @@ export function createArticleWriter(client: AiClient, kbDir = "docs/kb"): Articl }; } +/** AI-backed experiment designer for the experiment-orchestrator loop. */ +export function createExperimentDesigner(client: AiClient): ExperimentDesigner { + return async (hypothesis: Hypothesis): Promise => { + const system = + "You are an experimentation expert. Design a clean A/B test for the hypothesis. " + + "Respond ONLY with JSON: {\"variants\": string[], \"metric\": string, " + + "\"guardrailMetrics\": string[], \"minSampleSize\": number, \"durationDays\": number}. " + + "Always include guardrail metrics and a realistic sample size and duration."; + const user = JSON.stringify(hypothesis); + const text = await client.complete({ + messages: [ + { role: "system", content: system }, + { role: "user", content: user }, + ], + }); + return parseJsonResponse(text); + }; +} + async function safeResolve(repoRoot: string, globs: string[]): Promise { try { return await resolveFiles(repoRoot, globs); diff --git a/test/loops/experiment.test.ts b/test/loops/experiment.test.ts new file mode 100644 index 0000000..dc45449 --- /dev/null +++ b/test/loops/experiment.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "vitest"; +import { createGate, createMemoryStateStore, type StateStore } from "../../src/core/index.js"; +import { + advanceExperiment, + renderReadout, + type ExperimentPlatform, + type ExperimentResults, + type ExperimentServices, + type Hypothesis, +} from "../../loops/experiment/index.js"; + +const hypothesis: Hypothesis = { + id: "exp-1", + statement: "Bigger CTA increases signups", + metric: "signup_rate", + guardrailMetrics: ["latency_p95"], +}; + +const design = { + variants: ["control", "bigger-cta"], + metric: "signup_rate", + guardrailMetrics: ["latency_p95"], + minSampleSize: 10000, + durationDays: 14, +}; + +const shipResults: ExperimentResults = { + significant: true, + metricDelta: 0.04, + guardrailBreached: false, + recommendation: "ship", +}; + +/** Platform whose results readiness is toggled by the test. */ +function platform(ready: { value: ExperimentResults | null }): ExperimentPlatform { + return { + launch: async () => ({ experimentKey: "key-1" }), + results: async () => ready.value, + }; +} + +const services = (ready: { value: ExperimentResults | null }): ExperimentServices => ({ + designer: async () => design, + platform: platform(ready), +}); + +const planId = "exp-1"; + +describe("renderReadout", () => { + it("summarizes results and notes the human gate", () => { + const md = renderReadout(hypothesis, shipResults); + expect(md).toContain("Recommendation: **ship**"); + expect(md).toContain("human-gated"); + }); +}); + +describe("experiment orchestrator", () => { + it("runs the full lifecycle: blocks at design gate, bakes, blocks at decision, completes", async () => { + const store: StateStore = createMemoryStateStore(); + const ready: { value: ExperimentResults | null } = { value: null }; + const gate = createGate(store); + + // 1. First advance → blocked awaiting design approval. + let result = await advanceExperiment(planId, hypothesis, services(ready), store); + expect(result.status).toBe("blocked"); + expect(result.blockedGateId).toBe("exp-1:design"); + expect(result.currentStep).toBe("approve-design"); + + // 2. Approve the design → next advance proceeds to launch then waits to bake. + await gate.decide("exp-1:design", "approved"); + result = await advanceExperiment(planId, hypothesis, services(ready), store); + expect(result.status).toBe("waiting"); + expect(result.currentStep).toBe("bake"); + expect(result.completedSteps).toContain("launch"); + expect(result.memory["experimentKey"]).toBe("key-1"); + + // 3. Results land → next advance reads out and blocks at the decision gate. + ready.value = shipResults; + result = await advanceExperiment(planId, hypothesis, services(ready), store); + expect(result.status).toBe("blocked"); + expect(result.blockedGateId).toBe("exp-1:decision"); + expect(String(result.memory["readout"])).toContain("Recommendation: **ship**"); + + // 4. Approve the ship decision → plan completes. + await gate.decide("exp-1:decision", "approved"); + result = await advanceExperiment(planId, hypothesis, services(ready), store); + expect(result.status).toBe("completed"); + expect(result.memory["finalDecision"]).toBe("ship"); + }); + + it("short-circuits to rejected when the design is rejected", async () => { + const store: StateStore = createMemoryStateStore(); + const ready: { value: ExperimentResults | null } = { value: null }; + const gate = createGate(store); + + await advanceExperiment(planId, hypothesis, services(ready), store); + await gate.decide("exp-1:design", "rejected"); + const result = await advanceExperiment(planId, hypothesis, services(ready), store); + + expect(result.status).toBe("completed"); + expect(result.memory["finalDecision"]).toBe("rejected"); + // Never launched. + expect(result.memory["experimentKey"]).toBeUndefined(); + }); +});