Merge pull request #54 from hookdeck/skip-without-seed-credentials

Skip a scenario whose seed credentials are missing, rather than failing it

Author: leggetter

.github/workflows/eval-refresh.yml

@@ -303,6 +303,14 @@ jobs:
       # on 13 August scored outpost-001 as six agent failures rather than
       # skipping it, which is a false result published against named vendors.
       OUTPOST_API_KEY: ${{ secrets.OUTPOST_API_KEY }}
+      # Synthetic AWS credentials a scenario's workspace note carries as
+      # `${SEED_*}` placeholders. Not real, and not committed: a key realistic
+      # enough to convince an agent is realistic enough to trip push
+      # protection. Absent, `benchmark-outpost-004` is skipped rather than
+      # failed — an agent handed a literal placeholder is right to refuse, and
+      # scoring that is the 13 August mistake in a new form.
+      SEED_ACME_SQS_ACCESS_KEY: ${{ secrets.SEED_ACME_SQS_ACCESS_KEY }}
+      SEED_ACME_SQS_SECRET_KEY: ${{ secrets.SEED_ACME_SQS_SECRET_KEY }}
       # For `gh run cancel` when a provider stops answering. Needs
       # `actions: write` at the workflow level, which `publish-results` did
       # not previously require.
@@ -354,6 +362,8 @@ jobs:
             echo "HOOKDECK_API_KEY=${HOOKDECK_API_KEY}"
             echo "HOOKDECK_WEBHOOK_SECRET=${HOOKDECK_WEBHOOK_SECRET}"
             echo "OUTPOST_API_KEY=${OUTPOST_API_KEY}"
+            echo "SEED_ACME_SQS_ACCESS_KEY=${SEED_ACME_SQS_ACCESS_KEY}"
+            echo "SEED_ACME_SQS_SECRET_KEY=${SEED_ACME_SQS_SECRET_KEY}"
           } > .env
 
       - name: Run evals

apps/framework/harness/run-eval.ts

@@ -108,7 +108,50 @@ function unmetRequirements(ev: EvalManifest): string[] {
   const available: Record<string, boolean> = {
     outpost: Boolean(process.env.OUTPOST_API_KEY),
   };
-  return (ev.metadata.requires ?? []).filter((r) => !available[r]);
+  const unmet = (ev.metadata.requires ?? []).filter((r) => !available[r]);
+
+  // A `${SEED_*}` placeholder the environment cannot fill is an unmet
+  // requirement, not a task.
+  //
+  // `local/` files leave an unset variable as written, so the agent is handed
+  // the literal `${SEED_ACME_SQS_ACCESS_KEY}`. That reads as an obvious
+  // placeholder, and an agent that notices is right to stop rather than
+  // configure delivery that will silently fail — measured on 24 August, where
+  // exactly that scored 0/1 while three agents that did not notice scored 6/6.
+  //
+  // Scoring it is the 13 August mistake again: a missing credential became six
+  // agent failures against named vendors, and the fix then was to skip rather
+  // than score. The same reasoning applies to a credential the scenario
+  // supplies itself.
+  return [...unmet, ...unfilledSeedPlaceholders(ev)];
+}
+
+/** `SEED_*` names a scenario's workspace asks for and the environment lacks. */
+function unfilledSeedPlaceholders(ev: EvalManifest): string[] {
+  if (!ev.localDir || !existsSync(ev.localDir)) return [];
+  const missing = new Set<string>();
+
+  const walk = (dir: string) => {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const path = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        walk(path);
+        continue;
+      }
+      let text: string;
+      try {
+        text = readFileSync(path, 'utf8');
+      } catch {
+        continue; // binary; nothing to expand
+      }
+      for (const [, name] of text.matchAll(/\$\{(SEED_[A-Z0-9_]+)\}/g)) {
+        if (!process.env[name]) missing.add(name);
+      }
+    }
+  };
+  walk(ev.localDir);
+
+  return [...missing];
 }
 
 type ToolsSkill = { name: string; description: string; body: string };