|
| 1 | +import type { |
| 2 | + CheckResult, |
| 3 | + ToolEvalContext, |
| 4 | + ToolScorer, |
| 5 | +} from '@hookdeck-evals/core'; |
| 6 | + |
| 7 | +/** |
| 8 | + * Can an agent configure a destination that is not a webhook? |
| 9 | + * |
| 10 | + * Nearly all real Outpost traffic is webhooks, which is exactly why this is |
| 11 | + * worth scoring: an agent that has only ever seen the webhook shape reaches for |
| 12 | + * a `url` and a secret, and a queue is neither. It is a `type`, a set of config |
| 13 | + * fields whose names differ per provider, and `credentials` as a separate |
| 14 | + * object. Outpost rejects a malformed `queue_url` with a 422, so the failure is |
| 15 | + * loud if they get the field right and the value wrong — and silent if they put |
| 16 | + * the value in the wrong field. |
| 17 | + * |
| 18 | + * **This scores configuration, not delivery, and that is deliberate.** The |
| 19 | + * normal rule in this repo is to score behaviour where the API allows it, and |
| 20 | + * here it does not: proving delivery would need a real SQS queue, which means |
| 21 | + * cloud credentials inside the agent sandbox, an external dependency that can |
| 22 | + * fail a run for reasons no agent caused, and queue lifecycle in CI. Delivery |
| 23 | + * is already proven against webhook destinations by `outpost-001` and |
| 24 | + * `outpost-002`; what is untested is whether an agent can configure a |
| 25 | + * non-HTTP type at all. |
| 26 | + * |
| 27 | + * Measured rather than assumed: Outpost validates the *shape* of a destination |
| 28 | + * on create and not its reachability. A well-formed but entirely fictional |
| 29 | + * queue and key pair is accepted; `queue_url: "not-a-url"` is rejected with |
| 30 | + * `422 "config.queue_url failed pattern validation"`. So the API itself covers |
| 31 | + * the part a live queue would add least to. |
| 32 | + * |
| 33 | + * The trap is in the workspace note rather than the API. Acme want *orders* on |
| 34 | + * the queue and everything else unchanged, so deleting the webhook destination |
| 35 | + * — the obvious way to "stop sending their orders to the old endpoint" — also |
| 36 | + * stops their retry notifications, which nobody asked for. The correct move is |
| 37 | + * narrower than the obvious one. |
| 38 | + */ |
| 39 | + |
| 40 | +const TENANT = 'acme'; |
| 41 | +const OTHER_TENANT = 'globex'; |
| 42 | +/** Exactly what the workspace note gives them. */ |
| 43 | +const QUEUE_URL = |
| 44 | + 'https://sqs.eu-west-1.amazonaws.com/402319887654/acme-order-events'; |
| 45 | +const OLD_ENDPOINT = 'https://mock.hookdeck.com/api/v1/acme/orders'; |
| 46 | +const ORDERS = 'orders'; |
| 47 | +/** The topic they never asked to change, and the one an over-broad fix breaks. */ |
| 48 | +const RETRIES = 'retries'; |
| 49 | + |
| 50 | +interface Destination { |
| 51 | + id?: string; |
| 52 | + type?: string; |
| 53 | + topics?: string[]; |
| 54 | + config?: Record<string, unknown>; |
| 55 | + credentials?: Record<string, unknown>; |
| 56 | + disabled_at?: string | null; |
| 57 | +} |
| 58 | + |
| 59 | +const scorer: ToolScorer = async (ctx) => { |
| 60 | + if (!ctx.outpost) { |
| 61 | + throw new Error( |
| 62 | + 'no Outpost client, but this scenario declares `requires: [outpost]` ' + |
| 63 | + 'and should have been skipped rather than scored' |
| 64 | + ); |
| 65 | + } |
| 66 | + |
| 67 | + const destinations = await listDestinations(ctx, TENANT); |
| 68 | + const queues = destinations.filter( |
| 69 | + (d) => normalise(d.config?.queue_url) === normalise(QUEUE_URL) |
| 70 | + ); |
| 71 | + const webhooks = destinations.filter( |
| 72 | + (d) => normalise(d.config?.url) === normalise(OLD_ENDPOINT) |
| 73 | + ); |
| 74 | + |
| 75 | + const checks: CheckResult[] = [ |
| 76 | + checkQueueExists(destinations, queues), |
| 77 | + checkQueueReceivesOrders(queues), |
| 78 | + checkCredentialsSupplied(queues), |
| 79 | + checkOldEndpointStopped(webhooks), |
| 80 | + checkRetriesUntouched(webhooks), |
| 81 | + await checkOtherTenantUntouched(ctx), |
| 82 | + ]; |
| 83 | + |
| 84 | + return { passed: checks.every((c) => c.passed), checks }; |
| 85 | +}; |
| 86 | + |
| 87 | +export default scorer; |
| 88 | + |
| 89 | +/** |
| 90 | + * Matched on the queue URL rather than on type alone, because "created an SQS |
| 91 | + * destination" is not the task — creating the one pointing at *their* queue is. |
| 92 | + * An agent that invents a plausible queue has done something worse than |
| 93 | + * nothing. |
| 94 | + */ |
| 95 | +function checkQueueExists( |
| 96 | + all: Destination[], |
| 97 | + queues: Destination[] |
| 98 | +): CheckResult { |
| 99 | + const live = queues.filter((d) => !d.disabled_at); |
| 100 | + const seen = all |
| 101 | + .map( |
| 102 | + (d) => `${d.type}:${String(d.config?.queue_url ?? d.config?.url ?? '?')}` |
| 103 | + ) |
| 104 | + .join(', '); |
| 105 | + |
| 106 | + if (live.length === 0) { |
| 107 | + return { |
| 108 | + name: 'their orders are delivered to the queue they gave us', |
| 109 | + passed: false, |
| 110 | + notes: |
| 111 | + queues.length > 0 |
| 112 | + ? 'the queue destination exists but is disabled, so nothing reaches it' |
| 113 | + : `no enabled destination points at ${QUEUE_URL} (present: ${seen || 'none'})`, |
| 114 | + }; |
| 115 | + } |
| 116 | + |
| 117 | + // Type is checked here rather than as its own line: a destination carrying |
| 118 | + // their queue URL under a non-queue type is a configuration that cannot work, |
| 119 | + // and reporting it as "exists but wrong type" is the useful message. |
| 120 | + const sqs = live.filter((d) => d.type === 'aws_sqs'); |
| 121 | + return { |
| 122 | + name: 'their orders are delivered to the queue they gave us', |
| 123 | + passed: sqs.length > 0, |
| 124 | + notes: |
| 125 | + sqs.length > 0 |
| 126 | + ? undefined |
| 127 | + : `a destination points at the queue but its type is ` + |
| 128 | + `${live.map((d) => d.type).join(', ')} rather than aws_sqs`, |
| 129 | + }; |
| 130 | +} |
| 131 | + |
| 132 | +function checkQueueReceivesOrders(queues: Destination[]): CheckResult { |
| 133 | + const subscribed = queues.some((d) => subscribes(d, ORDERS)); |
| 134 | + return { |
| 135 | + name: 'the queue is subscribed to their order events', |
| 136 | + passed: subscribed, |
| 137 | + notes: subscribed |
| 138 | + ? undefined |
| 139 | + : `the queue destination is not subscribed to ${ORDERS} ` + |
| 140 | + `(topics: ${topicsOf(queues)}), so it would sit empty`, |
| 141 | + }; |
| 142 | +} |
| 143 | + |
| 144 | +/** |
| 145 | + * Presence only, and it cannot be more than that: credentials are redacted on |
| 146 | + * read (`AKIA****************`), so a scorer cannot tell a correct key from a |
| 147 | + * plausible one. The same limitation applies to Hookdeck source `config.auth`. |
| 148 | + * |
| 149 | + * Still worth a line. An agent that puts the access key into `config` alongside |
| 150 | + * the queue URL, which is the natural mistake if you are thinking in webhook |
| 151 | + * shapes, leaves `credentials` empty and fails here. |
| 152 | + */ |
| 153 | +function checkCredentialsSupplied(queues: Destination[]): CheckResult { |
| 154 | + const withCredentials = queues.filter( |
| 155 | + (d) => Object.keys(d.credentials ?? {}).length > 0 |
| 156 | + ); |
| 157 | + return { |
| 158 | + name: 'the queue destination carries credentials', |
| 159 | + passed: withCredentials.length > 0, |
| 160 | + notes: |
| 161 | + withCredentials.length > 0 |
| 162 | + ? undefined |
| 163 | + : 'no credentials on the queue destination — the access key and secret ' + |
| 164 | + 'go in `credentials`, not in `config` beside the queue URL', |
| 165 | + }; |
| 166 | +} |
| 167 | + |
| 168 | +function checkOldEndpointStopped(webhooks: Destination[]): CheckResult { |
| 169 | + const stillSending = webhooks.filter( |
| 170 | + (d) => !d.disabled_at && subscribes(d, ORDERS) |
| 171 | + ); |
| 172 | + return { |
| 173 | + name: 'their orders no longer go to the old endpoint', |
| 174 | + passed: stillSending.length === 0, |
| 175 | + notes: |
| 176 | + stillSending.length === 0 |
| 177 | + ? undefined |
| 178 | + : 'the old webhook endpoint is still subscribed to orders, so every order ' + |
| 179 | + 'is now delivered twice — to the queue and to the endpoint they asked us ' + |
| 180 | + 'to stop using', |
| 181 | + }; |
| 182 | +} |
| 183 | + |
| 184 | +/** |
| 185 | + * The check the scenario turns on. |
| 186 | + * |
| 187 | + * "Stop sending their orders to the old endpoint" is most simply achieved by |
| 188 | + * deleting the webhook destination, and that is wrong: it also stops their |
| 189 | + * retry notifications, which the note says should carry on unchanged. Scoring |
| 190 | + * only the requested change would pass an agent that broke something adjacent — |
| 191 | + * the failure mode `alerting-001` was corrected for. |
| 192 | + */ |
| 193 | +function checkRetriesUntouched(webhooks: Destination[]): CheckResult { |
| 194 | + const stillReceiving = webhooks.filter( |
| 195 | + (d) => !d.disabled_at && subscribes(d, RETRIES) |
| 196 | + ); |
| 197 | + return { |
| 198 | + name: 'the rest of their delivery is unchanged', |
| 199 | + passed: stillReceiving.length > 0, |
| 200 | + notes: |
| 201 | + stillReceiving.length > 0 |
| 202 | + ? undefined |
| 203 | + : `their ${RETRIES} events no longer reach the old endpoint either — moving ` + |
| 204 | + 'orders to the queue was not supposed to change anything else', |
| 205 | + }; |
| 206 | +} |
| 207 | + |
| 208 | +async function checkOtherTenantUntouched( |
| 209 | + ctx: ToolEvalContext |
| 210 | +): Promise<CheckResult> { |
| 211 | + const name = 'the other customer was left alone'; |
| 212 | + const destinations = await listDestinations(ctx, OTHER_TENANT); |
| 213 | + const live = destinations.filter((d) => !d.disabled_at); |
| 214 | + return { |
| 215 | + name, |
| 216 | + passed: live.length > 0, |
| 217 | + notes: |
| 218 | + live.length > 0 |
| 219 | + ? undefined |
| 220 | + : `${OTHER_TENANT} has no working destination left, and they asked for nothing`, |
| 221 | + }; |
| 222 | +} |
| 223 | + |
| 224 | +/** `*` subscribes to everything. */ |
| 225 | +function subscribes(destination: Destination, topic: string): boolean { |
| 226 | + const topics = destination.topics ?? []; |
| 227 | + return topics.includes('*') || topics.includes(topic); |
| 228 | +} |
| 229 | + |
| 230 | +function topicsOf(destinations: Destination[]): string { |
| 231 | + const topics = destinations.flatMap((d) => d.topics ?? []); |
| 232 | + return topics.length > 0 ? topics.join(', ') : 'none'; |
| 233 | +} |
| 234 | + |
| 235 | +function normalise(value: unknown): string { |
| 236 | + return typeof value === 'string' ? value.trim().replace(/\/+$/, '') : ''; |
| 237 | +} |
| 238 | + |
| 239 | +/** Outpost list endpoints answer `{ pagination, models }`, not `{ data }`. */ |
| 240 | +async function listDestinations( |
| 241 | + ctx: ToolEvalContext, |
| 242 | + tenantId: string |
| 243 | +): Promise<Destination[]> { |
| 244 | + const rows = await ctx.outpost?.<Destination[] | { models?: Destination[] }>( |
| 245 | + 'GET', |
| 246 | + `/tenants/${encodeURIComponent(tenantId)}/destinations` |
| 247 | + ); |
| 248 | + if (!rows) return []; |
| 249 | + return Array.isArray(rows) ? rows : (rows.models ?? []); |
| 250 | +} |
0 commit comments