fix(eval): harden GEPA discovery and reuse

.claude/skills/noetic-eval/SKILL.md

@@ -169,14 +169,15 @@ noetic-eval --watch                # Re-run on changes
 
 noetic-eval -u                     # Optimize (GEPA)
 noetic-eval -u --scope full        # Full program optimization
-noetic-eval -u --budget 10         # Cost cap
+noetic-eval --concurrency 4       # Bound cases running per suite
 noetic-eval -u --dry-run           # Preview without writing
+noetic-eval -u --force-dirty       # Override dirty-file write guard
 
 noetic-eval --save-baseline        # Save scores as regression baseline
 noetic-eval --check                # Fail if scores regress or baseline cases vanish
 ```
 
-Exit codes: `0` all cases passed (clean `--check`; no-baseline `--check` prints a notice and passes), `1` any failed/errored case, regression, missing baseline case, or unresolvable explicit file pattern, `2` usage error (unknown flag, invalid `--scope`/`--budget` value — never silently dropped).
+Exit codes: `0` all cases passed (clean `--check`; no-baseline `--check` prints a notice and passes), `1` any failed/errored case, regression, missing baseline case, or unresolvable explicit file pattern, `2` usage error (unknown flag or invalid `--scope`/`--concurrency` value — never silently dropped).
 
 Watch mode spawns a fresh subprocess per run (in-process re-import is module-cached and would never re-register suites), serializes runs, coalesces changes during a run into one follow-up, and always exits `0` itself. Only the eval files are watched, not transitive imports.
 

.claude/skills/noetic-eval/references/api-reference.md

@@ -187,8 +187,8 @@ interface OptimizeOptions {
   scope: 'prompts-only' | 'flow-structure' | 'full';
   runEval: (step: Step) => Promise<Record<string, number>>;
   maxMetricCalls?: number;
-  budget?: number;
   dryRun?: boolean;
+  forceDirty?: boolean;
   codingAgent?: CodingAgent;
   preEnrichedFields?: OptimizableField[];  // AST-enriched fields with source locations
   gepa?: GepaConfig;

packages/core/src/interpreter/execute-action.ts

@@ -111,7 +111,7 @@ export async function executeRunCode<TContext, I, O>(
       lastError = e instanceof Error ? e : new Error(String(e));
 
       if (attempt < maxAttempts - 1 && retry) {
-        const delay = computeDelay(retry, attempt);
+        const delay = computeRetryDelay(retry, attempt);
         await new Promise((r) => setTimeout(r, delay));
       }
     }
@@ -125,7 +125,8 @@ export async function executeRunCode<TContext, I, O>(
   });
 }
 
-function computeDelay(retry: RetryPolicy, attempt: number): number {
+/** @internal Pure retry delay calculation for deterministic tests. */
+export function computeRetryDelay(retry: RetryPolicy, attempt: number): number {
   let delay: number;
   switch (retry.backoff) {
     case 'fixed':

packages/core/test/interpreter/execute-run.test.ts

@@ -1,46 +1,14 @@
-import { afterEach, describe, expect, it } from 'bun:test';
+import { describe, expect, it } from 'bun:test';
 import assert from 'node:assert';
 import type { ContextData } from '@noetic-tools/context';
-import type { Context, StepRunCode } from '@noetic-tools/types';
+import type { Context, RetryPolicy, StepRunCode } from '@noetic-tools/types';
 import { isNoeticError, NoeticErrorImpl } from '@noetic-tools/types';
-import { executeRunCode } from '../../src/interpreter/execute-action';
+import { computeRetryDelay, executeRunCode } from '../../src/interpreter/execute-action';
 import { ContextImpl } from '../../src/runtime/context-impl';
 import { makeMockContext, makeMockHarness } from '../_helpers';
 
 const mockCtx: Context = makeMockContext();
 
-// Safety net: ensure setTimeout is always restored
-const _originalSetTimeout = globalThis.setTimeout;
-afterEach(() => {
-  globalThis.setTimeout = _originalSetTimeout;
-});
-
-/** Patch setTimeout to capture delay values and execute callbacks instantly. */
-function interceptDelays(): {
-  delays: number[];
-  restore: () => void;
-} {
-  const delays: number[] = [];
-  // Wrap the original to intercept delay values while preserving the full overloaded signature
-  const handler: ProxyHandler<typeof setTimeout> = {
-    apply(_target, thisArg, argsList: unknown[]) {
-      const delay = argsList[1];
-      if (typeof delay === 'number' && delay > 0) {
-        delays.push(delay);
-      }
-      argsList[1] = 1;
-      return Reflect.apply(_originalSetTimeout, thisArg, argsList);
-    },
-  };
-  globalThis.setTimeout = new Proxy(_originalSetTimeout, handler);
-  return {
-    delays,
-    restore: () => {
-      globalThis.setTimeout = _originalSetTimeout;
-    },
-  };
-}
-
 describe('executeRunCode', () => {
   it('calls execute function and returns output', async () => {
     const s: StepRunCode<ContextData, string, number> = {
@@ -88,177 +56,127 @@ describe('executeRunCode', () => {
   });
 
   it('retries with fixed backoff', async () => {
-    const { delays, restore } = interceptDelays();
-
     let attempts = 0;
-    const s: StepRunCode<ContextData, string, string> = {
+    const retry: RetryPolicy = {
+      maxAttempts: 3,
+      backoff: 'fixed',
+      initialDelay: 1,
+    };
+    const step: StepRunCode<ContextData, string, string> = {
       kind: 'runCode',
       id: 'retry-test',
-      execute: async (_input) => {
+      retry,
+      execute: async () => {
         attempts++;
         if (attempts < 3) {
           throw new Error('not yet');
         }
         return 'success';
       },
-      retry: {
-        maxAttempts: 3,
-        backoff: 'fixed',
-        initialDelay: 10,
-      },
     };
-    try {
-      const result = await executeRunCode(s, 'test', mockCtx);
-      expect(result).toBe('success');
-      expect(attempts).toBe(3);
-      // Fixed backoff: all delays should be 10
-      expect(delays).toEqual([
-        10,
-        10,
-      ]);
-    } finally {
-      restore();
-    }
+    expect(await executeRunCode(step, 'test', mockCtx)).toBe('success');
+    expect(attempts).toBe(3);
+    expect(
+      [
+        0,
+        1,
+      ].map((attempt) => computeRetryDelay(retry, attempt)),
+    ).toEqual([
+      1,
+      1,
+    ]);
   });
 
   it('retries with exponential backoff and exhausts', async () => {
-    const { delays, restore } = interceptDelays();
     let attempts = 0;
-    const s: StepRunCode<ContextData, string, string> = {
+    const retry: RetryPolicy = {
+      maxAttempts: 3,
+      backoff: 'exponential',
+      initialDelay: 1,
+    };
+    const step: StepRunCode<ContextData, string, string> = {
       kind: 'runCode',
       id: 'exhaust-test',
+      retry,
       execute: async () => {
         attempts++;
         throw new Error('always fails');
       },
-      retry: {
-        maxAttempts: 3,
-        backoff: 'exponential',
-        initialDelay: 10,
-      },
     };
-    try {
-      await executeRunCode(s, 'test', mockCtx);
-      expect.unreachable('should have thrown');
-    } catch (e) {
-      assert(isNoeticError(e));
-      const oe = e.noeticError;
-      assert(oe.kind === 'step_failed');
-      expect(oe.retriesExhausted).toBe(true);
-      expect(attempts).toBe(3);
-      expect(delays).toEqual([
-        10,
-        20,
-      ]);
-    } finally {
-      restore();
-    }
+    await expect(executeRunCode(step, 'test', mockCtx)).rejects.toThrow('always fails');
+    expect(attempts).toBe(3);
+    expect(
+      [
+        0,
+        1,
+      ].map((attempt) => computeRetryDelay(retry, attempt)),
+    ).toEqual([
+      1,
+      2,
+    ]);
   });
 
-  it('caps exponential backoff delay at maxDelay', async () => {
-    const { delays, restore } = interceptDelays();
-
-    let attempts = 0;
-    const s: StepRunCode<ContextData, string, string> = {
-      kind: 'runCode',
-      id: 'cap-test',
-      execute: async () => {
-        attempts++;
-        if (attempts < 5) {
-          throw new Error('fail');
-        }
-        return 'ok';
-      },
-      retry: {
-        maxAttempts: 5,
-        backoff: 'exponential',
-        initialDelay: 100,
-        maxDelay: 500,
-      },
+  it('caps exponential backoff delay at maxDelay', () => {
+    const retry: RetryPolicy = {
+      maxAttempts: 5,
+      backoff: 'exponential',
+      initialDelay: 100,
+      maxDelay: 500,
     };
-    try {
-      await executeRunCode(s, 'test', mockCtx);
-    } finally {
-      restore();
-    }
-    // Delays: 100, 200, 400, 500 (capped from 800)
-    for (const d of delays) {
-      expect(d).toBeLessThanOrEqual(500);
-    }
-    expect(delays.length).toBeGreaterThan(0);
-    expect(delays).toEqual([
+    expect(
+      [
+        0,
+        1,
+        2,
+        3,
+      ].map((attempt) => computeRetryDelay(retry, attempt)),
+    ).toEqual([
       100,
       200,
       400,
       500,
     ]);
   });
 
-  it('defaults maxDelay to 30000', async () => {
-    // With exponential backoff, delay = 100 * 2^attempt
-    // For attempt 9: 100 * 512 = 51200, should be capped at 30000
-    const { delays, restore } = interceptDelays();
-
-    let attempts = 0;
-    const s: StepRunCode<ContextData, string, string> = {
-      kind: 'runCode',
-      id: 'default-cap-test',
-      execute: async () => {
-        attempts++;
-        if (attempts < 11) {
-          throw new Error('fail');
-        }
-        return 'ok';
-      },
-      retry: {
-        maxAttempts: 11,
-        backoff: 'exponential',
-        initialDelay: 100,
-      },
+  it('defaults maxDelay to 30000', () => {
+    const retry: RetryPolicy = {
+      maxAttempts: 11,
+      backoff: 'exponential',
+      initialDelay: 100,
     };
-    try {
-      await executeRunCode(s, 'test', mockCtx);
-    } finally {
-      restore();
-    }
-    for (const d of delays) {
-      expect(d).toBeLessThanOrEqual(30_000);
-    }
-    expect(delays.length).toBeGreaterThan(0);
+    expect(computeRetryDelay(retry, 9)).toBe(30_000);
   });
 
   it('retries with linear backoff', async () => {
-    const { delays, restore } = interceptDelays();
-
     let attempts = 0;
-    const s: StepRunCode<ContextData, string, string> = {
+    const retry: RetryPolicy = {
+      maxAttempts: 3,
+      backoff: 'linear',
+      initialDelay: 1,
+    };
+    const step: StepRunCode<ContextData, string, string> = {
       kind: 'runCode',
       id: 'linear-test',
+      retry,
       execute: async () => {
         attempts++;
         if (attempts < 3) {
           throw new Error('not yet');
         }
         return 'ok';
       },
-      retry: {
-        maxAttempts: 3,
-        backoff: 'linear',
-        initialDelay: 10,
-      },
     };
-    try {
-      const result = await executeRunCode(s, 'test', mockCtx);
-      expect(result).toBe('ok');
-      expect(attempts).toBe(3);
-      // Linear backoff: delay = initialDelay * attempt
-      expect(delays).toEqual([
-        10,
-        20,
-      ]);
-    } finally {
-      restore();
-    }
+    expect(await executeRunCode(step, 'test', mockCtx)).toBe('ok');
+    expect(attempts).toBe(3);
+    expect(
+      [
+        0,
+        1,
+      ].map((attempt) => computeRetryDelay(retry, attempt)),
+    ).toEqual([
+      1,
+      2,
+    ]);
   });
 
   describe('cancellation (not retriable)', () => {

packages/eval/README.md

@@ -76,7 +76,9 @@ noetic-eval --watch             # re-run on change
 noetic-eval --json              # machine-readable results
 noetic-eval --save-baseline     # record current scores
 noetic-eval --check             # fail if scores regress from the baseline
-noetic-eval --scope <scope> --budget <n>   # optimization run
+noetic-eval -u --scope <scope>            # optimization run
+noetic-eval --concurrency <n>             # bound cases running per suite
+noetic-eval -u --force-dirty              # override dirty-file write guard
 ```
 
 ## License