fix: address 22 user-facing frictions

bin/schrute.cjs

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+'use strict';
+const major = parseInt(process.version.slice(1), 10);
+if (major < 22) {
+  console.error(`Error: Node >= 22 required (found ${process.version}). Run: nvm use 22`);
+  process.exit(1);
+}
+import('../dist/index.js');

package.json

@@ -15,10 +15,10 @@
     }
   },
   "bin": {
-    "schrute": "dist/index.js"
+    "schrute": "bin/schrute.cjs"
   },
   "scripts": {
-    "prebuild": "node scripts/sync-version.js",
+    "prebuild": "node -e \"if(parseInt(process.version.slice(1))<22){console.error('Node>=22 required');process.exit(1)}\" && node scripts/sync-version.js",
     "build": "tsc -p tsconfig.json",
     "prepublishOnly": "node scripts/sync-version.js && npm run build",
     "build:native": "cd native && cargo build --release && cp target/release/libschrute_native.dylib index.node 2>/dev/null; cp target/release/schrute_native.dll index.node 2>/dev/null; cp target/release/libschrute_native.so index.node 2>/dev/null; echo 'Native build complete'",
@@ -27,10 +27,10 @@
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
     "lint": "tsc --noEmit",
-    "start": "node dist/index.js",
-    "serve": "node dist/index.js serve",
-    "setup": "node dist/index.js setup",
-    "doctor": "node dist/index.js doctor",
+    "start": "node bin/schrute.cjs",
+    "serve": "node bin/schrute.cjs serve",
+    "setup": "node bin/schrute.cjs setup",
+    "doctor": "node bin/schrute.cjs doctor",
     "rebuild:native": "bash scripts/rebuild-native.sh",
     "build:binary": "npm run build && pkg dist/index.js --config pkg.config.json",
     "build:binary:macos": "npm run build:binary -- --target node22-macos-arm64",
@@ -39,11 +39,13 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.12.1",
-    "better-sqlite3": "^11.8.1",
+    "better-sqlite3": "^12.8.0",
+    "cheerio": "^1.2.0",
     "commander": "^13.1.0",
     "fastify": "^5.2.1",
     "ipaddr.js": "^2.2.0",
     "js-yaml": "^4.1.1",
+    "jsonpath-plus": "^10.4.0",
     "keytar": "^7.9.0",
     "patchright": "^1.57.0",
     "pdf-parse": "^2.4.5",
@@ -91,6 +93,7 @@
     "schrute"
   ],
   "files": [
+    "bin/",
     "dist/",
     ".claude-plugin/",
     "commands/",

src/app/import-service.ts

@@ -0,0 +1,261 @@
+import * as fs from 'node:fs';
+import * as readline from 'node:readline';
+import type { SkillSpec, SiteManifest, SitePolicy } from '../skill/types.js';
+import {
+  validateImportableSkill,
+  validateImportableSite,
+  validateAndNormalizeImportablePolicy,
+} from '../storage/import-validator.js';
+import { getSitePolicy, setSitePolicy } from '../core/policy.js';
+import type { SkillRepository } from '../storage/skill-repository.js';
+import type { SiteRepository } from '../storage/site-repository.js';
+import type { AgentDatabase } from '../storage/database.js';
+import type { SchruteConfig } from '../core/config.js';
+
+export interface ImportDeps {
+  db: AgentDatabase;
+  skillRepo: SkillRepository;
+  siteRepo: SiteRepository;
+  config: SchruteConfig;
+}
+
+export interface ImportOptions {
+  yes?: boolean;
+}
+
+export interface ImportResult {
+  created: number;
+  updated: number;
+  skipped: number;
+  siteAction?: 'created' | 'updated';
+  hasAuthSkills: boolean;
+  policyWarnings: string[];
+  cancelled?: boolean;
+}
+
+export async function performImport(
+  file: string,
+  deps: ImportDeps,
+  options: ImportOptions = {},
+): Promise<ImportResult> {
+  if (!fs.existsSync(file)) {
+    throw new Error(`File '${file}' not found.`);
+  }
+
+  let bundle: {
+    version: string;
+    site: SiteManifest;
+    skills: SkillSpec[];
+    policy?: SitePolicy;
+  };
+
+  try {
+    const raw = fs.readFileSync(file, 'utf-8');
+    bundle = JSON.parse(raw);
+  } catch (err) {
+    throw new Error(`Failed to parse bundle: ${err instanceof Error ? err.message : String(err)}`);
+  }
+
+  if (!bundle.site || !bundle.skills || !Array.isArray(bundle.skills)) {
+    throw new Error('Invalid bundle format: missing site or skills.');
+  }
+
+  // Validate site
+  const siteResult = validateImportableSite(bundle.site);
+  if (!siteResult.valid) {
+    throw new Error(`Site validation failed:\n  ${siteResult.errors.join('\n  ')}`);
+  }
+
+  // Validate each skill; warn + skip invalid ones
+  const validSkills: SkillSpec[] = [];
+  const skipped: string[] = [];
+  const expectedSiteId = bundle.site.id;
+
+  for (const skill of bundle.skills) {
+    const skillResult = validateImportableSkill(skill);
+    if (!skillResult.valid) {
+      const label = (skill as unknown as Record<string, unknown>).id ?? '(unknown)';
+      console.warn(
+        `Warning: skill '${label}' failed validation -- skipping.\n  ${skillResult.errors.join('\n  ')}`,
+      );
+      skipped.push(String(label));
+      continue;
+    }
+
+    if (Array.isArray(skill.allowedDomains) && skill.allowedDomains.length === 0) {
+      console.warn(
+        `Warning: skill '${skill.id}' has no allowedDomains -- may not execute without a domain policy.`,
+      );
+    }
+
+    if (skill.siteId !== expectedSiteId) {
+      console.warn(
+        `Warning: skill '${skill.id}' has siteId '${skill.siteId}', expected '${expectedSiteId}'. Skipping.`,
+      );
+      skipped.push(skill.id);
+      continue;
+    }
+
+    validSkills.push(skill);
+  }
+
+  // Check for overwrites — track corrupt rows separately
+  const { db, skillRepo, siteRepo } = deps;
+  let existingSite: SiteManifest | undefined;
+  let siteCorrupt = false;
+  try {
+    existingSite = siteRepo.getById(bundle.site.id);
+  } catch {
+    siteCorrupt = true;
+    console.warn(`Warning: existing site '${bundle.site.id}' has corrupt data — will overwrite.`);
+  }
+
+  const overwriteIds: string[] = [];
+  const corruptIds: string[] = [];
+  const existingCreatedAt = new Map<string, number>();
+  let newCount = 0;
+  for (const skill of validSkills) {
+    try {
+      const existing = skillRepo.getById(skill.id);
+      if (existing) {
+        overwriteIds.push(skill.id);
+        if (existing.createdAt) existingCreatedAt.set(skill.id, existing.createdAt);
+      } else {
+        newCount++;
+      }
+    } catch {
+      corruptIds.push(skill.id);
+      console.warn(`Warning: existing skill '${skill.id}' has corrupt data — will overwrite.`);
+    }
+  }
+  const existingCount = overwriteIds.length + corruptIds.length;
+
+  // Preview
+  console.log(`Import preview for '${file}':`);
+  console.log(`  Site:             ${bundle.site.id} (${existingSite ? 'will update' : 'will create'})`);
+  console.log(`  Valid skills:     ${validSkills.length}`);
+  if (skipped.length > 0) {
+    console.log(`  Skipped (invalid): ${skipped.length}`);
+  }
+  if (existingCount > 0) {
+    console.log(`  Will overwrite:   ${existingCount} existing skill(s)`);
+    for (const id of overwriteIds) console.log(`    overwrite: ${id}`);
+    for (const id of corruptIds) console.log(`    overwrite (corrupt): ${id}`);
+  }
+
+  // Policy preview
+  const policyWarnings: string[] = [];
+  if (bundle.policy) {
+    console.log(`  Policy:           will ${existingSite ? 'replace' : 'set'}`);
+    const currentPolicy = getSitePolicy(bundle.site.id, deps.config);
+    if (bundle.policy.maxConcurrent !== currentPolicy.maxConcurrent) {
+      policyWarnings.push(`maxConcurrent: current=${currentPolicy.maxConcurrent}, import=${bundle.policy.maxConcurrent}`);
+    }
+  }
+  if (policyWarnings.length > 0) {
+    console.log(`  Policy changes:   ${policyWarnings.join('; ')}`);
+  }
+
+  // Confirmation — require when anything will be overwritten
+  if ((existingCount > 0 || existingSite || siteCorrupt) && !options.yes) {
+    if (!process.stdin.isTTY) {
+      throw new Error('Non-interactive terminal: use --yes to confirm import.');
+    }
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    const answer = await new Promise<string>(resolve => rl.question('Proceed with import? [y/N] ', resolve));
+    rl.close();
+    if (answer.toLowerCase() !== 'y') {
+      return { created: 0, updated: 0, skipped: skipped.length, hasAuthSkills: false, policyWarnings, cancelled: true };
+    }
+  }
+
+  // Fill defaults for NOT NULL DB fields
+  const now = Date.now();
+  for (const skill of validSkills) {
+    if (!skill.name) {
+      const parts = skill.id.split('.');
+      skill.name = parts.length >= 2 ? parts[parts.length - 2] : skill.id;
+    }
+    if (skill.inputSchema === undefined) skill.inputSchema = {};
+    if (skill.sideEffectClass === undefined) skill.sideEffectClass = 'read-only';
+    if (skill.currentTier === undefined) skill.currentTier = 'tier_3';
+    if (skill.status === undefined) skill.status = 'draft';
+    if (skill.confidence === undefined) skill.confidence = 0;
+    if (skill.consecutiveValidations === undefined) skill.consecutiveValidations = 0;
+    if (skill.sampleCount === undefined) skill.sampleCount = 0;
+    if (skill.successRate === undefined) skill.successRate = 0;
+    if (skill.version === undefined) skill.version = 1;
+    if (skill.allowedDomains === undefined) skill.allowedDomains = [];
+    if (skill.isComposite === undefined) skill.isComposite = false;
+    if (skill.directCanaryEligible === undefined) skill.directCanaryEligible = false;
+    if (skill.directCanaryAttempts === undefined) skill.directCanaryAttempts = 0;
+    if (skill.validationsSinceLastCanary === undefined) skill.validationsSinceLastCanary = 0;
+    if (skill.createdAt === undefined) {
+      skill.createdAt = existingCreatedAt.get(skill.id) ?? now;
+    }
+    if (skill.updatedAt === undefined) skill.updatedAt = now;
+  }
+
+  // Phase 1: Site + skills in a single synchronous transaction
+  const corruptSet = new Set(corruptIds);
+  const overwriteSet = new Set(overwriteIds);
+  let created = 0;
+  let updated = 0;
+  let siteAction: 'created' | 'updated';
+
+  db.transaction(() => {
+    if (existingSite && !siteCorrupt) {
+      siteRepo.update(bundle.site.id, bundle.site);
+      siteAction = 'updated';
+    } else {
+      // Delete corrupt/stale row (cascade may delete skills too)
+      try { siteRepo.delete(bundle.site.id); } catch { /* row may not exist */ }
+      siteRepo.create(bundle.site);
+      siteAction = 'created';
+    }
+
+    if (siteCorrupt) {
+      // Site was deleted+recreated → cascade killed all skills → all are creates
+      for (const skill of validSkills) {
+        skillRepo.create(skill);
+        created++;
+      }
+    } else {
+      for (const skill of validSkills) {
+        if (corruptSet.has(skill.id)) {
+          try { skillRepo.delete(skill.id); } catch { /* may already be gone */ }
+          skillRepo.create(skill);
+          updated++;
+        } else if (overwriteSet.has(skill.id)) {
+          skillRepo.update(skill.id, skill);
+          updated++;
+        } else {
+          skillRepo.create(skill);
+          created++;
+        }
+      }
+    }
+  });
+
+  // Phase 2: Policy (separate write — setSitePolicy does its own DB call)
+  if (bundle.policy) {
+    const policyResult = validateAndNormalizeImportablePolicy(bundle.policy, bundle.site.id);
+    if (!policyResult.valid || !policyResult.value) {
+      console.error(`Warning: policy import failed validation: ${policyResult.errors.join('; ')}`);
+    } else {
+      const p = policyResult.value;
+      try {
+        const result = setSitePolicy(p, deps.config);
+        if (!result.persisted) {
+          console.error('Warning: policy imported to cache but failed to persist to DB.');
+        }
+      } catch (err) {
+        console.error(`Warning: policy import failed: ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+  }
+
+  const hasAuthSkills = validSkills.some((s: SkillSpec) => s.authType != null);
+
+  return { created, updated, skipped: skipped.length, siteAction: siteAction!, hasAuthSkills, policyWarnings };
+}

src/app/service.ts

@@ -14,6 +14,7 @@ import { shouldAutoConfirm } from '../server/skill-helpers.js';
 
 type ExecuteSkillResult =
   | { status: 'executed'; result: SkillExecutionResult }
+  | { status: 'browser_handoff_required'; result: SkillExecutionResult }
   | {
       status: 'confirmation_required';
       skillId: string;
@@ -70,6 +71,8 @@ interface ExportedCookie {
 export interface PipelineJobResult {
   skillsGenerated: number;
   signalCount: number;
+  htmlDocumentCount?: number;
+  ambiguousCount?: number;
   noiseCount: number;
   totalCount: number;
   warning?: string;
@@ -185,6 +188,9 @@ export class SchruteService {
     }
 
     const result = await this.deps.engine.executeSkill(skillId, params, callerId);
+    if (result.status === 'browser_handoff_required') {
+      return { status: 'browser_handoff_required', result };
+    }
     return { status: 'executed', result };
   }
 
@@ -263,7 +269,7 @@ export class SchruteService {
   listSessions(): SessionInfo[] {
     const msm = this.deps.engine.getMultiSessionManager();
     const activeName = msm.getActive();
-    return msm.list().map(s => ({
+    return msm.list(undefined, this.deps.config, { includeInternal: false }).map(s => ({
       name: s.name,
       siteId: s.siteId,
       isCdp: s.isCdp,

src/automation/classifier.ts

@@ -98,8 +98,9 @@ export function classifySite(
     // JS-computed fields require full browser for replay
     recommendedTier = ExecutionTier.FULL_BROWSER;
   } else if (authRequired) {
-    // Auth (with or without dynamic fields) typically needs cookie refresh tier
-    recommendedTier = ExecutionTier.COOKIE_REFRESH;
+    // Auth-required traffic should stay on the browser-backed path until
+    // a real persistent cookie-refresh tier exists.
+    recommendedTier = ExecutionTier.BROWSER_PROXIED;
   } else if (graphqlDetected) {
     // GraphQL without auth — direct fetch may work
     recommendedTier = ExecutionTier.DIRECT;