Use the native npm Computer Use bundle with unified macOS permission onboarding

.github/workflows/computer-use-native.yml

@@ -10,10 +10,6 @@ on:
       - "scripts/internal/shared/check-open-computer-use.mjs"
       - "scripts/internal/linux/**"
       - "scripts/internal/mac/**"
-      - "scripts/vendor/open-computer-use/**"
-      - "tests/ocu-native-reconnect.test.mjs"
-      - "tests/ocu-native-source.test.mjs"
-      - "tests/open-computer-use/**"
       - "tests/ocu-dev-install.test.mjs"
       - "scripts/internal/win/build-nsis.sh"
   workflow_dispatch:
@@ -66,16 +62,11 @@ jobs:
       - name: Check Windows MCP and desktop backend
         if: runner.os == 'Windows'
         run: node scripts/internal/shared/check-open-computer-use.mjs "$OCU_BINARY" --list-apps | tee ocu-native-result.json
-      - uses: maxim-lobanov/setup-xcode@v1
-        if: runner.os == 'macOS'
-        with:
-          xcode-version: latest-stable
-      - name: Build and test the patched macOS runtime
+      - name: Validate the published macOS bundle and installer
         if: runner.os == 'macOS'
         run: |
-          node --test tests/ocu-native-source.test.mjs
-          MEMMY_OCU_RUN_TESTS=1 node scripts/internal/mac/build-computer-use.mjs "$RUNNER_TEMP/ocu-native/node_modules/open-computer-use"
-          OCU_TEST_BINARY="$OCU_BINARY" node --test tests/ocu-native-reconnect.test.mjs
+          codesign --verify --deep --strict "$RUNNER_TEMP/ocu-native/node_modules/open-computer-use/dist/Open Computer Use.app"
+          lipo "$OCU_BINARY" -verify_arch arm64 x86_64
           node --test tests/ocu-dev-install.test.mjs
       - name: Check macOS MCP discovery
         if: runner.os == 'macOS'

.gitignore

@@ -2,6 +2,7 @@ node_modules/
 dist/
 release/
 App/shell/desktop/release/
+Mac软件打包/
 memmy-memory-*.tgz
 .vite/
 *.log

App/backend/local-api-contracts/src/computer-use-onboarding.ts

@@ -0,0 +1,41 @@
+export const COMPUTER_USE_ONBOARDING_PREFIX = 'memmy:computer-use-onboarding:';
+export type ComputerUseGuideReason = 'accessibility' | 'screenCaptureUnavailable';
+export type ComputerUseProbeTarget = { app: string; pid: number };
+export type ComputerUsePermissionState = 'granted' | 'required' | 'unknown';
+export type ComputerUsePermissions = {
+  accessibility: ComputerUsePermissionState;
+  screenRecording: ComputerUsePermissionState;
+  failure?: 'unavailable' | 'helperPauseFailed';
+};
+export function isComputerUsePermissions(value: any): value is ComputerUsePermissions {
+  return !!value && typeof value === 'object' && !Array.isArray(value)
+    && Object.keys(value).every(key => ['accessibility', 'screenRecording', 'failure'].includes(key))
+    && ['granted', 'required', 'unknown'].includes(value.accessibility)
+    && ['granted', 'required', 'unknown'].includes(value.screenRecording)
+    && (value.failure === undefined || ['unavailable', 'helperPauseFailed'].includes(value.failure));
+}
+export type ComputerUseOnboardingRequest = {
+  type: 'memmy:computer-use-onboarding:prepare' | 'memmy:computer-use-onboarding:guide';
+  requestId: string;
+  reason?: ComputerUseGuideReason;
+  helperApp?: string;
+  canContinue?: boolean;
+};
+export function isComputerUseGuideReason(value: unknown): value is ComputerUseGuideReason {
+  return value === 'accessibility' || value === 'screenCaptureUnavailable';
+}
+export function isComputerUseOnboardingRequest(value: any): value is ComputerUseOnboardingRequest {
+  return !!value && typeof value === 'object' && !Array.isArray(value)
+    && typeof value.requestId === 'string' && /^[a-zA-Z0-9-]{1,80}$/.test(value.requestId)
+    && Object.keys(value).every(key => ['type', 'requestId', 'reason', 'helperApp', 'canContinue'].includes(key))
+    && ((value.type === `${COMPUTER_USE_ONBOARDING_PREFIX}prepare` && value.reason === undefined && value.helperApp === undefined && value.canContinue === undefined)
+      || (value.type === `${COMPUTER_USE_ONBOARDING_PREFIX}guide` && isComputerUseGuideReason(value.reason)
+        && (value.canContinue === undefined || typeof value.canContinue === 'boolean')
+        && typeof value.helperApp === 'string' && value.helperApp.startsWith('/') && value.helperApp.endsWith('/Open Computer Use.app')
+        && value.helperApp.length <= 4096 && !/[\r\n\0]/.test(value.helperApp)));
+}
+export function isComputerUseProbeTarget(value: any): value is ComputerUseProbeTarget {
+  return !!value && typeof value === 'object' && typeof value.app === 'string'
+    && /^[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)+$/.test(value.app) && value.app.length <= 200
+    && Number.isSafeInteger(value.pid) && value.pid > 0;
+}

App/backend/local-api-contracts/src/desktop-screen-capture.ts

@@ -0,0 +1,31 @@
+export const SCREEN_CAPTURE_PROTOCOL = 1;
+export const SCREEN_CAPTURE_MAX_BYTES = 900 * 1024;
+export const SCREEN_CAPTURE_MAX_EDGE = 1280;
+export const SCREEN_CAPTURE_PREFIX = 'memmy:screen-capture:';
+export type ScreenCaptureResult = {
+  ok: true; pngBase64: string; displayId: string;
+  bounds: { x: number; y: number; width: number; height: number };
+  width: number; height: number;
+} | { ok: false; code: 'permission_required' | 'unavailable' | 'capture_failed' | 'cancelled'; message: string };
+export type ScreenCaptureRequest = { type: 'memmy:screen-capture:request'; requestId: string; displayId?: string };
+export function isScreenCaptureMessage(value: unknown): value is Record<string, any> {
+  return !!value && typeof value === 'object' && !Array.isArray(value)
+    && typeof (value as any).type === 'string' && (value as any).type.startsWith(SCREEN_CAPTURE_PREFIX)
+    && typeof (value as any).requestId === 'string' && /^[A-Za-z0-9-]{1,80}$/.test((value as any).requestId);
+}
+export function isScreenCaptureRequest(value: unknown): value is ScreenCaptureRequest {
+  return isScreenCaptureMessage(value) && value.type === `${SCREEN_CAPTURE_PREFIX}request`
+    && (value.displayId === undefined || (typeof value.displayId === 'string' && /^\d{1,16}$/.test(value.displayId)))
+    && Object.keys(value).every(key => ['type', 'requestId', 'displayId'].includes(key));
+}
+export function isScreenCaptureResult(value: any): value is ScreenCaptureResult {
+  if (!value || typeof value !== 'object') return false;
+  if (value.ok === false) return ['permission_required', 'unavailable', 'capture_failed', 'cancelled'].includes(value.code)
+    && typeof value.message === 'string' && value.message.length <= 2000;
+  return value.ok === true && typeof value.pngBase64 === 'string'
+    && value.pngBase64.length <= Math.ceil(SCREEN_CAPTURE_MAX_BYTES / 3) * 4
+    && typeof value.displayId === 'string' && /^\d{1,16}$/.test(value.displayId)
+    && [value.width, value.height].every(n => Number.isInteger(n) && n > 0 && n <= SCREEN_CAPTURE_MAX_EDGE)
+    && !!value.bounds && ['x', 'y', 'width', 'height'].every(key => Number.isFinite(value.bounds[key]))
+    && value.bounds.width > 0 && value.bounds.height > 0;
+}

App/backend/local-api-contracts/src/index.ts

@@ -1555,3 +1555,5 @@ export const TokenQuotaEligibilitySchema = z.object({
     latestReviewNote: z.string().nullable()
 });
 export type TokenQuotaEligibility = z.infer<typeof TokenQuotaEligibilitySchema>;
+export * from './desktop-screen-capture.js';
+export * from './computer-use-onboarding.js';

App/backend/local-api-contracts/tests/desktop-screen-capture.test.ts

@@ -0,0 +1,16 @@
+import { expect, it } from 'vitest';
+import { isScreenCaptureRequest, isScreenCaptureResult } from '../src/desktop-screen-capture.js';
+it('accepts only bounded display requests, never paths or execution payloads', () => {
+  const request = { type: 'memmy:screen-capture:request', requestId: 'test-1', displayId: '7' };
+  expect(isScreenCaptureRequest(request)).toBe(true);
+  expect(isScreenCaptureRequest({ ...request, displayId: '../../file' })).toBe(false);
+  expect(isScreenCaptureRequest({ ...request, command: 'open' })).toBe(false);
+  expect(isScreenCaptureRequest({ ...request, requestId: 'x'.repeat(100) })).toBe(false);
+});
+it('rejects malformed or oversized responses', () => {
+  const result = { ok: true, pngBase64: 'png', displayId: '7', bounds: { x: 0, y: 0, width: 1920, height: 1080 }, width: 1280, height: 720 };
+  expect(isScreenCaptureResult(result)).toBe(true);
+  expect(isScreenCaptureResult({ ...result, pngBase64: 'x'.repeat(2 * 1024 * 1024) })).toBe(false);
+  expect(isScreenCaptureResult({ ...result, width: Infinity })).toBe(false);
+  expect(isScreenCaptureResult({ ...result, bounds: null })).toBeFalsy();
+});

App/backend/src/project-version.ts

@@ -1,2 +1,2 @@
 /** Generated from the root package.json by scripts/sync-project-version.mjs. */
-export const MEMMY_VERSION = "1.1.4";
+export const MEMMY_VERSION = "1.1.6";

App/memmy-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memmy-agent",
-  "version": "1.1.4",
+  "version": "1.1.6",
   "description": "TypeScript refactor of memmy's agent runtime.",
   "type": "module",
   "main": "./dist/index.js",

App/memmy-agent/src/core/agent-runtime/loop.ts

@@ -1,3 +1,4 @@
+import { initializeDesktopScreenCapture } from '../../tools/computer-use/desktop-screen-capture.js';
 import fs from "node:fs";
 import path from "node:path";
 import { randomUUID } from "node:crypto";
@@ -1129,6 +1130,7 @@ export class AgentLoop {
   }
 
   async initializeRuntimeTools(): Promise<void> {
+    await initializeDesktopScreenCapture();
     await this.connectMcp();
     await this.browserSessionManager.initialize();
     if (!this.browserRegistryInitialized) {
@@ -3916,6 +3918,7 @@ export class AgentLoop {
       ctx.session!,
       compactionOptions,
     );
+    await initializeDesktopScreenCapture();
     ctx.tools = this.createToolRegistry("turn", sessionWorkspace, {
       includeConnectedMcp: true,
       messageSendCallback: ctx.messageSendCallback,
@@ -4054,7 +4057,7 @@ export class AgentLoop {
       channel: ctx.msg.channel,
       chatId: ctx.msg.chatId,
       messageId: ctx.msg.metadata?.message_id ?? ctx.msg.metadata?.messageId,
-      metadata: ctx.msg.metadata,
+      metadata: { ...ctx.msg.metadata, ...(ctx.msg.internal ? { computerUseInteractive: false } : {}) },
       sessionKey: ctx.sessionKey,
       pendingQueue: ctx.pendingQueue,
       abortSignal: ctx.abortSignal,
@@ -4440,7 +4443,7 @@ export class AgentLoop {
       channel,
       chatId,
       messageId: msg.metadata?.message_id ?? msg.metadata?.messageId ?? null,
-      metadata: msg.metadata,
+      metadata: { ...msg.metadata, computerUseInteractive: false },
       sessionKey: key,
       pendingQueue,
       abortSignal,

App/memmy-agent/src/core/agent-runtime/runner.ts

@@ -1007,12 +1007,16 @@ export class AgentRunner {
     calls: ToolCallRequest[],
     externalLookupCounts: Record<string, number> = {},
     workspaceViolationCounts: Record<string, number> = {},
-  ): Promise<Array<{ call: any; result: any; event: Record<string, any>; error?: any }>> {
+  ): Promise<Array<{ call: any; result: any; event: Record<string, any>; error?: any; stopTurn?: string }>> {
     if (!spec.tools) return [];
     if (spec.abortSignal?.aborted) return [];
     const batches = this.partitionToolBatches(spec, calls);
-    const out: Array<{ call: any; result: any; event: Record<string, any>; error?: any }> = [];
+    const out: Array<{ call: any; result: any; event: Record<string, any>; error?: any; stopTurn?: string }> = [];
     for (const batch of batches) {
+      if (out.some(item => item.stopTurn)) {
+        out.push(...batch.map(call => ({ call, result: 'Not executed: this turn is waiting for user action.', event: { name: call.name, status: 'error', detail: 'turn stopped' } })));
+        continue;
+      }
       if (spec.concurrentTools && batch.length > 1) {
         out.push(...(await Promise.all(batch.map((call) => this.runTool(spec, call, externalLookupCounts, workspaceViolationCounts)))));
       } else {
@@ -1030,7 +1034,7 @@ export class AgentRunner {
     call: ToolCallRequest,
     externalLookupCounts: Record<string, number>,
     workspaceViolationCounts: Record<string, number>,
-  ): Promise<{ call: ToolCallRequest; result: any; event: Record<string, any>; error?: any }> {
+  ): Promise<{ call: ToolCallRequest; result: any; event: Record<string, any>; error?: any; stopTurn?: string }> {
     const hint = "\n\n[Analyze the error above and try a different approach.]";
     if (spec.abortSignal?.aborted) {
       const event = { name: call.name, status: "error", detail: "task cancelled" };
@@ -1088,6 +1092,7 @@ export class AgentRunner {
       );
     }
 
+    let stopTurn: string | undefined;
     try {
       let raw: any;
       const fileMutationOutcomes = new Map<string, FileMutationOutcome>();
@@ -1096,6 +1101,7 @@ export class AgentRunner {
         abortSignal: spec.abortSignal ?? null,
         toolName: call.name,
         callId: call.id ?? null,
+        stopTurn: (message) => { stopTurn ??= message; },
         reportFileMutation: (outcome) => {
           fileMutationOutcomes.set(path.resolve(outcome.path), {
             path: path.resolve(outcome.path),
@@ -1121,7 +1127,7 @@ export class AgentRunner {
         const handled = this.classifyViolation(raw, raw + hint, event, call, workspaceViolationCounts);
         if (handled) {
           await spec.hook?.afterToolCall(new AgentHookContext({ spec, toolCalls: [call], toolResults: [handled.result], toolEvents: [handled.event] }), call, handled.result);
-          return { call, ...handled };
+          return { call, ...handled, ...(stopTurn ? { stopTurn } : {}) };
         }
         error = spec.failOnToolError ? new Error(raw) : null;
         raw = raw + hint;
@@ -1141,8 +1147,9 @@ export class AgentRunner {
         );
       }
       const result = this.normalizeToolResult(spec, call.id, call.name, raw);
+      if (stopTurn) event.status = "error";
       await spec.hook?.afterToolCall(new AgentHookContext({ spec, toolCalls: [call], toolResults: [result], toolEvents: [event] }), call, result);
-      return { call, result, event, error };
+      return { call, result, event, error, ...(stopTurn ? { stopTurn } : {}) };
     } catch (error) {
       const message = isAbortError(error)
         ? "Error: task cancelled"
@@ -1155,9 +1162,9 @@ export class AgentRunner {
       }
       const event = { name: call.name, status: "error", detail: eventDetail("", message, 120) };
       const handled = this.classifyViolation(String((error as Error).message ?? error), message, event, call, workspaceViolationCounts);
-      if (handled) return { call, ...handled };
+      if (handled) return { call, ...handled, ...(stopTurn ? { stopTurn } : {}) };
       const result = this.normalizeToolResult(spec, call.id, call.name, message);
-      return { call, result, event, error: spec.failOnToolError ? error : null };
+      return { call, result, event, error: spec.failOnToolError ? error : null, ...(stopTurn ? { stopTurn } : {}) };
     }
   }
 
@@ -1643,6 +1650,18 @@ export class AgentRunner {
           await hook.afterIteration(context);
           break;
         }
+        const stopped = executed.find(item => item.stopTurn)?.stopTurn;
+        if (stopped) {
+          finalContent = stopped;
+          stopReason = 'completed';
+          AgentRunner.appendFinalMessage(messages, finalContent);
+          await this.emitCheckpoint(spec, { phase: 'toolsCompleted', iteration, model: spec.model,
+            assistantMessage: assistant, completedToolResults: completed, pendingToolCalls: [] });
+          context.finalContent = finalContent;
+          context.stopReason = stopReason;
+          await hook.afterIteration(context);
+          break;
+        }
         const fatal = executed.find((item) => item.error)?.error;
         if (fatal) {
           error = `Error: ${fatal.constructor?.name ?? "Error"}: ${fatal.message ?? fatal}`;

App/memmy-agent/src/core/agent-runtime/tools/base.ts

@@ -9,6 +9,8 @@ export type ToolExecutionContext = {
   abortSignal?: AbortSignal | null;
   toolName?: string;
   callId?: string | null;
+  /** Stop this run without asking the model to select another executor. */
+  stopTurn?: (message: string) => void;
   reportFileMutation?: (outcome: FileMutationOutcome) => void;
 };
 

App/memmy-agent/src/core/agent-runtime/tools/loader.ts

@@ -1,3 +1,4 @@
+import { DesktopScreenCaptureTool } from '../../../tools/computer-use/desktop-screen-capture.js';
 import fs from "node:fs";
 import path from "node:path";
 import { createRequire } from "node:module";
@@ -45,6 +46,7 @@ type ToolClass = (new (...args: any[]) => Tool) & {
 };
 
 const BUILTIN_TOOL_CLASSES: ToolClass[] = [
+  DesktopScreenCaptureTool,
   AgentSourceTool,
   ApplyPatchTool,
   ComputerHistoryTool,