Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions IOSMCPPreferences.h
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,14 @@
#define IOS_MCP_DEFAULT_PORT 8090
#define IOS_MCP_MIN_PORT 1024
#define IOS_MCP_MAX_PORT 65535
#define IOS_MCP_DEFAULT_MAX_CONCURRENT_SCREEN_TASKS 1
#define IOS_MCP_MIN_CONCURRENT_SCREEN_TASKS 1
#define IOS_MCP_MAX_CONCURRENT_SCREEN_TASKS 2
#define IOS_MCP_PREFERENCES_DOMAIN @"com.witchan.ios-mcp.preferences"
#define IOS_MCP_ENABLED_PREFERENCE_KEY @"enabled"
#define IOS_MCP_PORT_PREFERENCE_KEY @"port"
#define IOS_MCP_DEBUG_LOGGING_PREFERENCE_KEY @"debugLoggingEnabled"
#define IOS_MCP_MAX_CONCURRENT_SCREEN_TASKS_PREFERENCE_KEY @"maxConcurrentScreenTasks"
#define IOS_MCP_DARWIN_NOTIFICATION_START CFSTR("com.witchan.ios-mcp.control/start")
#define IOS_MCP_DARWIN_NOTIFICATION_STOP CFSTR("com.witchan.ios-mcp.control/stop")

Expand Down Expand Up @@ -50,6 +54,23 @@ static inline uint16_t IOSMCPConfiguredPort(void) {
return port;
}

static inline NSInteger IOSMCPConfiguredMaxConcurrentScreenTasks(void) {
NSInteger limit = IOS_MCP_DEFAULT_MAX_CONCURRENT_SCREEN_TASKS;
CFPreferencesAppSynchronize((__bridge CFStringRef)IOS_MCP_PREFERENCES_DOMAIN);
CFPropertyListRef value = CFPreferencesCopyAppValue(
(__bridge CFStringRef)IOS_MCP_MAX_CONCURRENT_SCREEN_TASKS_PREFERENCE_KEY,
(__bridge CFStringRef)IOS_MCP_PREFERENCES_DOMAIN);
if (value) {
id preference = (__bridge id)value;
if ([preference respondsToSelector:@selector(integerValue)]) {
limit = [preference integerValue];
}
CFRelease(value);
}
return MAX(IOS_MCP_MIN_CONCURRENT_SCREEN_TASKS,
MIN(limit, IOS_MCP_MAX_CONCURRENT_SCREEN_TASKS));
}

static inline NSString *IOSMCPCurrentLANIPAddress(void) {
struct ifaddrs *interfaces = NULL;
NSString *preferredAddress = nil;
Expand Down
55 changes: 50 additions & 5 deletions MCPServer.m
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#import "LogManager.h"
#import "OCRManager.h"
#import "MCPLogger.h"
#import "IOSMCPPreferences.h"
#import <UIKit/UIKit.h>
#import <sys/socket.h>
#import <netinet/in.h>
Expand Down Expand Up @@ -655,6 +656,7 @@ - (NSDictionary *)routeMCPRequest:(NSDictionary *)request;
- (NSDictionary *)handleInitialize:(id)reqId params:(NSDictionary *)params;
- (NSDictionary *)handleToolsList:(id)reqId;
- (NSDictionary *)handleToolsCall:(id)reqId params:(NSDictionary *)params;
- (NSDictionary *)performScreenTaskForRequest:(id)reqId task:(NSDictionary *(^)(void))task;
- (NSDictionary *)lockedScreenGuardResponseForTool:(NSString *)toolName reqId:(id)reqId;
- (NSDictionary *)executeButtonPress:(id)reqId button:(HIDButtonType)button args:(NSDictionary *)args label:(NSString *)label;
- (BOOL)pressButtonSynchronously:(HIDButtonType)button duration:(NSTimeInterval)duration timeout:(NSTimeInterval)timeout error:(NSString **)error;
Expand Down Expand Up @@ -725,6 +727,7 @@ @implementation MCPServer {
int _serverSocket;
dispatch_source_t _acceptSource;
dispatch_queue_t _clientQueue;
dispatch_semaphore_t _screenTaskSemaphore;
NSString *_sessionId;
NSString *_negotiatedProtocolVersion;
}
Expand All @@ -742,7 +745,12 @@ - (instancetype)init {
self = [super init];
if (self) {
_serverSocket = -1;
_clientQueue = dispatch_queue_create("com.witchan.ios-mcp.client", DISPATCH_QUEUE_CONCURRENT);
dispatch_queue_attr_t clientAttributes =
dispatch_queue_attr_make_with_autorelease_frequency(DISPATCH_QUEUE_CONCURRENT,
DISPATCH_AUTORELEASE_FREQUENCY_WORK_ITEM);
_clientQueue = dispatch_queue_create("com.witchan.ios-mcp.client", clientAttributes);
_screenTaskSemaphore =
dispatch_semaphore_create(IOSMCPConfiguredMaxConcurrentScreenTasks());
_sessionId = [[NSUUID UUID] UUIDString];
_negotiatedProtocolVersion = MCP_PROTOCOL_VERSION_LATEST;
}
Expand Down Expand Up @@ -808,7 +816,10 @@ - (void)startOnPort:(uint16_t)port {
_port = port;
_running = YES;

dispatch_queue_t queue = dispatch_queue_create("com.witchan.ios-mcp.accept", DISPATCH_QUEUE_CONCURRENT);
dispatch_queue_attr_t acceptAttributes =
dispatch_queue_attr_make_with_autorelease_frequency(DISPATCH_QUEUE_CONCURRENT,
DISPATCH_AUTORELEASE_FREQUENCY_WORK_ITEM);
dispatch_queue_t queue = dispatch_queue_create("com.witchan.ios-mcp.accept", acceptAttributes);
_acceptSource = dispatch_source_create(DISPATCH_SOURCE_TYPE_READ, sock, 0, queue);

__weak typeof(self) weakSelf = self;
Expand All @@ -819,7 +830,9 @@ - (void)startOnPort:(uint16_t)port {
if (client >= 0) {
MCPSetCloseOnExec(client);
dispatch_async(self->_clientQueue, ^{
[self handleClient:client];
@autoreleasepool {
[self handleClient:client];
}
});
}
});
Expand Down Expand Up @@ -2150,6 +2163,27 @@ - (NSDictionary *)handleToolsList:(id)reqId {

#pragma mark - MCP: tools/call

- (NSDictionary *)performScreenTaskForRequest:(id)reqId task:(NSDictionary *(^)(void))task {
if (!task) {
return [self mcpError:reqId code:-32603 message:@"Missing screen task"];
}

dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, 30 * NSEC_PER_SEC);
if (dispatch_semaphore_wait(_screenTaskSemaphore, timeout) != 0) {
return [self mcpError:reqId code:-32000 message:@"Timed out waiting for screen processing"];
}

NSDictionary *result = nil;
@try {
@autoreleasepool {
result = task();
}
} @finally {
dispatch_semaphore_signal(_screenTaskSemaphore);
}
return result;
}

- (NSDictionary *)handleToolsCall:(id)reqId params:(NSDictionary *)params {
if (![params isKindOfClass:[NSDictionary class]]) {
return [self mcpError:reqId code:-32602 message:@"Invalid params: expected object"];
Expand Down Expand Up @@ -2207,7 +2241,9 @@ - (NSDictionary *)handleToolsCall:(id)reqId params:(NSDictionary *)params {
else if ([toolName isEqualToString:@"get_screen_info"]) {
return [self executeScreenInfo:reqId];
} else if ([toolName isEqualToString:@"screenshot"]) {
return [self executeScreenshot:reqId args:args];
return [self performScreenTaskForRequest:reqId task:^{
return [self executeScreenshot:reqId args:args];
}];
}
// Clipboard tools
else if ([toolName isEqualToString:@"get_clipboard"]) {
Expand All @@ -2233,8 +2269,17 @@ - (NSDictionary *)handleToolsCall:(id)reqId params:(NSDictionary *)params {
} else if ([toolName isEqualToString:@"get_element_at_point"]) {
return [self executeGetElementAtPoint:reqId args:args];
} else if ([toolName isEqualToString:@"ocr_screen"]) {
return [self executeOCRScreen:reqId args:args];
return [self performScreenTaskForRequest:reqId task:^{
return [self executeOCRScreen:reqId args:args];
}];
} else if ([toolName isEqualToString:@"describe_screen"]) {
BOOL includesImageWork = [args[@"include_ocr"] boolValue] ||
[args[@"include_screenshot"] boolValue];
if (includesImageWork) {
return [self performScreenTaskForRequest:reqId task:^{
return [self executeDescribeScreen:reqId args:args];
}];
}
return [self executeDescribeScreen:reqId args:args];
}
// Text input tools
Expand Down
6 changes: 4 additions & 2 deletions OCRManager.m
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,11 @@ - (NSDictionary *)recognizeTextWithLanguages:(NSArray<NSString *> *)languages

// Downsample large captures before OCR (longest edge cap). Speeds up Vision on
// high-res iPad screens; coordinates still map back via the logical image.size.
CGImageRef ocrImage = image.CGImage;
CGImageRef downsampled = OCRCreateDownsampled(image.CGImage, 1600.0);
if (downsampled) ocrImage = downsampled;
CGImageRef ocrImage = downsampled ?: image.CGImage;
if (downsampled) {
image = nil;
}

VNImageRequestHandler *handler = [[VNImageRequestHandler alloc] initWithCGImage:ocrImage options:@{}];
NSError *performError = nil;
Expand Down
44 changes: 28 additions & 16 deletions ScreenManager.m
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
} while (0)

typedef struct __IOSurface *IOSurfaceRef;
typedef UIImage *(*UICreateScreenUIImageFunc)(void);
typedef UIImage *(*UICreateScreenUIImageFunc)(void) NS_RETURNS_RETAINED;
typedef CGImageRef (*UICreateCGImageFromIOSurfaceFunc)(IOSurfaceRef surface);
typedef CGImageRef (*CARenderServerCaptureDisplayFunc)(uint32_t serverPort, CFStringRef displayName, CFDictionaryRef options);

Expand All @@ -27,6 +27,14 @@
static const CGFloat kMCPScreenshotMinimumJPEGQuality = 0.30;
static const NSInteger kMCPScreenshotJPEGSearchPasses = 6;

static NSData *MCPJPEGRepresentation(UIImage *image, CGFloat quality) {
NSData *data = nil;
@autoreleasepool {
data = UIImageJPEGRepresentation(image, quality);
}
return data;
}

/// Point-space target size for a pixel-space capture of `pixelSize`.
///
/// Screenshots are downsampled so that one image pixel equals one screen point, which makes the
Expand Down Expand Up @@ -295,11 +303,13 @@ - (NSDictionary *)deviceInteractionStateOnMainThread {
- (NSDictionary *)takeScreenshotPayload {
__block NSDictionary *payload = nil;
dispatch_block_t block = ^{
payload = [self privateScreenshotPayload];
if (!payload) {
SCREEN_LOG(@"Private screenshot APIs produced no encodable image, falling back to window capture");
UIImage *image = [self fallbackScreenshotImage];
payload = [self payloadByEncodingImage:image source:@"window_capture"];
@autoreleasepool {
payload = [self privateScreenshotPayload];
if (!payload) {
SCREEN_LOG(@"Private screenshot APIs produced no encodable image, falling back to window capture");
UIImage *image = [self fallbackScreenshotImage];
payload = [self payloadByEncodingImage:image source:@"window_capture"];
}
}
};

Expand All @@ -315,10 +325,6 @@ - (NSDictionary *)privateScreenshotPayload {
UIImage *image = nil;
NSDictionary *payload = nil;

NSData *screenData = [ScreenManager getScreenDataWithQuantity:(NSInteger)round(kMCPScreenshotInitialJPEGQuality * 100.0)];
payload = [self payloadByEncodingImageData:screenData source:@"getScreenDataWithQuantity"];
if (payload) return payload;

image = [self screenshotImageFromRenderServerCapture];
payload = [self payloadByEncodingImage:image source:@"render_server"];
if (payload) return payload;
Expand All @@ -331,6 +337,10 @@ - (NSDictionary *)privateScreenshotPayload {
payload = [self payloadByEncodingImage:image source:@"createScreenIOSurface"];
if (payload) return payload;

NSData *screenData = [ScreenManager getScreenDataWithQuantity:(NSInteger)round(kMCPScreenshotInitialJPEGQuality * 100.0)];
payload = [self payloadByEncodingImageData:screenData source:@"getScreenDataWithQuantity"];
if (payload) return payload;

return nil;
}

Expand Down Expand Up @@ -400,9 +410,11 @@ - (UIImage *)privateScreenshotImage {
- (UIImage *)captureScreenImage {
__block UIImage *image = nil;
dispatch_block_t block = ^{
image = [self privateScreenshotImage];
if (!image) {
image = [self fallbackScreenshotImage];
@autoreleasepool {
image = [self privateScreenshotImage];
if (!image) {
image = [self fallbackScreenshotImage];
}
}
};
if ([NSThread isMainThread]) {
Expand Down Expand Up @@ -589,11 +601,11 @@ - (UIImage *)pointSizedImageFromImage:(UIImage *)image {
}

- (NSData *)JPEGDataForImage:(UIImage *)image maxBytes:(NSUInteger)maxBytes {
NSData *bestData = UIImageJPEGRepresentation(image, kMCPScreenshotInitialJPEGQuality);
NSData *bestData = MCPJPEGRepresentation(image, kMCPScreenshotInitialJPEGQuality);
if (!bestData) return nil;
if (bestData.length <= maxBytes) return bestData;

NSData *minimumData = UIImageJPEGRepresentation(image, kMCPScreenshotMinimumJPEGQuality);
NSData *minimumData = MCPJPEGRepresentation(image, kMCPScreenshotMinimumJPEGQuality);
if (!minimumData) return bestData;
if (minimumData.length > maxBytes) return minimumData;

Expand All @@ -605,7 +617,7 @@ - (NSData *)JPEGDataForImage:(UIImage *)image maxBytes:(NSUInteger)maxBytes {
CGFloat high = kMCPScreenshotInitialJPEGQuality;
for (NSInteger pass = 0; pass < kMCPScreenshotJPEGSearchPasses; pass++) {
CGFloat quality = (low + high) / 2.0;
NSData *candidate = UIImageJPEGRepresentation(image, quality);
NSData *candidate = MCPJPEGRepresentation(image, quality);
if (!candidate) break;

if (candidate.length > maxBytes) {
Expand Down
30 changes: 30 additions & 0 deletions prefs/Resources/Root.plist
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,36 @@
<key>placeholder</key>
<string>8090</string>
</dict>
<dict>
<key>cell</key>
<string>PSGroupCell</string>
<key>label</key>
<string>图像处理</string>
<key>footerText</key>
<string>限制截图和 OCR 同时处理的任务数。1 更节省内存,2 可提高高内存设备的吞吐量。修改后需要重启 SpringBoard。</string>
</dict>
<dict>
<key>cell</key>
<string>PSSegmentCell</string>
<key>default</key>
<integer>1</integer>
<key>defaults</key>
<string>com.witchan.ios-mcp.preferences</string>
<key>key</key>
<string>maxConcurrentScreenTasks</string>
<key>label</key>
<string>并发任务</string>
<key>validTitles</key>
<array>
<string>1</string>
<string>2</string>
</array>
<key>validValues</key>
<array>
<integer>1</integer>
<integer>2</integer>
</array>
</dict>
<dict>
<key>cell</key>
<string>PSGroupCell</string>
Expand Down