Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

metal : add debug capture backend function #694

Merged
merged 2 commits into from
Jan 29, 2024
Merged

metal : add debug capture backend function #694

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
21815c6
metal : add debug capture backend function
jmousseau Jan 11, 2024
40cbc4e
Merge branch 'master' into HEAD
ggerganov Jan 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/ggml-metal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(voi
// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
GGML_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family);

// capture all command buffers committed the next time `ggml_backend_graph_compute` is called
GGML_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@slaren Is it a good idea to extend the backend API with these type of calls or should we try to avoid it. In this case, it could remain a compile-time option (ggml-org/llama.cpp@2a7c94d), but it might be useful to control at runtime in some cases

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Generally I don't see an issue with adding more backend-specific functions, as long as they are not required for normal usage.


#ifdef __cplusplus
}
#endif
Expand Down
40 changes: 34 additions & 6 deletions src/ggml-metal.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,8 @@

bool support_simdgroup_reduction;
bool support_simdgroup_mm;

bool should_capture_next_compute;
};

// MSL code
Expand Down Expand Up @@ -684,6 +686,20 @@ static bool ggml_metal_graph_compute(
const int n_cb = ctx->n_cb;
const int n_nodes_per_cb = (n_nodes + n_cb - 1) / n_cb;

const bool should_capture = ctx->should_capture_next_compute;
if (should_capture) {
ctx->should_capture_next_compute = false;

MTLCaptureDescriptor * descriptor = [MTLCaptureDescriptor new];
descriptor.captureObject = ctx->queue;

NSError * error = nil;
if (![[MTLCaptureManager sharedCaptureManager] startCaptureWithDescriptor:descriptor error:&error]) {
GGML_METAL_LOG_ERROR("%s: error: unable to start capture '%s'\n", __func__, [[error localizedDescription] UTF8String]);
GGML_ASSERT(!"capture failed");
}
}

id<MTLCommandBuffer> command_buffer_builder[n_cb];
for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) {
id<MTLCommandBuffer> command_buffer = [ctx->queue commandBufferWithUnretainedReferences];
Expand All @@ -692,6 +708,7 @@ static bool ggml_metal_graph_compute(
// enqueue the command buffers in order to specify their execution order
[command_buffer enqueue];
}

const id<MTLCommandBuffer> *command_buffers = command_buffer_builder;

dispatch_apply(n_cb, ctx->d_queue, ^(size_t iter) {
Expand Down Expand Up @@ -738,9 +755,9 @@ static bool ggml_metal_graph_compute(
GGML_ASSERT(!"unsupported op");
}

#ifndef GGML_METAL_NDEBUG
[encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]];
#endif
if (should_capture) {
[encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]];
}

const int64_t ne00 = src0 ? src0->ne[0] : 0;
const int64_t ne01 = src0 ? src0->ne[1] : 0;
Expand Down Expand Up @@ -2190,9 +2207,9 @@ static bool ggml_metal_graph_compute(
}
}

#ifndef GGML_METAL_NDEBUG
[encoder popDebugGroup];
#endif
if (should_capture) {
[encoder popDebugGroup];
}
}

[encoder endEncoding];
Expand All @@ -2214,6 +2231,10 @@ static bool ggml_metal_graph_compute(
}
}

if (should_capture) {
[[MTLCaptureManager sharedCaptureManager] stopCapture];
}

return true;
}

Expand Down Expand Up @@ -2575,6 +2596,13 @@ bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {
return [ctx->device supportsFamily:(MTLGPUFamilyApple1 + family - 1)];
}

void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {
GGML_ASSERT(ggml_backend_is_metal(backend));

struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
ctx->should_capture_next_compute = true;
}

GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data); // silence warning

GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) {
Expand Down
Loading