From 1dac10675d08b010b8654539b028e841d65875f0 Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Wed, 1 Jul 2026 10:55:51 +0800 Subject: [PATCH 1/5] gh-152721: Fix quadratic RLE replay time in the profiling binary reader --- .../test_binary_format.py | 43 +++++++++++++++++++ ...07-01-18-00-00.gh-issue-152721.rlequad.rst | 2 + Modules/_remote_debugging/binary_io_reader.c | 11 ++++- 3 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-01-18-00-00.gh-issue-152721.rlequad.rst diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index 5efc60a92111754..fce1809fedd6869 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -1175,6 +1175,49 @@ def test_alternating_threads_status_changes(self): self.assertEqual(count, 100) self.assert_samples_equal(samples, collector) + def test_rle_alternating_status_batches_correctly(self): + """A repeat record whose status alternates every sample replays as N + single-status batches with the right cumulative timestamps.""" + class BatchCollector: + def __init__(self): + self.batches = [] + + def collect(self, stack_frames, timestamps_us): + for interp in stack_frames: + for thread in interp.threads: + self.batches.append( + (thread.status, list(timestamps_us)) + ) + + def export(self, filename): + pass + + num_samples = 2000 + frame = make_frame("rle.py", 42, "rle_func") + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + writer = BinaryCollector(filename, 1000, compression="none") + expected = [] + for i in range(num_samples): + status = THREAD_STATUS_HAS_GIL if i % 2 else 0 + ts = 1000 + i + expected.append((status, [ts])) + sample = [ + make_interpreter(0, [make_thread(1, [frame], status)]) + ] + writer.collect(sample, timestamp_us=ts) + writer.export(None) + + collector = BatchCollector() + with BinaryReader(filename) as reader: + count = reader.replay_samples(collector) + + self.assertEqual(count, num_samples) + self.assertEqual(len(collector.batches), num_samples) + self.assertEqual(collector.batches, expected) + class TestBinaryStress(BinaryFormatTestBase): """Randomized stress tests for binary format.""" diff --git a/Misc/NEWS.d/next/Library/2026-07-01-18-00-00.gh-issue-152721.rlequad.rst b/Misc/NEWS.d/next/Library/2026-07-01-18-00-00.gh-issue-152721.rlequad.rst new file mode 100644 index 000000000000000..4dac0ed245bd672 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-01-18-00-00.gh-issue-152721.rlequad.rst @@ -0,0 +1,2 @@ +Fix quadratic replay time in the :mod:`profiling.sampling` binary reader when a +profile's run-length-encoded samples alternate thread status. diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index ce1c3d232c94e0f..91073dfbcb95805 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -1105,7 +1105,8 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre return -1; } } - timestamps_list = PyList_New(count - i); + /* Exact-size the list; alloc+trim is O(count^2). */ + timestamps_list = PyList_New(0); if (!timestamps_list) { return -1; } @@ -1118,7 +1119,13 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre Py_DECREF(timestamps_list); return -1; } - PyList_SET_ITEM(timestamps_list, batch_idx++, ts_obj); + int append_rc = PyList_Append(timestamps_list, ts_obj); + Py_DECREF(ts_obj); + if (append_rc < 0) { + Py_DECREF(timestamps_list); + return -1; + } + batch_idx++; } /* Emit final batch */ From beb5980422b91f9ce36e1342f912d01e2d3063cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Thu, 2 Jul 2026 10:30:58 +0800 Subject: [PATCH 2/5] gh-152721: Clarify the batch-list comment Reword per review: the list is built with append per element, not pre-sized; note the old alloc(count - i) + trim per batch was O(count^2). --- Modules/_remote_debugging/binary_io_reader.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 91073dfbcb95805..ee8e748bfda7490 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -1105,7 +1105,7 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre return -1; } } - /* Exact-size the list; alloc+trim is O(count^2). */ + /* Append per element; the old alloc(count - i) + trim per batch was O(count^2). */ timestamps_list = PyList_New(0); if (!timestamps_list) { return -1; From 3e71061c1728f3b9093239f4e071037c79a97d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Thu, 2 Jul 2026 22:23:47 +0800 Subject: [PATCH 3/5] Cap RLE batch size to bound the per-batch timestamp list --- Modules/_remote_debugging/binary_io_reader.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index ee8e748bfda7490..2186018f773ddb1 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -27,6 +27,7 @@ /* Progress callback frequency */ #define PROGRESS_CALLBACK_INTERVAL 1000 +#define MAX_RLE_BATCH_SAMPLES 8192 /* Cap per-batch samples to bound the timestamp list (gh-152089) */ /* ============================================================================ * BINARY READER IMPLEMENTATION @@ -1095,7 +1096,8 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre ts->prev_timestamp += delta; /* Start new batch on first sample or status change */ - if (i == 0 || status != batch_status) { + if (i == 0 || status != batch_status + || batch_idx >= MAX_RLE_BATCH_SAMPLES) { if (timestamps_list) { int rc = emit_batch(state, collector, thread_id, interpreter_id, batch_status, ts->current_stack, ts->current_stack_depth, From 65943181238e3cb8e952247be352fe84822ecd9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Thu, 2 Jul 2026 22:35:53 +0800 Subject: [PATCH 4/5] Move MAX_RLE_BATCH_SAMPLES comment to its own line (keep under 79 cols) --- Modules/_remote_debugging/binary_io_reader.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 2186018f773ddb1..b2a89b1177ef097 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -27,7 +27,9 @@ /* Progress callback frequency */ #define PROGRESS_CALLBACK_INTERVAL 1000 -#define MAX_RLE_BATCH_SAMPLES 8192 /* Cap per-batch samples to bound the timestamp list (gh-152089) */ + +/* Cap per-batch RLE samples to bound the timestamp list (gh-152089) */ +#define MAX_RLE_BATCH_SAMPLES 8192 /* ============================================================================ * BINARY READER IMPLEMENTATION From 70a2e418346ac3d53e94c6016cf85ab29a9c6b27 Mon Sep 17 00:00:00 2001 From: tonghuaroot Date: Tue, 18 Aug 2026 18:31:20 +0800 Subject: [PATCH 5/5] Drop the now-redundant emit_batch trim and fix the review nits The batch list is built to its exact size, so the PyList_SetSlice trim was a no-op; call emit_sample directly. Also point the cap comment at the issue (gh-151378) and drop the stale over-long comment. --- Modules/_remote_debugging/binary_io_reader.c | 30 +++++--------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index b2a89b1177ef097..407020d26d9d180 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -28,7 +28,7 @@ /* Progress callback frequency */ #define PROGRESS_CALLBACK_INTERVAL 1000 -/* Cap per-batch RLE samples to bound the timestamp list (gh-152089) */ +/* Cap per-batch RLE samples to bound the timestamp list (gh-151378) */ #define MAX_RLE_BATCH_SAMPLES 8192 /* ============================================================================ @@ -950,21 +950,6 @@ emit_sample(RemoteDebuggingState *state, PyObject *collector, return 0; } -/* Helper to trim timestamp list and emit batch. Returns 0 on success, -1 on error. */ -static int -emit_batch(RemoteDebuggingState *state, PyObject *collector, - uint64_t thread_id, uint32_t interpreter_id, uint8_t status, - const uint32_t *frame_indices, size_t stack_depth, - BinaryReader *reader, PyObject *timestamps_list, Py_ssize_t actual_size) -{ - /* Trim list to actual size */ - if (PyList_SetSlice(timestamps_list, actual_size, PyList_GET_SIZE(timestamps_list), NULL) < 0) { - return -1; - } - return emit_sample(state, collector, thread_id, interpreter_id, status, - frame_indices, stack_depth, reader, timestamps_list); -} - /* Helper to invoke progress callback, returns -1 on error */ static inline int invoke_progress_callback(PyObject *callback, Py_ssize_t current, uint32_t total) @@ -1101,15 +1086,14 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre if (i == 0 || status != batch_status || batch_idx >= MAX_RLE_BATCH_SAMPLES) { if (timestamps_list) { - int rc = emit_batch(state, collector, thread_id, interpreter_id, - batch_status, ts->current_stack, ts->current_stack_depth, - reader, timestamps_list, batch_idx); + int rc = emit_sample(state, collector, thread_id, interpreter_id, + batch_status, ts->current_stack, ts->current_stack_depth, + reader, timestamps_list); Py_DECREF(timestamps_list); if (rc < 0) { return -1; } } - /* Append per element; the old alloc(count - i) + trim per batch was O(count^2). */ timestamps_list = PyList_New(0); if (!timestamps_list) { return -1; @@ -1134,9 +1118,9 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre /* Emit final batch */ if (timestamps_list) { - int rc = emit_batch(state, collector, thread_id, interpreter_id, - batch_status, ts->current_stack, ts->current_stack_depth, - reader, timestamps_list, batch_idx); + int rc = emit_sample(state, collector, thread_id, interpreter_id, + batch_status, ts->current_stack, ts->current_stack_depth, + reader, timestamps_list); Py_DECREF(timestamps_list); if (rc < 0) { return -1;