From 543b267aed32aad5e05dc66e1f961e9f0b9fbf3a Mon Sep 17 00:00:00 2001 From: TCMalloc Team Date: Thu, 5 Nov 2026 06:34:14 -0800 Subject: [PATCH] Preserve allocating thread and CPU in DeallocationProfiler. It previously attributed pre-existing live allocations seeded via DeallocationProfilerList::Add to the thread starting the profiler rather than the allocating thread. Storing cpu_id, vcpu_id, and thread_id in StackTrace preserves thread and CPU matching across seeded allocations. Co-authored-by: Chris Kennelly PiperOrigin-RevId: 1014653130 --- tcmalloc/allocation_sampling.cc | 5 ++ tcmalloc/deallocation_profiler.cc | 12 +++-- tcmalloc/experimental_pow2_size_class.cc | 8 +-- ...flexible_no_reuse_no_40_56_size_classes.cc | 54 +++++++++---------- tcmalloc/flexible_no_reuse_size_classes.cc | 54 +++++++++---------- tcmalloc/flexible_size_classes.cc | 24 ++++----- tcmalloc/internal/logging.h | 4 ++ tcmalloc/size_classes.cc | 22 ++++---- .../testing/deallocation_profiler_test.cc | 33 ++++++++++++ 9 files changed, 130 insertions(+), 86 deletions(-) diff --git a/tcmalloc/allocation_sampling.cc b/tcmalloc/allocation_sampling.cc index 0e2e5cef3..b1b046d8f 100644 --- a/tcmalloc/allocation_sampling.cc +++ b/tcmalloc/allocation_sampling.cc @@ -25,6 +25,7 @@ #include #include "absl/base/attributes.h" +#include "absl/base/internal/sysinfo.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/debugging/stacktrace.h" @@ -238,6 +239,10 @@ SampleifyAllocation(Static& state, DynamicPolicy policy, size_t requested_size, }; MallocHook::InvokeSampledNewHook(sampled_alloc); + stack_trace.cpu_id = subtle::percpu::GetRealCpu(); + stack_trace.vcpu_id = subtle::percpu::VirtualCpu::get(); + stack_trace.thread_id = absl::base_internal::GetTID(); + state.allocation_samples.ReportMalloc(stack_trace); state.deallocation_samples.ReportMalloc(stack_trace); diff --git a/tcmalloc/deallocation_profiler.cc b/tcmalloc/deallocation_profiler.cc index 65bfd2d30..e381c0f2a 100644 --- a/tcmalloc/deallocation_profiler.cc +++ b/tcmalloc/deallocation_profiler.cc @@ -522,11 +522,11 @@ class DeallocationProfiler { allocation.stack_trace = stack_trace; // TODO(mmaas): Do we need to worry about b/65384231 anymore? - allocation.cpu_id = tcmalloc_internal::subtle::percpu::GetRealCpu(); - allocation.vcpu_id = tcmalloc_internal::subtle::percpu::VirtualCpu::get(); + allocation.cpu_id = stack_trace.cpu_id; + allocation.vcpu_id = stack_trace.vcpu_id; allocation.l3_id = GetL3Id(allocation.cpu_id); allocation.numa_id = GetNumaId(allocation.cpu_id); - allocation.thread_id = absl::base_internal::GetTID(); + allocation.thread_id = stack_trace.thread_id; } void ReportFree(tcmalloc_internal::AllocHandle handle, @@ -569,7 +569,8 @@ class DeallocationProfiler { } }; -void DeallocationProfilerList::Add(DeallocationProfiler* profiler) { +void DeallocationProfilerList::Add( + DeallocationProfiler* absl_nonnull profiler) { AllocationGuardSpinLockHolder h(profilers_lock_); profiler->next_ = first_.load(std::memory_order_relaxed); first_.store(profiler, std::memory_order_release); @@ -583,7 +584,8 @@ void DeallocationProfilerList::Add(DeallocationProfiler* profiler) { } // This list is very short and we're nowhere near a hot path, just walk -void DeallocationProfilerList::Remove(DeallocationProfiler* profiler) { +void DeallocationProfilerList::Remove( + DeallocationProfiler* absl_nonnull profiler) { AllocationGuardSpinLockHolder h(profilers_lock_); DeallocationProfiler* cur = first_.load(std::memory_order_relaxed); if (cur == profiler) { diff --git a/tcmalloc/experimental_pow2_size_class.cc b/tcmalloc/experimental_pow2_size_class.cc index 987004810..210c7cbd3 100644 --- a/tcmalloc/experimental_pow2_size_class.cc +++ b/tcmalloc/experimental_pow2_size_class.cc @@ -70,7 +70,7 @@ static constexpr SizeClassInfo List[] = { { 128, 8192, 32}, // 5 64 0.78% 0.42% --------- 100.00% ----------- { 256, 8192, 32}, // 6 32 0.78% 0.41% --------- 100.00% ----------- { 512, 8192, 32}, // 7 16 0.78% 0.40% --------- 100.00% ----------- - { 1024, 8192, 32}, // 8 8 0.78% 0.37% --------- 100.00% ----------- + { 1024, 8192, 32}, // 8 8 0.78% 0.38% --------- 100.00% ----------- { 2048, 8192, 32}, // 9 4 0.78% 0.33% --------- 100.00% ----------- { 4096, 8192, 16}, // 10 2 0.78% 0.23% --------- 100.00% ----------- { 8192, 8192, 8}, // 11 1 0.78% 0.03% --------- 100.00% ----------- @@ -94,7 +94,7 @@ static constexpr SizeClassInfo List[] = { // bytes span_bytes batch class objs | fixed sampling int.frag | inc used { 0, 0, 0}, // 0 0 0.00% 0.00% --------- 0.00% ----------- { 8, 32768, 32}, // 1 4096 0.20% 1.60% --------- 0.00% ----------- - { 16, 32768, 32}, // 2 2048 0.20% 1.59% --------- 100.00% ----------- + { 16, 32768, 32}, // 2 2048 0.20% 1.60% --------- 100.00% ----------- { 32, 32768, 32}, // 3 1024 0.20% 1.59% --------- 100.00% ----------- { 64, 32768, 32}, // 4 512 0.20% 1.59% --------- 100.00% ----------- { 128, 32768, 32}, // 5 256 0.20% 1.59% --------- 100.00% ----------- @@ -189,7 +189,7 @@ static constexpr SizeClassInfo List[] = { { 128, 8192, 32}, // 5 64 0.78% 0.42% --------- 100.00% ----------- { 256, 8192, 32}, // 6 32 0.78% 0.41% --------- 100.00% ----------- { 512, 8192, 32}, // 7 16 0.78% 0.40% --------- 100.00% ----------- - { 1024, 8192, 32}, // 8 8 0.78% 0.37% --------- 100.00% ----------- + { 1024, 8192, 32}, // 8 8 0.78% 0.38% --------- 100.00% ----------- { 2048, 8192, 32}, // 9 4 0.78% 0.33% --------- 100.00% ----------- { 4096, 8192, 16}, // 10 2 0.78% 0.23% --------- 100.00% ----------- { 8192, 8192, 8}, // 11 1 0.78% 0.03% --------- 100.00% ----------- @@ -213,7 +213,7 @@ static constexpr SizeClassInfo List[] = { // bytes span_bytes batch class objs | fixed sampling int.frag | inc used { 0, 0, 0}, // 0 0 0.00% 0.00% --------- 0.00% ----------- { 8, 32768, 32}, // 1 4096 0.20% 1.60% --------- 0.00% ----------- - { 16, 32768, 32}, // 2 2048 0.20% 1.59% --------- 100.00% ----------- + { 16, 32768, 32}, // 2 2048 0.20% 1.60% --------- 100.00% ----------- { 32, 32768, 32}, // 3 1024 0.20% 1.59% --------- 100.00% ----------- { 64, 32768, 32}, // 4 512 0.20% 1.59% --------- 100.00% ----------- { 128, 32768, 32}, // 5 256 0.20% 1.59% --------- 100.00% ----------- diff --git a/tcmalloc/flexible_no_reuse_no_40_56_size_classes.cc b/tcmalloc/flexible_no_reuse_no_40_56_size_classes.cc index 66b8b2ca3..78ff94821 100644 --- a/tcmalloc/flexible_no_reuse_no_40_56_size_classes.cc +++ b/tcmalloc/flexible_no_reuse_no_40_56_size_classes.cc @@ -81,7 +81,7 @@ static constexpr SizeClassInfo List[] = { { 144, 8192, 32}, // 16 56 2.38% 0.42% --------- 5.88% ----------- { 160, 8192, 32}, // 17 51 1.18% 0.42% --------- 11.11% ----------- { 176, 8192, 32}, // 18 46 1.98% 0.42% --------- 10.00% ----------- - { 192, 8192, 32}, // 19 42 2.38% 0.41% --------- 9.09% ----------- + { 192, 8192, 32}, // 19 42 2.38% 0.42% --------- 9.09% ----------- { 208, 8192, 32}, // 20 39 1.78% 0.41% --------- 8.33% ----------- { 224, 8192, 32}, // 21 36 2.38% 0.41% --------- 7.69% ----------- { 240, 8192, 32}, // 22 34 1.18% 0.41% --------- 7.14% ----------- @@ -98,9 +98,9 @@ static constexpr SizeClassInfo List[] = { { 584, 8192, 32}, // 33 14 0.98% 0.40% --------- 14.06% ----------- { 680, 8192, 32}, // 34 12 1.18% 0.39% --------- 16.44% ----------- { 744, 8192, 32}, // 35 11 0.88% 0.39% --------- 9.41% ----------- - { 816, 8192, 32}, // 36 10 1.18% 0.38% --------- 9.68% ----------- + { 816, 8192, 32}, // 36 10 1.18% 0.39% --------- 9.68% ----------- { 904, 8192, 32}, // 37 9 1.47% 0.38% --------- 10.78% ----------- - { 1024, 8192, 32}, // 38 8 0.78% 0.37% --------- 13.27% ----------- + { 1024, 8192, 32}, // 38 8 0.78% 0.38% --------- 13.27% ----------- { 1152, 16384, 32}, // 39 14 1.98% 0.37% --------- 12.50% ----------- { 1280, 16384, 32}, // 40 12 7.08% 0.36% --------- 11.11% ----------- { 1408, 16384, 32}, // 41 11 6.20% 0.36% --------- 10.00% ----------- @@ -162,7 +162,7 @@ static constexpr SizeClassInfo List[] = { // bytes span_bytes batch class objs | fixed sampling int.frag | inc used { 0, 0, 0}, // 0 0 0.00% 0.00% --------- 0.00% ----------- { 8, 32768, 32}, // 1 4096 0.20% 1.60% --------- 0.00% ----------- - { 16, 32768, 32}, // 2 2048 0.20% 1.59% --------- 100.00% ----------- + { 16, 32768, 32}, // 2 2048 0.20% 1.60% --------- 100.00% ----------- { 24, 32768, 32}, // 3 1365 0.22% 1.59% --------- 50.00% ----------- { 32, 32768, 32}, // 4 1024 0.20% 1.59% --------- 33.33% ----------- { 48, 32768, 32}, // 5 682 0.29% 1.59% --------- 50.00% ----------- @@ -200,7 +200,7 @@ static constexpr SizeClassInfo List[] = { { 840, 32768, 32}, // 37 39 0.22% 1.56% --------- 8.25% ----------- { 1024, 32768, 32}, // 38 32 0.20% 1.55% --------- 21.90% ----------- { 1152, 32768, 32}, // 39 28 1.79% 1.54% --------- 12.50% ----------- - { 1280, 32768, 32}, // 40 25 2.60% 1.53% --------- 11.11% ----------- + { 1280, 32768, 32}, // 40 25 2.60% 1.54% --------- 11.11% ----------- { 1408, 32768, 32}, // 41 23 1.38% 1.53% --------- 10.00% ----------- { 1536, 32768, 32}, // 42 21 1.79% 1.52% --------- 9.09% ----------- { 1792, 32768, 32}, // 43 18 1.79% 1.51% --------- 16.67% ----------- @@ -212,12 +212,12 @@ static constexpr SizeClassInfo List[] = { { 2688, 32768, 24}, // 49 12 1.79% 1.47% --------- 10.53% ----------- { 2944, 32768, 22}, // 50 11 1.38% 1.46% --------- 9.52% ----------- { 3200, 32768, 20}, // 51 10 2.60% 1.44% --------- 8.70% ----------- - { 3584, 32768, 18}, // 52 9 1.79% 1.42% --------- 12.00% ----------- + { 3584, 32768, 18}, // 52 9 1.79% 1.43% --------- 12.00% ----------- { 4096, 32768, 16}, // 53 8 0.20% 1.40% --------- 14.29% ----------- { 4608, 32768, 14}, // 54 7 1.79% 1.38% --------- 12.50% ----------- { 4992, 65536, 13}, // 55 13 1.08% 1.36% --------- 8.33% ----------- { 5376, 32768, 12}, // 56 6 1.79% 1.34% --------- 7.69% ----------- - { 5888, 65536, 11}, // 57 11 1.28% 1.31% --------- 9.52% ----------- + { 5888, 65536, 11}, // 57 11 1.28% 1.32% --------- 9.52% ----------- { 6528, 32768, 10}, // 58 5 0.59% 1.28% --------- 10.87% ----------- { 7168, 65536, 9}, // 59 9 1.69% 1.25% --------- 9.80% ----------- { 8192, 32768, 8}, // 60 4 0.20% 1.21% --------- 14.29% ----------- @@ -276,7 +276,7 @@ static constexpr SizeClassInfo List[] = { { 136, 262144, 32}, // 15 1927 0.05% 12.53% --------- 6.25% ----------- { 152, 262144, 32}, // 16 1724 0.06% 12.53% --------- 11.76% ----------- { 160, 262144, 32}, // 17 1638 0.05% 12.53% --------- 5.26% ----------- - { 176, 262144, 32}, // 18 1489 0.05% 12.52% --------- 10.00% ----------- + { 176, 262144, 32}, // 18 1489 0.05% 12.53% --------- 10.00% ----------- { 192, 262144, 32}, // 19 1365 0.05% 12.52% --------- 9.09% ----------- { 208, 262144, 32}, // 20 1260 0.05% 12.52% --------- 8.33% ----------- { 240, 262144, 32}, // 21 1092 0.05% 12.52% --------- 15.38% ----------- @@ -284,7 +284,7 @@ static constexpr SizeClassInfo List[] = { { 272, 262144, 32}, // 23 963 0.10% 12.52% --------- 6.25% ----------- { 304, 262144, 32}, // 24 862 0.06% 12.52% --------- 11.76% ----------- { 352, 262144, 32}, // 25 744 0.12% 12.52% --------- 15.79% ----------- - { 384, 262144, 32}, // 26 682 0.12% 12.51% --------- 9.09% ----------- + { 384, 262144, 32}, // 26 682 0.12% 12.52% --------- 9.09% ----------- { 408, 262144, 32}, // 27 642 0.10% 12.51% --------- 6.25% ----------- { 448, 262144, 32}, // 28 585 0.05% 12.51% --------- 9.80% ----------- { 512, 262144, 32}, // 29 512 0.02% 12.51% --------- 14.29% ----------- @@ -308,7 +308,7 @@ static constexpr SizeClassInfo List[] = { { 4096, 262144, 16}, // 47 64 0.02% 12.34% --------- 10.34% ----------- { 4480, 262144, 14}, // 48 58 0.91% 12.32% --------- 9.38% ----------- { 4864, 262144, 13}, // 49 53 1.71% 12.30% --------- 8.57% ----------- - { 5632, 262144, 11}, // 50 46 1.21% 12.26% --------- 15.79% ----------- + { 5632, 262144, 11}, // 50 46 1.21% 12.27% --------- 15.79% ----------- { 6144, 262144, 10}, // 51 42 1.61% 12.24% --------- 9.09% ----------- { 6656, 262144, 9}, // 52 39 1.01% 12.22% --------- 8.33% ----------- { 8192, 262144, 8}, // 53 32 0.02% 12.14% --------- 23.08% ----------- @@ -364,7 +364,7 @@ static constexpr SizeClassInfo List[] = { { 48, 4096, 32}, // 5 85 1.96% 0.23% --------- 50.00% ----------- { 64, 4096, 32}, // 6 64 1.56% 0.23% --------- 33.33% ----------- { 72, 4096, 32}, // 7 56 3.17% 0.23% --------- 12.50% ----------- - { 80, 4096, 32}, // 8 51 1.96% 0.22% --------- 11.11% ----------- + { 80, 4096, 32}, // 8 51 1.96% 0.23% --------- 11.11% ----------- { 88, 4096, 32}, // 9 46 2.77% 0.22% --------- 10.00% ----------- { 96, 4096, 32}, // 10 42 3.17% 0.22% --------- 9.09% ----------- { 104, 4096, 32}, // 11 39 2.56% 0.22% --------- 8.33% ----------- @@ -385,7 +385,7 @@ static constexpr SizeClassInfo List[] = { { 240, 4096, 32}, // 26 17 1.96% 0.22% --------- 7.14% ----------- { 256, 4096, 32}, // 27 16 1.56% 0.22% --------- 6.67% ----------- { 272, 4096, 32}, // 28 15 1.96% 0.22% --------- 6.25% ----------- - { 288, 4096, 32}, // 29 14 3.17% 0.21% --------- 5.88% ----------- + { 288, 4096, 32}, // 29 14 3.17% 0.22% --------- 5.88% ----------- { 312, 4096, 32}, // 30 13 2.56% 0.21% --------- 8.33% ----------- { 336, 4096, 32}, // 31 12 3.17% 0.21% --------- 7.69% ----------- { 368, 4096, 32}, // 32 11 2.77% 0.21% --------- 9.52% ----------- @@ -410,7 +410,7 @@ static constexpr SizeClassInfo List[] = { { 1664, 20480, 32}, // 51 12 2.88% 0.15% --------- 8.33% ----------- { 1792, 28672, 32}, // 52 16 0.22% 0.14% --------- 7.69% ----------- { 2048, 4096, 32}, // 53 2 1.56% 0.13% --------- 14.29% ----------- - { 2176, 32768, 30}, // 54 15 0.59% 0.12% --------- 6.25% ----------- + { 2176, 32768, 30}, // 54 15 0.59% 0.13% --------- 6.25% ----------- { 2304, 16384, 28}, // 55 7 1.98% 0.12% --------- 5.88% ----------- { 2432, 24576, 26}, // 56 10 1.32% 0.11% --------- 5.56% ----------- { 2560, 20480, 25}, // 57 8 0.31% 0.11% --------- 5.26% ----------- @@ -464,7 +464,7 @@ static constexpr SizeClassInfo List[] = { { 144, 8192, 32}, // 10 56 2.38% 0.42% --------- 12.50% ----------- { 160, 8192, 32}, // 11 51 1.18% 0.42% --------- 11.11% ----------- { 176, 8192, 32}, // 12 46 1.98% 0.42% --------- 10.00% ----------- - { 192, 8192, 32}, // 13 42 2.38% 0.41% --------- 9.09% ----------- + { 192, 8192, 32}, // 13 42 2.38% 0.42% --------- 9.09% ----------- { 208, 8192, 32}, // 14 39 1.78% 0.41% --------- 8.33% ----------- { 224, 8192, 32}, // 15 36 2.38% 0.41% --------- 7.69% ----------- { 240, 8192, 32}, // 16 34 1.18% 0.41% --------- 7.14% ----------- @@ -475,7 +475,7 @@ static constexpr SizeClassInfo List[] = { { 320, 8192, 32}, // 21 25 3.20% 0.41% --------- 5.26% ----------- { 352, 8192, 32}, // 22 23 1.98% 0.41% --------- 10.00% ----------- { 384, 8192, 32}, // 23 21 2.38% 0.41% --------- 9.09% ----------- - { 400, 8192, 32}, // 24 20 3.20% 0.40% --------- 4.17% ----------- + { 400, 8192, 32}, // 24 20 3.20% 0.41% --------- 4.17% ----------- { 448, 8192, 32}, // 25 18 2.38% 0.40% --------- 12.00% ----------- { 480, 8192, 32}, // 26 17 1.18% 0.40% --------- 7.14% ----------- { 512, 8192, 32}, // 27 16 0.78% 0.40% --------- 6.67% ----------- @@ -483,9 +483,9 @@ static constexpr SizeClassInfo List[] = { { 624, 8192, 32}, // 29 13 1.78% 0.39% --------- 8.33% ----------- { 672, 8192, 32}, // 30 12 2.38% 0.39% --------- 7.69% ----------- { 736, 8192, 32}, // 31 11 1.98% 0.39% --------- 9.52% ----------- - { 816, 8192, 32}, // 32 10 1.18% 0.38% --------- 10.87% ----------- + { 816, 8192, 32}, // 32 10 1.18% 0.39% --------- 10.87% ----------- { 896, 8192, 32}, // 33 9 2.38% 0.38% --------- 9.80% ----------- - { 1024, 8192, 32}, // 34 8 0.78% 0.37% --------- 14.29% ----------- + { 1024, 8192, 32}, // 34 8 0.78% 0.38% --------- 14.29% ----------- { 1152, 16384, 32}, // 35 14 1.98% 0.37% --------- 12.50% ----------- { 1280, 16384, 32}, // 36 12 7.08% 0.36% --------- 11.11% ----------- { 1408, 16384, 32}, // 37 11 6.20% 0.36% --------- 10.00% ----------- @@ -551,7 +551,7 @@ static constexpr SizeClassInfo List[] = { // bytes span_bytes batch class objs | fixed sampling int.frag | inc used { 0, 0, 0}, // 0 0 0.00% 0.00% --------- 0.00% ----------- { 8, 32768, 32}, // 1 4096 0.20% 1.60% --------- 0.00% ----------- - { 16, 32768, 32}, // 2 2048 0.20% 1.59% --------- 100.00% ----------- + { 16, 32768, 32}, // 2 2048 0.20% 1.60% --------- 100.00% ----------- { 32, 32768, 32}, // 3 1024 0.20% 1.59% --------- 100.00% ----------- { 48, 32768, 32}, // 4 682 0.29% 1.59% --------- 50.00% ----------- { 64, 32768, 32}, // 5 512 0.20% 1.59% --------- 33.33% ----------- @@ -584,7 +584,7 @@ static constexpr SizeClassInfo List[] = { { 896, 32768, 32}, // 32 36 1.79% 1.55% --------- 9.80% ----------- { 1024, 32768, 32}, // 33 32 0.20% 1.55% --------- 14.29% ----------- { 1152, 32768, 32}, // 34 28 1.79% 1.54% --------- 12.50% ----------- - { 1280, 32768, 32}, // 35 25 2.60% 1.53% --------- 11.11% ----------- + { 1280, 32768, 32}, // 35 25 2.60% 1.54% --------- 11.11% ----------- { 1408, 32768, 32}, // 36 23 1.38% 1.53% --------- 10.00% ----------- { 1536, 32768, 32}, // 37 21 1.79% 1.52% --------- 9.09% ----------- { 1664, 32768, 32}, // 38 19 3.85% 1.52% --------- 8.33% ----------- @@ -597,13 +597,13 @@ static constexpr SizeClassInfo List[] = { { 2688, 32768, 24}, // 45 12 1.79% 1.47% --------- 10.53% ----------- { 2944, 32768, 22}, // 46 11 1.38% 1.46% --------- 9.52% ----------- { 3200, 32768, 20}, // 47 10 2.60% 1.44% --------- 8.70% ----------- - { 3584, 32768, 18}, // 48 9 1.79% 1.42% --------- 12.00% ----------- + { 3584, 32768, 18}, // 48 9 1.79% 1.43% --------- 12.00% ----------- { 4096, 32768, 16}, // 49 8 0.20% 1.40% --------- 14.29% ----------- { 4352, 65536, 15}, // 50 15 0.49% 1.39% --------- 6.25% ----------- { 4608, 32768, 14}, // 51 7 1.79% 1.38% --------- 5.88% ----------- { 4992, 65536, 13}, // 52 13 1.08% 1.36% --------- 8.33% ----------- { 5376, 32768, 12}, // 53 6 1.79% 1.34% --------- 7.69% ----------- - { 5888, 65536, 11}, // 54 11 1.28% 1.31% --------- 9.52% ----------- + { 5888, 65536, 11}, // 54 11 1.28% 1.32% --------- 9.52% ----------- { 6528, 32768, 10}, // 55 5 0.59% 1.28% --------- 10.87% ----------- { 7168, 65536, 9}, // 56 9 1.69% 1.25% --------- 9.80% ----------- { 8192, 32768, 8}, // 57 4 0.20% 1.21% --------- 14.29% ----------- @@ -659,7 +659,7 @@ static constexpr SizeClassInfo List[] = { { 128, 262144, 32}, // 9 2048 0.02% 12.53% --------- 14.29% ----------- { 144, 262144, 32}, // 10 1820 0.05% 12.53% --------- 12.50% ----------- { 160, 262144, 32}, // 11 1638 0.05% 12.53% --------- 11.11% ----------- - { 176, 262144, 32}, // 12 1489 0.05% 12.52% --------- 10.00% ----------- + { 176, 262144, 32}, // 12 1489 0.05% 12.53% --------- 10.00% ----------- { 192, 262144, 32}, // 13 1365 0.05% 12.52% --------- 9.09% ----------- { 208, 262144, 32}, // 14 1260 0.05% 12.52% --------- 8.33% ----------- { 224, 262144, 32}, // 15 1170 0.05% 12.52% --------- 7.69% ----------- @@ -669,7 +669,7 @@ static constexpr SizeClassInfo List[] = { { 304, 262144, 32}, // 19 862 0.06% 12.52% --------- 11.76% ----------- { 336, 262144, 32}, // 20 780 0.05% 12.52% --------- 10.53% ----------- { 352, 262144, 32}, // 21 744 0.12% 12.52% --------- 4.76% ----------- - { 384, 262144, 32}, // 22 682 0.12% 12.51% --------- 9.09% ----------- + { 384, 262144, 32}, // 22 682 0.12% 12.52% --------- 9.09% ----------- { 416, 262144, 32}, // 23 630 0.05% 12.51% --------- 8.33% ----------- { 448, 262144, 32}, // 24 585 0.05% 12.51% --------- 7.69% ----------- { 512, 262144, 32}, // 25 512 0.02% 12.51% --------- 14.29% ----------- @@ -695,7 +695,7 @@ static constexpr SizeClassInfo List[] = { { 4096, 262144, 16}, // 45 64 0.02% 12.34% --------- 10.34% ----------- { 4480, 262144, 14}, // 46 58 0.91% 12.32% --------- 9.38% ----------- { 4864, 262144, 13}, // 47 53 1.71% 12.30% --------- 8.57% ----------- - { 5632, 262144, 11}, // 48 46 1.21% 12.26% --------- 15.79% ----------- + { 5632, 262144, 11}, // 48 46 1.21% 12.27% --------- 15.79% ----------- { 6144, 262144, 10}, // 49 42 1.61% 12.24% --------- 9.09% ----------- { 6656, 262144, 9}, // 50 39 1.01% 12.22% --------- 8.33% ----------- { 8192, 262144, 8}, // 51 32 0.02% 12.14% --------- 23.08% ----------- @@ -751,7 +751,7 @@ static constexpr SizeClassInfo List[] = { { 32, 4096, 32}, // 3 128 1.56% 0.23% --------- 100.00% ----------- { 48, 4096, 32}, // 4 85 1.96% 0.23% --------- 50.00% ----------- { 64, 4096, 32}, // 5 64 1.56% 0.23% --------- 33.33% ----------- - { 80, 4096, 32}, // 6 51 1.96% 0.22% --------- 25.00% ----------- + { 80, 4096, 32}, // 6 51 1.96% 0.23% --------- 25.00% ----------- { 96, 4096, 32}, // 7 42 3.17% 0.22% --------- 20.00% ----------- { 112, 4096, 32}, // 8 36 3.17% 0.22% --------- 16.67% ----------- { 128, 4096, 32}, // 9 32 1.56% 0.22% --------- 14.29% ----------- @@ -764,7 +764,7 @@ static constexpr SizeClassInfo List[] = { { 240, 4096, 32}, // 16 17 1.96% 0.22% --------- 7.14% ----------- { 256, 4096, 32}, // 17 16 1.56% 0.22% --------- 6.67% ----------- { 272, 4096, 32}, // 18 15 1.96% 0.22% --------- 6.25% ----------- - { 288, 4096, 32}, // 19 14 3.17% 0.21% --------- 5.88% ----------- + { 288, 4096, 32}, // 19 14 3.17% 0.22% --------- 5.88% ----------- { 304, 4096, 32}, // 20 13 5.26% 0.21% --------- 5.56% ----------- { 336, 4096, 32}, // 21 12 3.17% 0.21% --------- 10.53% ----------- { 368, 4096, 32}, // 22 11 2.77% 0.21% --------- 9.52% ----------- @@ -789,7 +789,7 @@ static constexpr SizeClassInfo List[] = { { 1664, 20480, 32}, // 41 12 2.88% 0.15% --------- 8.33% ----------- { 1792, 28672, 32}, // 42 16 0.22% 0.14% --------- 7.69% ----------- { 2048, 4096, 32}, // 43 2 1.56% 0.13% --------- 14.29% ----------- - { 2176, 32768, 30}, // 44 15 0.59% 0.12% --------- 6.25% ----------- + { 2176, 32768, 30}, // 44 15 0.59% 0.13% --------- 6.25% ----------- { 2304, 16384, 28}, // 45 7 1.98% 0.12% --------- 5.88% ----------- { 2432, 24576, 26}, // 46 10 1.32% 0.11% --------- 5.56% ----------- { 2560, 20480, 25}, // 47 8 0.31% 0.11% --------- 5.26% ----------- diff --git a/tcmalloc/flexible_no_reuse_size_classes.cc b/tcmalloc/flexible_no_reuse_size_classes.cc index 2ba546b22..8f071e4cf 100644 --- a/tcmalloc/flexible_no_reuse_size_classes.cc +++ b/tcmalloc/flexible_no_reuse_size_classes.cc @@ -83,7 +83,7 @@ static constexpr SizeClassInfo List[] = { { 144, 8192, 32}, // 18 56 2.38% 0.42% --------- 5.88% ----------- { 160, 8192, 32}, // 19 51 1.18% 0.42% --------- 11.11% ----------- { 176, 8192, 32}, // 20 46 1.98% 0.42% --------- 10.00% ----------- - { 192, 8192, 32}, // 21 42 2.38% 0.41% --------- 9.09% ----------- + { 192, 8192, 32}, // 21 42 2.38% 0.42% --------- 9.09% ----------- { 208, 8192, 32}, // 22 39 1.78% 0.41% --------- 8.33% ----------- { 224, 8192, 32}, // 23 36 2.38% 0.41% --------- 7.69% ----------- { 240, 8192, 32}, // 24 34 1.18% 0.41% --------- 7.14% ----------- @@ -100,9 +100,9 @@ static constexpr SizeClassInfo List[] = { { 584, 8192, 32}, // 35 14 0.98% 0.40% --------- 14.06% ----------- { 680, 8192, 32}, // 36 12 1.18% 0.39% --------- 16.44% ----------- { 744, 8192, 32}, // 37 11 0.88% 0.39% --------- 9.41% ----------- - { 816, 8192, 32}, // 38 10 1.18% 0.38% --------- 9.68% ----------- + { 816, 8192, 32}, // 38 10 1.18% 0.39% --------- 9.68% ----------- { 904, 8192, 32}, // 39 9 1.47% 0.38% --------- 10.78% ----------- - { 1024, 8192, 32}, // 40 8 0.78% 0.37% --------- 13.27% ----------- + { 1024, 8192, 32}, // 40 8 0.78% 0.38% --------- 13.27% ----------- { 1152, 16384, 32}, // 41 14 1.98% 0.37% --------- 12.50% ----------- { 1280, 16384, 32}, // 42 12 7.08% 0.36% --------- 11.11% ----------- { 1408, 16384, 32}, // 43 11 6.20% 0.36% --------- 10.00% ----------- @@ -162,7 +162,7 @@ static constexpr SizeClassInfo List[] = { // bytes span_bytes batch class objs | fixed sampling int.frag | inc used { 0, 0, 0}, // 0 0 0.00% 0.00% --------- 0.00% ----------- { 8, 32768, 32}, // 1 4096 0.20% 1.60% --------- 0.00% ----------- - { 16, 32768, 32}, // 2 2048 0.20% 1.59% --------- 100.00% ----------- + { 16, 32768, 32}, // 2 2048 0.20% 1.60% --------- 100.00% ----------- { 24, 32768, 32}, // 3 1365 0.22% 1.59% --------- 50.00% ----------- { 32, 32768, 32}, // 4 1024 0.20% 1.59% --------- 33.33% ----------- { 40, 32768, 32}, // 5 819 0.22% 1.59% --------- 25.00% ----------- @@ -201,7 +201,7 @@ static constexpr SizeClassInfo List[] = { { 840, 32768, 32}, // 38 39 0.22% 1.56% --------- 8.25% ----------- { 1024, 32768, 32}, // 39 32 0.20% 1.55% --------- 21.90% ----------- { 1152, 32768, 32}, // 40 28 1.79% 1.54% --------- 12.50% ----------- - { 1280, 32768, 32}, // 41 25 2.60% 1.53% --------- 11.11% ----------- + { 1280, 32768, 32}, // 41 25 2.60% 1.54% --------- 11.11% ----------- { 1408, 32768, 32}, // 42 23 1.38% 1.53% --------- 10.00% ----------- { 1536, 32768, 32}, // 43 21 1.79% 1.52% --------- 9.09% ----------- { 1792, 32768, 32}, // 44 18 1.79% 1.51% --------- 16.67% ----------- @@ -212,12 +212,12 @@ static constexpr SizeClassInfo List[] = { { 2688, 32768, 24}, // 49 12 1.79% 1.47% --------- 10.53% ----------- { 2944, 32768, 22}, // 50 11 1.38% 1.46% --------- 9.52% ----------- { 3200, 32768, 20}, // 51 10 2.60% 1.44% --------- 8.70% ----------- - { 3584, 32768, 18}, // 52 9 1.79% 1.42% --------- 12.00% ----------- + { 3584, 32768, 18}, // 52 9 1.79% 1.43% --------- 12.00% ----------- { 4096, 32768, 16}, // 53 8 0.20% 1.40% --------- 14.29% ----------- { 4608, 32768, 14}, // 54 7 1.79% 1.38% --------- 12.50% ----------- { 4992, 65536, 13}, // 55 13 1.08% 1.36% --------- 8.33% ----------- { 5376, 32768, 12}, // 56 6 1.79% 1.34% --------- 7.69% ----------- - { 5888, 65536, 11}, // 57 11 1.28% 1.31% --------- 9.52% ----------- + { 5888, 65536, 11}, // 57 11 1.28% 1.32% --------- 9.52% ----------- { 6528, 32768, 10}, // 58 5 0.59% 1.28% --------- 10.87% ----------- { 7168, 65536, 9}, // 59 9 1.69% 1.25% --------- 9.80% ----------- { 8192, 32768, 8}, // 60 4 0.20% 1.21% --------- 14.29% ----------- @@ -278,7 +278,7 @@ static constexpr SizeClassInfo List[] = { { 136, 262144, 32}, // 17 1927 0.05% 12.53% --------- 6.25% ----------- { 152, 262144, 32}, // 18 1724 0.06% 12.53% --------- 11.76% ----------- { 160, 262144, 32}, // 19 1638 0.05% 12.53% --------- 5.26% ----------- - { 176, 262144, 32}, // 20 1489 0.05% 12.52% --------- 10.00% ----------- + { 176, 262144, 32}, // 20 1489 0.05% 12.53% --------- 10.00% ----------- { 192, 262144, 32}, // 21 1365 0.05% 12.52% --------- 9.09% ----------- { 208, 262144, 32}, // 22 1260 0.05% 12.52% --------- 8.33% ----------- { 240, 262144, 32}, // 23 1092 0.05% 12.52% --------- 15.38% ----------- @@ -286,7 +286,7 @@ static constexpr SizeClassInfo List[] = { { 272, 262144, 32}, // 25 963 0.10% 12.52% --------- 6.25% ----------- { 304, 262144, 32}, // 26 862 0.06% 12.52% --------- 11.76% ----------- { 352, 262144, 32}, // 27 744 0.12% 12.52% --------- 15.79% ----------- - { 384, 262144, 32}, // 28 682 0.12% 12.51% --------- 9.09% ----------- + { 384, 262144, 32}, // 28 682 0.12% 12.52% --------- 9.09% ----------- { 408, 262144, 32}, // 29 642 0.10% 12.51% --------- 6.25% ----------- { 448, 262144, 32}, // 30 585 0.05% 12.51% --------- 9.80% ----------- { 512, 262144, 32}, // 31 512 0.02% 12.51% --------- 14.29% ----------- @@ -310,7 +310,7 @@ static constexpr SizeClassInfo List[] = { { 4096, 262144, 16}, // 49 64 0.02% 12.34% --------- 10.34% ----------- { 4480, 262144, 14}, // 50 58 0.91% 12.32% --------- 9.38% ----------- { 4864, 262144, 13}, // 51 53 1.71% 12.30% --------- 8.57% ----------- - { 5632, 262144, 11}, // 52 46 1.21% 12.26% --------- 15.79% ----------- + { 5632, 262144, 11}, // 52 46 1.21% 12.27% --------- 15.79% ----------- { 6144, 262144, 10}, // 53 42 1.61% 12.24% --------- 9.09% ----------- { 6656, 262144, 9}, // 54 39 1.01% 12.22% --------- 8.33% ----------- { 8192, 262144, 8}, // 55 32 0.02% 12.14% --------- 23.08% ----------- @@ -366,7 +366,7 @@ static constexpr SizeClassInfo List[] = { { 56, 4096, 32}, // 7 73 1.76% 0.23% --------- 16.67% ----------- { 64, 4096, 32}, // 8 64 1.56% 0.23% --------- 14.29% ----------- { 72, 4096, 32}, // 9 56 3.17% 0.23% --------- 12.50% ----------- - { 80, 4096, 32}, // 10 51 1.96% 0.22% --------- 11.11% ----------- + { 80, 4096, 32}, // 10 51 1.96% 0.23% --------- 11.11% ----------- { 88, 4096, 32}, // 11 46 2.77% 0.22% --------- 10.00% ----------- { 96, 4096, 32}, // 12 42 3.17% 0.22% --------- 9.09% ----------- { 104, 4096, 32}, // 13 39 2.56% 0.22% --------- 8.33% ----------- @@ -387,7 +387,7 @@ static constexpr SizeClassInfo List[] = { { 240, 4096, 32}, // 28 17 1.96% 0.22% --------- 7.14% ----------- { 256, 4096, 32}, // 29 16 1.56% 0.22% --------- 6.67% ----------- { 272, 4096, 32}, // 30 15 1.96% 0.22% --------- 6.25% ----------- - { 288, 4096, 32}, // 31 14 3.17% 0.21% --------- 5.88% ----------- + { 288, 4096, 32}, // 31 14 3.17% 0.22% --------- 5.88% ----------- { 312, 4096, 32}, // 32 13 2.56% 0.21% --------- 8.33% ----------- { 336, 4096, 32}, // 33 12 3.17% 0.21% --------- 7.69% ----------- { 368, 4096, 32}, // 34 11 2.77% 0.21% --------- 9.52% ----------- @@ -412,7 +412,7 @@ static constexpr SizeClassInfo List[] = { { 1664, 20480, 32}, // 53 12 2.88% 0.15% --------- 8.33% ----------- { 1792, 28672, 32}, // 54 16 0.22% 0.14% --------- 7.69% ----------- { 2048, 4096, 32}, // 55 2 1.56% 0.13% --------- 14.29% ----------- - { 2176, 32768, 30}, // 56 15 0.59% 0.12% --------- 6.25% ----------- + { 2176, 32768, 30}, // 56 15 0.59% 0.13% --------- 6.25% ----------- { 2304, 16384, 28}, // 57 7 1.98% 0.12% --------- 5.88% ----------- { 2432, 24576, 26}, // 58 10 1.32% 0.11% --------- 5.56% ----------- { 2560, 20480, 25}, // 59 8 0.31% 0.11% --------- 5.26% ----------- @@ -466,7 +466,7 @@ static constexpr SizeClassInfo List[] = { { 144, 8192, 32}, // 10 56 2.38% 0.42% --------- 12.50% ----------- { 160, 8192, 32}, // 11 51 1.18% 0.42% --------- 11.11% ----------- { 176, 8192, 32}, // 12 46 1.98% 0.42% --------- 10.00% ----------- - { 192, 8192, 32}, // 13 42 2.38% 0.41% --------- 9.09% ----------- + { 192, 8192, 32}, // 13 42 2.38% 0.42% --------- 9.09% ----------- { 208, 8192, 32}, // 14 39 1.78% 0.41% --------- 8.33% ----------- { 224, 8192, 32}, // 15 36 2.38% 0.41% --------- 7.69% ----------- { 240, 8192, 32}, // 16 34 1.18% 0.41% --------- 7.14% ----------- @@ -477,7 +477,7 @@ static constexpr SizeClassInfo List[] = { { 320, 8192, 32}, // 21 25 3.20% 0.41% --------- 5.26% ----------- { 352, 8192, 32}, // 22 23 1.98% 0.41% --------- 10.00% ----------- { 384, 8192, 32}, // 23 21 2.38% 0.41% --------- 9.09% ----------- - { 400, 8192, 32}, // 24 20 3.20% 0.40% --------- 4.17% ----------- + { 400, 8192, 32}, // 24 20 3.20% 0.41% --------- 4.17% ----------- { 448, 8192, 32}, // 25 18 2.38% 0.40% --------- 12.00% ----------- { 480, 8192, 32}, // 26 17 1.18% 0.40% --------- 7.14% ----------- { 512, 8192, 32}, // 27 16 0.78% 0.40% --------- 6.67% ----------- @@ -485,9 +485,9 @@ static constexpr SizeClassInfo List[] = { { 624, 8192, 32}, // 29 13 1.78% 0.39% --------- 8.33% ----------- { 672, 8192, 32}, // 30 12 2.38% 0.39% --------- 7.69% ----------- { 736, 8192, 32}, // 31 11 1.98% 0.39% --------- 9.52% ----------- - { 816, 8192, 32}, // 32 10 1.18% 0.38% --------- 10.87% ----------- + { 816, 8192, 32}, // 32 10 1.18% 0.39% --------- 10.87% ----------- { 896, 8192, 32}, // 33 9 2.38% 0.38% --------- 9.80% ----------- - { 1024, 8192, 32}, // 34 8 0.78% 0.37% --------- 14.29% ----------- + { 1024, 8192, 32}, // 34 8 0.78% 0.38% --------- 14.29% ----------- { 1152, 16384, 32}, // 35 14 1.98% 0.37% --------- 12.50% ----------- { 1280, 16384, 32}, // 36 12 7.08% 0.36% --------- 11.11% ----------- { 1408, 16384, 32}, // 37 11 6.20% 0.36% --------- 10.00% ----------- @@ -553,7 +553,7 @@ static constexpr SizeClassInfo List[] = { // bytes span_bytes batch class objs | fixed sampling int.frag | inc used { 0, 0, 0}, // 0 0 0.00% 0.00% --------- 0.00% ----------- { 8, 32768, 32}, // 1 4096 0.20% 1.60% --------- 0.00% ----------- - { 16, 32768, 32}, // 2 2048 0.20% 1.59% --------- 100.00% ----------- + { 16, 32768, 32}, // 2 2048 0.20% 1.60% --------- 100.00% ----------- { 32, 32768, 32}, // 3 1024 0.20% 1.59% --------- 100.00% ----------- { 48, 32768, 32}, // 4 682 0.29% 1.59% --------- 50.00% ----------- { 64, 32768, 32}, // 5 512 0.20% 1.59% --------- 33.33% ----------- @@ -586,7 +586,7 @@ static constexpr SizeClassInfo List[] = { { 896, 32768, 32}, // 32 36 1.79% 1.55% --------- 9.80% ----------- { 1024, 32768, 32}, // 33 32 0.20% 1.55% --------- 14.29% ----------- { 1152, 32768, 32}, // 34 28 1.79% 1.54% --------- 12.50% ----------- - { 1280, 32768, 32}, // 35 25 2.60% 1.53% --------- 11.11% ----------- + { 1280, 32768, 32}, // 35 25 2.60% 1.54% --------- 11.11% ----------- { 1408, 32768, 32}, // 36 23 1.38% 1.53% --------- 10.00% ----------- { 1536, 32768, 32}, // 37 21 1.79% 1.52% --------- 9.09% ----------- { 1664, 32768, 32}, // 38 19 3.85% 1.52% --------- 8.33% ----------- @@ -599,13 +599,13 @@ static constexpr SizeClassInfo List[] = { { 2688, 32768, 24}, // 45 12 1.79% 1.47% --------- 10.53% ----------- { 2944, 32768, 22}, // 46 11 1.38% 1.46% --------- 9.52% ----------- { 3200, 32768, 20}, // 47 10 2.60% 1.44% --------- 8.70% ----------- - { 3584, 32768, 18}, // 48 9 1.79% 1.42% --------- 12.00% ----------- + { 3584, 32768, 18}, // 48 9 1.79% 1.43% --------- 12.00% ----------- { 4096, 32768, 16}, // 49 8 0.20% 1.40% --------- 14.29% ----------- { 4352, 65536, 15}, // 50 15 0.49% 1.39% --------- 6.25% ----------- { 4608, 32768, 14}, // 51 7 1.79% 1.38% --------- 5.88% ----------- { 4992, 65536, 13}, // 52 13 1.08% 1.36% --------- 8.33% ----------- { 5376, 32768, 12}, // 53 6 1.79% 1.34% --------- 7.69% ----------- - { 5888, 65536, 11}, // 54 11 1.28% 1.31% --------- 9.52% ----------- + { 5888, 65536, 11}, // 54 11 1.28% 1.32% --------- 9.52% ----------- { 6528, 32768, 10}, // 55 5 0.59% 1.28% --------- 10.87% ----------- { 7168, 65536, 9}, // 56 9 1.69% 1.25% --------- 9.80% ----------- { 8192, 32768, 8}, // 57 4 0.20% 1.21% --------- 14.29% ----------- @@ -661,7 +661,7 @@ static constexpr SizeClassInfo List[] = { { 128, 262144, 32}, // 9 2048 0.02% 12.53% --------- 14.29% ----------- { 144, 262144, 32}, // 10 1820 0.05% 12.53% --------- 12.50% ----------- { 160, 262144, 32}, // 11 1638 0.05% 12.53% --------- 11.11% ----------- - { 176, 262144, 32}, // 12 1489 0.05% 12.52% --------- 10.00% ----------- + { 176, 262144, 32}, // 12 1489 0.05% 12.53% --------- 10.00% ----------- { 192, 262144, 32}, // 13 1365 0.05% 12.52% --------- 9.09% ----------- { 208, 262144, 32}, // 14 1260 0.05% 12.52% --------- 8.33% ----------- { 224, 262144, 32}, // 15 1170 0.05% 12.52% --------- 7.69% ----------- @@ -671,7 +671,7 @@ static constexpr SizeClassInfo List[] = { { 304, 262144, 32}, // 19 862 0.06% 12.52% --------- 11.76% ----------- { 336, 262144, 32}, // 20 780 0.05% 12.52% --------- 10.53% ----------- { 352, 262144, 32}, // 21 744 0.12% 12.52% --------- 4.76% ----------- - { 384, 262144, 32}, // 22 682 0.12% 12.51% --------- 9.09% ----------- + { 384, 262144, 32}, // 22 682 0.12% 12.52% --------- 9.09% ----------- { 416, 262144, 32}, // 23 630 0.05% 12.51% --------- 8.33% ----------- { 448, 262144, 32}, // 24 585 0.05% 12.51% --------- 7.69% ----------- { 512, 262144, 32}, // 25 512 0.02% 12.51% --------- 14.29% ----------- @@ -697,7 +697,7 @@ static constexpr SizeClassInfo List[] = { { 4096, 262144, 16}, // 45 64 0.02% 12.34% --------- 10.34% ----------- { 4480, 262144, 14}, // 46 58 0.91% 12.32% --------- 9.38% ----------- { 4864, 262144, 13}, // 47 53 1.71% 12.30% --------- 8.57% ----------- - { 5632, 262144, 11}, // 48 46 1.21% 12.26% --------- 15.79% ----------- + { 5632, 262144, 11}, // 48 46 1.21% 12.27% --------- 15.79% ----------- { 6144, 262144, 10}, // 49 42 1.61% 12.24% --------- 9.09% ----------- { 6656, 262144, 9}, // 50 39 1.01% 12.22% --------- 8.33% ----------- { 8192, 262144, 8}, // 51 32 0.02% 12.14% --------- 23.08% ----------- @@ -753,7 +753,7 @@ static constexpr SizeClassInfo List[] = { { 32, 4096, 32}, // 3 128 1.56% 0.23% --------- 100.00% ----------- { 48, 4096, 32}, // 4 85 1.96% 0.23% --------- 50.00% ----------- { 64, 4096, 32}, // 5 64 1.56% 0.23% --------- 33.33% ----------- - { 80, 4096, 32}, // 6 51 1.96% 0.22% --------- 25.00% ----------- + { 80, 4096, 32}, // 6 51 1.96% 0.23% --------- 25.00% ----------- { 96, 4096, 32}, // 7 42 3.17% 0.22% --------- 20.00% ----------- { 112, 4096, 32}, // 8 36 3.17% 0.22% --------- 16.67% ----------- { 128, 4096, 32}, // 9 32 1.56% 0.22% --------- 14.29% ----------- @@ -766,7 +766,7 @@ static constexpr SizeClassInfo List[] = { { 240, 4096, 32}, // 16 17 1.96% 0.22% --------- 7.14% ----------- { 256, 4096, 32}, // 17 16 1.56% 0.22% --------- 6.67% ----------- { 272, 4096, 32}, // 18 15 1.96% 0.22% --------- 6.25% ----------- - { 288, 4096, 32}, // 19 14 3.17% 0.21% --------- 5.88% ----------- + { 288, 4096, 32}, // 19 14 3.17% 0.22% --------- 5.88% ----------- { 304, 4096, 32}, // 20 13 5.26% 0.21% --------- 5.56% ----------- { 336, 4096, 32}, // 21 12 3.17% 0.21% --------- 10.53% ----------- { 368, 4096, 32}, // 22 11 2.77% 0.21% --------- 9.52% ----------- @@ -791,7 +791,7 @@ static constexpr SizeClassInfo List[] = { { 1664, 20480, 32}, // 41 12 2.88% 0.15% --------- 8.33% ----------- { 1792, 28672, 32}, // 42 16 0.22% 0.14% --------- 7.69% ----------- { 2048, 4096, 32}, // 43 2 1.56% 0.13% --------- 14.29% ----------- - { 2176, 32768, 30}, // 44 15 0.59% 0.12% --------- 6.25% ----------- + { 2176, 32768, 30}, // 44 15 0.59% 0.13% --------- 6.25% ----------- { 2304, 16384, 28}, // 45 7 1.98% 0.12% --------- 5.88% ----------- { 2432, 24576, 26}, // 46 10 1.32% 0.11% --------- 5.56% ----------- { 2560, 20480, 25}, // 47 8 0.31% 0.11% --------- 5.26% ----------- diff --git a/tcmalloc/flexible_size_classes.cc b/tcmalloc/flexible_size_classes.cc index 4e6d12826..c52f4a2c2 100644 --- a/tcmalloc/flexible_size_classes.cc +++ b/tcmalloc/flexible_size_classes.cc @@ -84,8 +84,8 @@ static constexpr SizeClassInfo List[] = { { 448, 8192, 32}, // 19 18 2.38% 0.40% --------- 27.27% ----------- { 512, 8192, 32}, // 20 16 0.78% 0.40% --------- 14.29% ----------- { 680, 8192, 32}, // 21 12 1.18% 0.39% --------- 32.81% ----------- - { 816, 8192, 32}, // 22 10 1.18% 0.38% --------- 20.00% ----------- - { 1024, 8192, 32}, // 23 8 0.78% 0.37% --------- 25.49% ----------- + { 816, 8192, 32}, // 22 10 1.18% 0.39% --------- 20.00% ----------- + { 1024, 8192, 32}, // 23 8 0.78% 0.38% --------- 25.49% ----------- { 1152, 16384, 32}, // 24 14 1.98% 0.37% --------- 12.50% ----------- { 1280, 16384, 32}, // 25 12 7.08% 0.36% --------- 11.11% ----------- { 1536, 24576, 32}, // 26 16 0.26% 0.35% --------- 20.00% ----------- @@ -133,7 +133,7 @@ static constexpr SizeClassInfo List[] = { // bytes span_bytes batch class objs | fixed sampling int.frag | inc used { 0, 0, 0}, // 0 0 0.00% 0.00% --------- 0.00% ----------- { 8, 32768, 32}, // 1 4096 0.20% 1.60% --------- 0.00% ----------- - { 16, 32768, 32}, // 2 2048 0.20% 1.59% --------- 100.00% ----------- + { 16, 32768, 32}, // 2 2048 0.20% 1.60% --------- 100.00% ----------- { 24, 32768, 32}, // 3 1365 0.22% 1.59% --------- 50.00% ----------- { 32, 32768, 32}, // 4 1024 0.20% 1.59% --------- 33.33% ----------- { 40, 32768, 32}, // 5 819 0.22% 1.59% --------- 25.00% ----------- @@ -214,7 +214,7 @@ static constexpr SizeClassInfo List[] = { { 112, 262144, 32}, // 11 2340 0.05% 12.53% --------- 16.67% ----------- { 128, 262144, 32}, // 12 2048 0.02% 12.53% --------- 14.29% ----------- { 152, 262144, 32}, // 13 1724 0.06% 12.53% --------- 18.75% ----------- - { 176, 262144, 32}, // 14 1489 0.05% 12.52% --------- 15.79% ----------- + { 176, 262144, 32}, // 14 1489 0.05% 12.53% --------- 15.79% ----------- { 208, 262144, 32}, // 15 1260 0.05% 12.52% --------- 18.18% ----------- { 256, 262144, 32}, // 16 1024 0.02% 12.52% --------- 23.08% ----------- { 304, 262144, 32}, // 17 862 0.06% 12.52% --------- 18.75% ----------- @@ -285,14 +285,14 @@ static constexpr SizeClassInfo List[] = { { 56, 4096, 32}, // 7 73 1.76% 0.23% --------- 16.67% ----------- { 64, 4096, 32}, // 8 64 1.56% 0.23% --------- 14.29% ----------- { 72, 4096, 32}, // 9 56 3.17% 0.23% --------- 12.50% ----------- - { 80, 4096, 32}, // 10 51 1.96% 0.22% --------- 11.11% ----------- + { 80, 4096, 32}, // 10 51 1.96% 0.23% --------- 11.11% ----------- { 96, 4096, 32}, // 11 42 3.17% 0.22% --------- 20.00% ----------- { 128, 4096, 32}, // 12 32 1.56% 0.22% --------- 33.33% ----------- { 144, 4096, 32}, // 13 28 3.17% 0.22% --------- 12.50% ----------- { 160, 4096, 32}, // 14 25 4.00% 0.22% --------- 11.11% ----------- { 224, 4096, 32}, // 15 18 3.17% 0.22% --------- 40.00% ----------- { 256, 4096, 32}, // 16 16 1.56% 0.22% --------- 14.29% ----------- - { 288, 4096, 32}, // 17 14 3.17% 0.21% --------- 12.50% ----------- + { 288, 4096, 32}, // 17 14 3.17% 0.22% --------- 12.50% ----------- { 368, 4096, 32}, // 18 11 2.77% 0.21% --------- 27.78% ----------- { 512, 4096, 32}, // 19 8 1.56% 0.20% --------- 39.13% ----------- { 584, 8192, 32}, // 20 14 0.98% 0.20% --------- 14.06% ----------- @@ -344,8 +344,8 @@ static constexpr SizeClassInfo List[] = { { 448, 8192, 32}, // 16 18 2.38% 0.40% --------- 27.27% ----------- { 512, 8192, 32}, // 17 16 0.78% 0.40% --------- 14.29% ----------- { 672, 8192, 32}, // 18 12 2.38% 0.39% --------- 31.25% ----------- - { 816, 8192, 32}, // 19 10 1.18% 0.38% --------- 21.43% ----------- - { 1024, 8192, 32}, // 20 8 0.78% 0.37% --------- 25.49% ----------- + { 816, 8192, 32}, // 19 10 1.18% 0.39% --------- 21.43% ----------- + { 1024, 8192, 32}, // 20 8 0.78% 0.38% --------- 25.49% ----------- { 1152, 16384, 32}, // 21 14 1.98% 0.37% --------- 12.50% ----------- { 1280, 16384, 32}, // 22 12 7.08% 0.36% --------- 11.11% ----------- { 1536, 24576, 32}, // 23 16 0.26% 0.35% --------- 20.00% ----------- @@ -393,7 +393,7 @@ static constexpr SizeClassInfo List[] = { // bytes span_bytes batch class objs | fixed sampling int.frag | inc used { 0, 0, 0}, // 0 0 0.00% 0.00% --------- 0.00% ----------- { 8, 32768, 32}, // 1 4096 0.20% 1.60% --------- 0.00% ----------- - { 16, 32768, 32}, // 2 2048 0.20% 1.59% --------- 100.00% ----------- + { 16, 32768, 32}, // 2 2048 0.20% 1.60% --------- 100.00% ----------- { 32, 32768, 32}, // 3 1024 0.20% 1.59% --------- 100.00% ----------- { 48, 32768, 32}, // 4 682 0.29% 1.59% --------- 50.00% ----------- { 64, 32768, 32}, // 5 512 0.20% 1.59% --------- 33.33% ----------- @@ -471,7 +471,7 @@ static constexpr SizeClassInfo List[] = { { 208, 262144, 32}, // 12 1260 0.05% 12.52% --------- 8.33% ----------- { 256, 262144, 32}, // 13 1024 0.02% 12.52% --------- 23.08% ----------- { 304, 262144, 32}, // 14 862 0.06% 12.52% --------- 18.75% ----------- - { 384, 262144, 32}, // 15 682 0.12% 12.51% --------- 26.32% ----------- + { 384, 262144, 32}, // 15 682 0.12% 12.52% --------- 26.32% ----------- { 448, 262144, 32}, // 16 585 0.05% 12.51% --------- 16.67% ----------- { 512, 262144, 32}, // 17 512 0.02% 12.51% --------- 14.29% ----------- { 608, 262144, 32}, // 18 431 0.06% 12.50% --------- 18.75% ----------- @@ -532,14 +532,14 @@ static constexpr SizeClassInfo List[] = { { 32, 4096, 32}, // 3 128 1.56% 0.23% --------- 100.00% ----------- { 48, 4096, 32}, // 4 85 1.96% 0.23% --------- 50.00% ----------- { 64, 4096, 32}, // 5 64 1.56% 0.23% --------- 33.33% ----------- - { 80, 4096, 32}, // 6 51 1.96% 0.22% --------- 25.00% ----------- + { 80, 4096, 32}, // 6 51 1.96% 0.23% --------- 25.00% ----------- { 96, 4096, 32}, // 7 42 3.17% 0.22% --------- 20.00% ----------- { 128, 4096, 32}, // 8 32 1.56% 0.22% --------- 33.33% ----------- { 144, 4096, 32}, // 9 28 3.17% 0.22% --------- 12.50% ----------- { 160, 4096, 32}, // 10 25 4.00% 0.22% --------- 11.11% ----------- { 224, 4096, 32}, // 11 18 3.17% 0.22% --------- 40.00% ----------- { 256, 4096, 32}, // 12 16 1.56% 0.22% --------- 14.29% ----------- - { 288, 4096, 32}, // 13 14 3.17% 0.21% --------- 12.50% ----------- + { 288, 4096, 32}, // 13 14 3.17% 0.22% --------- 12.50% ----------- { 368, 4096, 32}, // 14 11 2.77% 0.21% --------- 27.78% ----------- { 512, 4096, 32}, // 15 8 1.56% 0.20% --------- 39.13% ----------- { 576, 8192, 32}, // 16 14 2.38% 0.20% --------- 12.50% ----------- diff --git a/tcmalloc/internal/logging.h b/tcmalloc/internal/logging.h index 95360aa6b..be2b35749 100644 --- a/tcmalloc/internal/logging.h +++ b/tcmalloc/internal/logging.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -80,11 +81,14 @@ struct StackTrace { absl::Time allocation_time; TokenId token_id; + int16_t cpu_id; Profile::Sample::GuardedStatus guarded_status; // How the memory was allocated (new/malloc/etc.) AllocationType allocation_type; + int16_t vcpu_id; + pid_t thread_id; // If not nullptr, this is the start address of the span corresponding to this // sampled allocation. This may be nullptr for cases where it is not useful diff --git a/tcmalloc/size_classes.cc b/tcmalloc/size_classes.cc index 9c307d4fd..de091ba26 100644 --- a/tcmalloc/size_classes.cc +++ b/tcmalloc/size_classes.cc @@ -84,7 +84,7 @@ static constexpr SizeClassInfo List[] = { { 576, 8192, 32}, // 19 14 2.38% 0.40% --------- 12.50% ----------- { 704, 8192, 32}, // 20 11 6.61% 0.39% --------- 22.22% ----------- { 896, 8192, 32}, // 21 9 2.38% 0.38% --------- 27.27% ----------- - { 1024, 8192, 32}, // 22 8 0.78% 0.37% --------- 14.29% ----------- + { 1024, 8192, 32}, // 22 8 0.78% 0.38% --------- 14.29% ----------- { 1152, 16384, 32}, // 23 14 1.98% 0.37% --------- 12.50% ----------- { 1408, 16384, 32}, // 24 11 6.20% 0.36% --------- 22.22% ----------- { 1792, 16384, 32}, // 25 9 1.98% 0.34% --------- 27.27% ----------- @@ -127,7 +127,7 @@ static constexpr SizeClassInfo List[] = { // bytes span_bytes batch class objs | fixed sampling int.frag | inc used { 0, 0, 0}, // 0 0 0.00% 0.00% --------- 0.00% ----------- { 8, 32768, 32}, // 1 4096 0.20% 1.60% --------- 0.00% ----------- - { 16, 32768, 32}, // 2 2048 0.20% 1.59% --------- 100.00% ----------- + { 16, 32768, 32}, // 2 2048 0.20% 1.60% --------- 100.00% ----------- { 24, 32768, 32}, // 3 1365 0.22% 1.59% --------- 50.00% ----------- { 32, 32768, 32}, // 4 1024 0.20% 1.59% --------- 33.33% ----------- { 48, 32768, 32}, // 5 682 0.29% 1.59% --------- 50.00% ----------- @@ -198,11 +198,11 @@ static constexpr SizeClassInfo List[] = { { 120, 262144, 32}, // 9 2184 0.05% 12.53% --------- 25.00% ----------- { 128, 262144, 32}, // 10 2048 0.02% 12.53% --------- 6.67% ----------- { 152, 262144, 32}, // 11 1724 0.06% 12.53% --------- 18.75% ----------- - { 176, 262144, 32}, // 12 1489 0.05% 12.52% --------- 15.79% ----------- + { 176, 262144, 32}, // 12 1489 0.05% 12.53% --------- 15.79% ----------- { 208, 262144, 32}, // 13 1260 0.05% 12.52% --------- 18.18% ----------- { 256, 262144, 32}, // 14 1024 0.02% 12.52% --------- 23.08% ----------- { 304, 262144, 32}, // 15 862 0.06% 12.52% --------- 18.75% ----------- - { 384, 262144, 32}, // 16 682 0.12% 12.51% --------- 26.32% ----------- + { 384, 262144, 32}, // 16 682 0.12% 12.52% --------- 26.32% ----------- { 448, 262144, 32}, // 17 585 0.05% 12.51% --------- 16.67% ----------- { 512, 262144, 32}, // 18 512 0.02% 12.51% --------- 14.29% ----------- { 640, 262144, 32}, // 19 409 0.17% 12.50% --------- 25.00% ----------- @@ -257,13 +257,13 @@ static constexpr SizeClassInfo List[] = { { 32, 4096, 32}, // 4 128 1.56% 0.23% --------- 33.33% ----------- { 48, 4096, 32}, // 5 85 1.96% 0.23% --------- 50.00% ----------- { 64, 4096, 32}, // 6 64 1.56% 0.23% --------- 33.33% ----------- - { 80, 4096, 32}, // 7 51 1.96% 0.22% --------- 25.00% ----------- + { 80, 4096, 32}, // 7 51 1.96% 0.23% --------- 25.00% ----------- { 96, 4096, 32}, // 8 42 3.17% 0.22% --------- 20.00% ----------- { 128, 4096, 32}, // 9 32 1.56% 0.22% --------- 33.33% ----------- { 160, 4096, 32}, // 10 25 4.00% 0.22% --------- 25.00% ----------- { 224, 4096, 32}, // 11 18 3.17% 0.22% --------- 40.00% ----------- { 256, 4096, 32}, // 12 16 1.56% 0.22% --------- 14.29% ----------- - { 288, 4096, 32}, // 13 14 3.17% 0.21% --------- 12.50% ----------- + { 288, 4096, 32}, // 13 14 3.17% 0.22% --------- 12.50% ----------- { 368, 4096, 32}, // 14 11 2.77% 0.21% --------- 27.78% ----------- { 512, 4096, 32}, // 15 8 1.56% 0.20% --------- 39.13% ----------- { 576, 8192, 32}, // 16 14 2.38% 0.20% --------- 12.50% ----------- @@ -314,7 +314,7 @@ static constexpr SizeClassInfo List[] = { { 576, 8192, 32}, // 18 14 2.38% 0.40% --------- 12.50% ----------- { 704, 8192, 32}, // 19 11 6.61% 0.39% --------- 22.22% ----------- { 896, 8192, 32}, // 20 9 2.38% 0.38% --------- 27.27% ----------- - { 1024, 8192, 32}, // 21 8 0.78% 0.37% --------- 14.29% ----------- + { 1024, 8192, 32}, // 21 8 0.78% 0.38% --------- 14.29% ----------- { 1152, 16384, 32}, // 22 14 1.98% 0.37% --------- 12.50% ----------- { 1408, 16384, 32}, // 23 11 6.20% 0.36% --------- 22.22% ----------- { 1792, 16384, 32}, // 24 9 1.98% 0.34% --------- 27.27% ----------- @@ -357,7 +357,7 @@ static constexpr SizeClassInfo List[] = { // bytes span_bytes batch class objs | fixed sampling int.frag | inc used { 0, 0, 0}, // 0 0 0.00% 0.00% --------- 0.00% ----------- { 8, 32768, 32}, // 1 4096 0.20% 1.60% --------- 0.00% ----------- - { 16, 32768, 32}, // 2 2048 0.20% 1.59% --------- 100.00% ----------- + { 16, 32768, 32}, // 2 2048 0.20% 1.60% --------- 100.00% ----------- { 32, 32768, 32}, // 3 1024 0.20% 1.59% --------- 100.00% ----------- { 48, 32768, 32}, // 4 682 0.29% 1.59% --------- 50.00% ----------- { 64, 32768, 32}, // 5 512 0.20% 1.59% --------- 33.33% ----------- @@ -429,7 +429,7 @@ static constexpr SizeClassInfo List[] = { { 192, 262144, 32}, // 10 1365 0.05% 12.52% --------- 20.00% ----------- { 256, 262144, 32}, // 11 1024 0.02% 12.52% --------- 33.33% ----------- { 304, 262144, 32}, // 12 862 0.06% 12.52% --------- 18.75% ----------- - { 384, 262144, 32}, // 13 682 0.12% 12.51% --------- 26.32% ----------- + { 384, 262144, 32}, // 13 682 0.12% 12.52% --------- 26.32% ----------- { 448, 262144, 32}, // 14 585 0.05% 12.51% --------- 16.67% ----------- { 512, 262144, 32}, // 15 512 0.02% 12.51% --------- 14.29% ----------- { 640, 262144, 32}, // 16 409 0.17% 12.50% --------- 25.00% ----------- @@ -485,13 +485,13 @@ static constexpr SizeClassInfo List[] = { { 32, 4096, 32}, // 3 128 1.56% 0.23% --------- 100.00% ----------- { 48, 4096, 32}, // 4 85 1.96% 0.23% --------- 50.00% ----------- { 64, 4096, 32}, // 5 64 1.56% 0.23% --------- 33.33% ----------- - { 80, 4096, 32}, // 6 51 1.96% 0.22% --------- 25.00% ----------- + { 80, 4096, 32}, // 6 51 1.96% 0.23% --------- 25.00% ----------- { 96, 4096, 32}, // 7 42 3.17% 0.22% --------- 20.00% ----------- { 128, 4096, 32}, // 8 32 1.56% 0.22% --------- 33.33% ----------- { 160, 4096, 32}, // 9 25 4.00% 0.22% --------- 25.00% ----------- { 224, 4096, 32}, // 10 18 3.17% 0.22% --------- 40.00% ----------- { 256, 4096, 32}, // 11 16 1.56% 0.22% --------- 14.29% ----------- - { 288, 4096, 32}, // 12 14 3.17% 0.21% --------- 12.50% ----------- + { 288, 4096, 32}, // 12 14 3.17% 0.22% --------- 12.50% ----------- { 368, 4096, 32}, // 13 11 2.77% 0.21% --------- 27.78% ----------- { 512, 4096, 32}, // 14 8 1.56% 0.20% --------- 39.13% ----------- { 576, 8192, 32}, // 15 14 2.38% 0.20% --------- 12.50% ----------- diff --git a/tcmalloc/testing/deallocation_profiler_test.cc b/tcmalloc/testing/deallocation_profiler_test.cc index 703757a1a..51e16e0e6 100644 --- a/tcmalloc/testing/deallocation_profiler_test.cc +++ b/tcmalloc/testing/deallocation_profiler_test.cc @@ -946,4 +946,37 @@ TEST(EventTracingTest, DefaultMemoryLimitTruncation) { EXPECT_THAT(profile, testing::Not(ContainsAlloc(kLateSize))); } +TEST(DeallocationProfilerTest, SeedingFromOtherThreadPreservesThreadMatched) { + if (CheckerIsActive()) { + return; + } + + tcmalloc::ScopedProfileSamplingInterval test_sample_interval(1); + constexpr size_t kAllocSize = 1024 * 1024 + 512 * 1024; + + // Allocate on main thread BEFORE starting the deallocation profiler. + void* ptr = SingleAlloc(1, kAllocSize); + ASSERT_NE(ptr, nullptr); + + // Start the deallocation profiler on a DIFFERENT thread, which seeds + // pre-existing live allocations from sampled_allocation_recorder(). + tcmalloc::MallocExtension::AllocationProfilingToken token; + std::thread starter( + [&]() { token = tcmalloc::MallocExtension::StartLifetimeProfiling(); }); + starter.join(); + + // Free on the main thread (same thread that allocated `ptr`). + SingleDealloc(1, ptr); + + tcmalloc::Profile profile = std::move(token).Stop(); + int matched_samples = 0; + profile.Iterate([&](const tcmalloc::Profile::Sample& s) { + if (s.requested_size == kAllocSize && !s.is_censored && s.count > 0 && + s.allocator_deallocator_thread_matched.value_or(false)) { + ++matched_samples; + } + }); + EXPECT_GE(matched_samples, 1); +} + } // namespace