JIT: assertion failure in pycore_backoff.h

[devdanzin]

Crash report

What happened?

It's possible to trigger an assertion failure in a patched (diff follows) JIT build by running the code below. It's a finicky issue to reproduce, so the MRE isn't as compact as I'd like and requires a special environment, but changing minor details stops reproducing the abort.

To prepare the environment, run:

# Create a venv, then:
git clone https://github.com/devdanzin/lafleur.git
git clone https://github.com/devdanzin/fusil.git
pip install -e lafleur
pip install -e fusil

Also, the way the script is invoked interferes with reproduction: calling it with pythoninside the active venv or with path/to/venv/bin/python works, while path/to/built/python doesn't work.

Given the sensitivity of the MRE to small changes, please let me know whether you can reproduce this issue.

The diff applied was (it might be possible to reduce this diff and keep reproducing):

diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h
index 71066f1bd9f..e4a31f76ff2 100644
--- a/Include/internal/pycore_backoff.h
+++ b/Include/internal/pycore_backoff.h
@@ -112,8 +112,8 @@ trigger_backoff_counter(void)
 // For example, 4095 does not work for the nqueens benchmark on pyperformance
 // as we always end up tracing the loop iteration's
 // exhaustion iteration. Which aborts our current tracer.
-#define JUMP_BACKWARD_INITIAL_VALUE 4000
-#define JUMP_BACKWARD_INITIAL_BACKOFF 12
+#define JUMP_BACKWARD_INITIAL_VALUE 63
+#define JUMP_BACKWARD_INITIAL_BACKOFF 6
 static inline _Py_BackoffCounter
 initial_jump_backoff_counter(void)
 {
@@ -125,8 +125,8 @@ initial_jump_backoff_counter(void)
  * Must be larger than ADAPTIVE_COOLDOWN_VALUE,
  * otherwise when a side exit warms up we may construct
  * a new trace before the Tier 1 code has properly re-specialized. */
-#define SIDE_EXIT_INITIAL_VALUE 4000
-#define SIDE_EXIT_INITIAL_BACKOFF 12
+#define SIDE_EXIT_INITIAL_VALUE 63
+#define SIDE_EXIT_INITIAL_BACKOFF 6

 static inline _Py_BackoffCounter
 initial_temperature_backoff_counter(void)
diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
index e7177552cf6..0d76a5a3df0 100644
--- a/Include/internal/pycore_optimizer.h
+++ b/Include/internal/pycore_optimizer.h
@@ -86,7 +86,7 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp);
 // Used as the threshold to trigger executor invalidation when
 // executor_creation_counter is greater than this value.
 // This value is arbitrary and was not optimized.
-#define JIT_CLEANUP_THRESHOLD 1000
+#define JIT_CLEANUP_THRESHOLD 10000

 int _Py_uop_analyze_and_optimize(
     PyFunctionObject *func,
@@ -118,7 +118,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
 }

 // Holds locals, stack, locals, stack ... co_consts (in that order)
-#define MAX_ABSTRACT_INTERP_SIZE 4096
+#define MAX_ABSTRACT_INTERP_SIZE 8192

 #define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5)

@@ -129,7 +129,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
 // progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this
 // is the "maximum amount of polymorphism" that an isolated trace tree can
 // handle before rejoining the rest of the program.
-#define MAX_CHAIN_DEPTH 4
+#define MAX_CHAIN_DEPTH 16

 /* Symbols */
 /* See explanation in optimizer_symbols.c */
diff --git a/Python/optimizer.c b/Python/optimizer.c
index 9db894f0bf0..14cbf670dec 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -509,7 +509,7 @@ guard_ip_uop[MAX_UOP_ID + 1] = {


 #define CONFIDENCE_RANGE 1000
-#define CONFIDENCE_CUTOFF 333
+#define CONFIDENCE_CUTOFF 100

 #ifdef Py_DEBUG
 #define DPRINTF(level, ...) \

The admitedly large, but much reduced, MRE is:

import builtins
import gc
import math  # Not used


def g():
    gc.set_threshold(5)
    original_len = builtins.len
    try:
        builtins.len = lambda x: 'evil_string'
        _ = len([])
    except Exception:
        pass
    finally:
        builtins.len = original_len

THRESHOLD = 300

def f1():

    class Base1:
        def __init__(self):
            pass

    class Sub1(Base1):
        def __init__(self):
            super().__init__()
        def f(self):
            super().f()
            if False:
                Sub1.__bases__ = (object,)

    instance_1 = Sub1()
    for _ in range(THRESHOLD):
        try:
            _ = instance_1.f()
        except AttributeError:
            pass
    Sub1.__bases__ = (object,)

    g()


    class Dummy1:
        def __init__(self):
            pass
        def f(self):
            pass

    class Sub2(Base1):
        def __init__(self):
            super().__init__()
        def f(self):
            super().f()
            if False:
                Sub2.__bases__ = (object,)

    instance_2 = Sub2()
    for _ in range(THRESHOLD):
        try:
            _ = instance_2.f()
        except AttributeError:
            pass
    Sub2.__bases__ = (object,)

    class Base2:
        def __init__(self):
            pass
        def f(self):
            pass

    class Dummy2:
        def __init__(self):
            pass
        def f(self):
            pass

    class Sub4(Base1):
        def __init__(self):
            super().__init__()
        def f(self):
            super().f()
            if False:
                Sub4.__bases__ = (object,)

    instance_4 = Sub4()
    for _ in range(THRESHOLD):
        try:
            _ = instance_4.f()
        except AttributeError:
            pass
    Sub4.__bases__ = (object,)

    class Sub3(Base2):
        def __init__(self):
            pass
        def f(self):
            super().f()
            if self.non_existing_attribute:
                Sub3.__bases__ = (object,)


    instance_3 = Sub3()
    for _ in range(THRESHOLD):
        try:
            _ = instance_3.f()
        except AttributeError:
            pass
    Sub3.__bases__ = (object,)
    for _ in range(50):
        try:
            _ = instance_3.f()
        except AttributeError:
            pass

for i_f1 in range(300):
    print(i_f1)
    f1()

The backtrace (with printing of iteration number) is:

1
[...]
64
65
python: ./Include/internal/pycore_backoff.h:65: _Py_BackoffCounter restart_backoff_counter(_Py_BackoffCounter): Assertion `!is_unreachable_backoff_counter(counter)' failed.

Program received signal SIGABRT, Aborted.

#0  __pthread_kill_implementation (threadid=<optimized out>, signo=6, no_tid=0) at ./nptl/pthread_kill.c:44
#1  __pthread_kill_internal (threadid=<optimized out>, signo=6) at ./nptl/pthread_kill.c:89
#2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6) at ./nptl/pthread_kill.c:100
#3  0x00007ffff7c45e2e in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#4  0x00007ffff7c28888 in __GI_abort () at ./stdlib/abort.c:77
#5  0x00007ffff7c287f0 in __assert_fail_base (fmt=<optimized out>, assertion=<optimized out>, file=<optimized out>, line=<optimized out>, function=<optimized out>) at ./assert/assert.c:118
#6  0x00007ffff7c3c19f in __assert_fail (assertion=<optimized out>, file=<optimized out>, line=<optimized out>, function=<optimized out>) at ./assert/assert.c:127
#7  0x0000555555ec14c5 in restart_backoff_counter (counter=...) at ./Include/internal/pycore_backoff.h:65
#8  0x0000555555ec14c5 in stop_tracing_and_jit (tstate=0x555556aebf00 <_PyRuntime+358560>, frame=frame@entry=0x7e8ff6de5298)
#9  0x0000555555e6e4f3 in _PyEval_EvalFrameDefault (tstate=<optimized out>, frame=<optimized out>, throwflag=<optimized out>) at Python/generated_cases.c.h:11712
#10 0x0000555555e4a67b in _PyEval_EvalFrame (tstate=0x555556aebf00 <_PyRuntime+358560>, frame=0x7e8ff6de5220, throwflag=0) at ./Include/internal/pycore_ceval.h:121
#11 _PyEval_Vector (tstate=<optimized out>, func=<optimized out>, locals=<optimized out>, args=<optimized out>, argcount=<optimized out>, kwnames=0x0) at Python/ceval.c:2159
#12 0x0000555555e4a095 in PyEval_EvalCode (co=<optimized out>, globals=<optimized out>, locals=0x7c7ff6e871c0) at Python/ceval.c:995
#13 0x00005555561bbc6f in run_eval_code_obj (tstate=tstate@entry=0x555556aebf00 <_PyRuntime+358560>, co=co@entry=0x7d2ff6e36c10, globals=globals@entry=0x7c7ff6e871c0,
    locals=locals@entry=0x7c7ff6e871c0) at Python/pythonrun.c:1372
#14 0x00005555561bae3c in run_mod (mod=<optimized out>, filename=<optimized out>, globals=<optimized out>, locals=<optimized out>, flags=<optimized out>, arena=<optimized out>,
    interactive_src=<optimized out>, generate_new_source=<optimized out>) at Python/pythonrun.c:1475

Output from running with PYTHON_LLTRACE=4:
89_abort_lltrace.txt

Output from running with PYTHON_OPT_DEBUG=4:
89_abort_opt_debug.txt

Sorry for the convoluted repro steps, I've been trying to make it work with a simpler setup for many days, without success.

Found using lafleur.

CPython versions tested on:

CPython main branch

Operating systems tested on:

Linux

Output from running 'python -VV' on the command line:

Python 3.15.0a2+ (heads/main-dirty:41b9ad5b38e, Nov 20 2025, 08:38:13) [Clang 21.1.2 (2ubuntu6)]

Linked PRs

• gh-141786: Fix missing parent executor during JIT of trace #142286
• gh-141786: Fix missing parent executor during JIT trace #142344

[Fidget-Spinner]

What I think is happening is the following:

1. During trace recording of a side trace, the original parent trace's executor is cleared.
2. When the trace record concludes, it tries to JIT the trace, though the parent executor is already cleared.
3. The assertion fails.

Can you please try the following diff and see it if still fails? Thank you

diff --git a/Python/optimizer.c b/Python/optimizer.c
index 9db894f0bf0..9707f46c02f 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -1671,7 +1671,7 @@ executor_clear(PyObject *op)
     _PyExecutorObject *cold = _PyExecutor_GetColdExecutor();
     Py_INCREF(executor);
     for (uint32_t i = 0; i < executor->exit_count; i++) {
-        executor->exits[i].temperature = initial_unreachable_backoff_counter();
+        // executor->exits[i].temperature = initial_unreachable_backoff_counter();
         _PyExecutorObject *e = executor->exits[i].executor;
         executor->exits[i].executor = cold;
         Py_DECREF(e);

[devdanzin]

Applying the diff fixes the crash, thank you!

[devdanzin]

Here's another long MRE that results in a different abort, but is fixed by the same patch above:

import random

rand = random.Random(38720)
import sys

def b():
    for c in range(23):
        rand.random()
    rand.random()
    if rand.random() > 0.1:
        rand.random()
    rand.random()
    rand.random()
    for c in range(4):
        rand.random()
    if rand.random():
        0
    if rand.random() > 0.1:
        rand.random()
    rand.random()
    if rand.random():
        rand.random()
    if rand.random():
        rand.random()
    rand.random()
    if rand.random():
        rand.random()
    rand.random()
    if rand.random():
        if rand.random() > 0.1:
            rand.random()
    if rand.random() < rand.random():
        ...
    if rand.random() < rand.random():
        ...
    if rand.random():
        ...
    if rand.random() > 0.1:
        rand.random()
    rand.random()
    for c in range(12):
        rand.random()

    def random(B):
        if rand.random() < 0.1:
            random(B)

    def Random(B):
        if rand.random():
            random(B)

    def b(B):
        if rand.random():
            Random(B)

    def c(B):
        if rand.random() < 0.1:
            b(B)

    def a(B):
        if rand.random() > 0.1:
            c(B)

    def A(B):
        if rand.random() > 0.1:
            a(B)

    def C(B):
        if rand.random() > 0.1:
            A(B)

    def B(B):
        if rand.random() < 0.1:
            C(B)

    for D in range(200):
        try:
            B(D)
        except:
            ...

    class A:
        def __del__(C):
            b = sys._getframe(1)
            exec("D=0", b.f_globals, b.f_locals)

    B = A()
    if rand.random() < 0.1:
        if rand.random() < 0.1:
            if rand.random() < 0.1:
                for D in range(D):
                    if rand.random() < 0.1:
                        if rand.random() > 0.1:
                            if rand.random():
                                if rand.random() > rand.random():
                                    C
                    if rand.random() < 0.1:
                        if rand.random() < 0.1:
                            if rand.random() < 0.1:
                                del B
                    if rand.random() < 0.1:
                        if rand.random() < 0.1:
                            if rand.random():
                                0
    if rand.random() > 0.1:
        if rand.random() > 0.1:
            rand.random()
    rand.random()
    if rand.random() > 0.1:
        rand.random()
    rand.random()
    if rand.random() > 0.1:
        if rand.random() > 0.1:
            rand.random()
    rand.random()
    rand.random()
    rand.random()
    if rand.random() > 0.1:
        if rand.random() > 0.1:
            rand.random()
            if rand.random() > rand.random():
                0
    if rand.random():
        C
    if rand.random() > 0.1:
        rand.random()
    rand.random()
    for c in range(3):
        if rand.random():
            0
    for c in range(2):
        rand.random()

for D in range(5064):
    b()

Backtrace:

python: ./Include/internal/pycore_backoff.h:81: _Py_BackoffCounter restart_backoff_counter(_Py_BackoffCounter): Assertion `backoff <= MAX_BACKOFF' failed.

Program received signal SIGABRT, Aborted.
#0  __pthread_kill_implementation (threadid=<optimized out>, signo=6, no_tid=0) at ./nptl/pthread_kill.c:44
#1  __pthread_kill_internal (threadid=<optimized out>, signo=6) at ./nptl/pthread_kill.c:89
#2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6) at ./nptl/pthread_kill.c:100
#3  0x00007ffff7c45e2e in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#4  0x00007ffff7c28888 in __GI_abort () at ./stdlib/abort.c:77
#5  0x00007ffff7c287f0 in __assert_fail_base (fmt=<optimized out>, assertion=<optimized out>, file=<optimized out>, line=<optimized out>, function=<optimized out>) at ./assert/assert.c:118
#6  0x00007ffff7c3c19f in __assert_fail (assertion=<optimized out>, file=<optimized out>, line=<optimized out>, function=<optimized out>) at ./assert/assert.c:127
#7  0x000055555586e596 in restart_backoff_counter (counter=...) at ./Include/internal/pycore_backoff.h:81
#8  0x000055555586e596 in stop_tracing_and_jit (tstate=tstate@entry=0x555555d988a8 <_PyRuntime+358616>, frame=frame@entry=0x7ffff7fa7090)
#9  0x0000555555865724 in _PyEval_EvalFrameDefault (tstate=tstate@entry=0x555555d988a8 <_PyRuntime+358616>, frame=<optimized out>, frame@entry=0x7ffff7fa7020, throwflag=throwflag@entry=0)
    at Python/generated_cases.c.h:11712
#10 0x000055555582d56b in _PyEval_EvalFrame (tstate=0x555555d988a8 <_PyRuntime+358616>, frame=0x7ffff7fa7020, throwflag=0) at ./Include/internal/pycore_ceval.h:121
#11 _PyEval_Vector (tstate=tstate@entry=0x555555d988a8 <_PyRuntime+358616>, func=func@entry=0x7ffff6bf6a50, locals=locals@entry=0x7ffff6c05190, args=args@entry=0x0,
    argcount=argcount@entry=0, kwnames=kwnames@entry=0x0) at Python/ceval.c:2159

This MRE was reduced in about 8 hours, first manually then running Shrinkray.

[caje731]

With @Fidget-Spinner 's help I have now raised a PR to fix this. ☝️