diff --git a/Cargo.lock b/Cargo.lock index 0b7436570a..6dc4b699aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -193,6 +193,12 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" + [[package]] name = "atomic" version = "0.6.1" @@ -3005,8 +3011,10 @@ name = "rustpython-common" version = "0.4.0" dependencies = [ "ascii", + "atomic 0.5.3", "bitflags 2.10.0", "cfg-if", + "crossbeam-utils", "getrandom 0.3.4", "itertools 0.14.0", "libc", @@ -3014,6 +3022,7 @@ dependencies = [ "malachite-base", "malachite-bigint", "malachite-q", + "memoffset", "nix 0.30.1", "num-complex", "num-traits", @@ -3022,6 +3031,7 @@ dependencies = [ "radium", "rustpython-literal", "rustpython-wtf8", + "scopeguard", "siphasher", "unicode_names2 2.0.0", "widestring", @@ -4197,7 +4207,7 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ - "atomic", + "atomic 0.6.1", "js-sys", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index 988e99efac..394eef33cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -205,6 +205,8 @@ serde = { version = "1.0.225", default-features = false } schannel = "0.1.28" scoped-tls = "1" scopeguard = "1" +memoffset = "0.9" +atomic = "0.5" static_assertions = "1.1" strum = "0.27" strum_macros = "0.27" diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 444ca2219c..f83b8bf1ed 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -869,13 +869,6 @@ def disable_gc(): @contextlib.contextmanager def gc_threshold(*args): - # TODO: RUSTPYTHON; GC is not supported yet - try: - yield - finally: - pass - return - import gc old_threshold = gc.get_threshold() gc.set_threshold(*args) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 7d8c7a5e01..7b34a9c6a1 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2328,8 +2328,6 @@ def test_baddecorator(self): class ShutdownTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cleanup(self): # Issue #19255: builtins are still available at shutdown code = """if 1: diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 9598a7ab96..ce0f09dd76 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -369,8 +369,6 @@ def test_copy_fuzz(self): self.assertNotEqual(d, d2) self.assertEqual(len(d2), len(d) + 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_copy_maintains_tracking(self): class A: pass diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py new file mode 100644 index 0000000000..81577470c5 --- /dev/null +++ b/Lib/test/test_gc.py @@ -0,0 +1,1586 @@ +import unittest +import unittest.mock +from test import support +from test.support import (verbose, refcount_test, + cpython_only, requires_subprocess, + requires_gil_enabled, suppress_immortalization, + Py_GIL_DISABLED) +from test.support.import_helper import import_module +from test.support.os_helper import temp_dir, TESTFN, unlink +from test.support.script_helper import assert_python_ok, make_script +from test.support import threading_helper, gc_threshold + +import gc +import sys +import sysconfig +import textwrap +import threading +import time +import weakref + +try: + import _testcapi + from _testcapi import with_tp_del + from _testcapi import ContainerNoGC +except ImportError: + _testcapi = None + def with_tp_del(cls): + class C(object): + def __new__(cls, *args, **kwargs): + raise unittest.SkipTest('requires _testcapi.with_tp_del') + return C + ContainerNoGC = None + +### Support code +############################################################################### + +# Bug 1055820 has several tests of longstanding bugs involving weakrefs and +# cyclic gc. + +# An instance of C1055820 has a self-loop, so becomes cyclic trash when +# unreachable. +class C1055820(object): + def __init__(self, i): + self.i = i + self.loop = self + +class GC_Detector(object): + # Create an instance I. Then gc hasn't happened again so long as + # I.gc_happened is false. + + def __init__(self): + self.gc_happened = False + + def it_happened(ignored): + self.gc_happened = True + + # Create a piece of cyclic trash that triggers it_happened when + # gc collects it. + self.wr = weakref.ref(C1055820(666), it_happened) + +@with_tp_del +class Uncollectable(object): + """Create a reference cycle with multiple __del__ methods. + + An object in a reference cycle will never have zero references, + and so must be garbage collected. If one or more objects in the + cycle have __del__ methods, the gc refuses to guess an order, + and leaves the cycle uncollected.""" + def __init__(self, partner=None): + if partner is None: + self.partner = Uncollectable(partner=self) + else: + self.partner = partner + def __tp_del__(self): + pass + +if sysconfig.get_config_vars().get('PY_CFLAGS', ''): + BUILD_WITH_NDEBUG = ('-DNDEBUG' in sysconfig.get_config_vars()['PY_CFLAGS']) +else: + # Usually, sys.gettotalrefcount() is only present if Python has been + # compiled in debug mode. If it's missing, expect that Python has + # been released in release mode: with NDEBUG defined. + BUILD_WITH_NDEBUG = (not hasattr(sys, 'gettotalrefcount')) + +### Tests +############################################################################### + +class GCTests(unittest.TestCase): + def test_list(self): + l = [] + l.append(l) + gc.collect() + del l + self.assertEqual(gc.collect(), 1) + + def test_dict(self): + d = {} + d[1] = d + gc.collect() + del d + self.assertEqual(gc.collect(), 1) + + def test_tuple(self): + # since tuples are immutable we close the loop with a list + l = [] + t = (l,) + l.append(t) + gc.collect() + del t + del l + self.assertEqual(gc.collect(), 2) + + @suppress_immortalization() + def test_class(self): + class A: + pass + A.a = A + gc.collect() + del A + self.assertNotEqual(gc.collect(), 0) + + @suppress_immortalization() + def test_newstyleclass(self): + class A(object): + pass + gc.collect() + del A + self.assertNotEqual(gc.collect(), 0) + + def test_instance(self): + class A: + pass + a = A() + a.a = a + gc.collect() + del a + self.assertNotEqual(gc.collect(), 0) + + @suppress_immortalization() + def test_newinstance(self): + class A(object): + pass + a = A() + a.a = a + gc.collect() + del a + self.assertNotEqual(gc.collect(), 0) + class B(list): + pass + class C(B, A): + pass + a = C() + a.a = a + gc.collect() + del a + self.assertNotEqual(gc.collect(), 0) + del B, C + self.assertNotEqual(gc.collect(), 0) + A.a = A() + del A + self.assertNotEqual(gc.collect(), 0) + self.assertEqual(gc.collect(), 0) + + def test_method(self): + # Tricky: self.__init__ is a bound method, it references the instance. + class A: + def __init__(self): + self.init = self.__init__ + a = A() + gc.collect() + del a + self.assertNotEqual(gc.collect(), 0) + + @cpython_only + def test_legacy_finalizer(self): + # A() is uncollectable if it is part of a cycle, make sure it shows up + # in gc.garbage. + @with_tp_del + class A: + def __tp_del__(self): pass + class B: + pass + a = A() + a.a = a + id_a = id(a) + b = B() + b.b = b + gc.collect() + del a + del b + self.assertNotEqual(gc.collect(), 0) + for obj in gc.garbage: + if id(obj) == id_a: + del obj.a + break + else: + self.fail("didn't find obj in garbage (finalizer)") + gc.garbage.remove(obj) + + @cpython_only + def test_legacy_finalizer_newclass(self): + # A() is uncollectable if it is part of a cycle, make sure it shows up + # in gc.garbage. + @with_tp_del + class A(object): + def __tp_del__(self): pass + class B(object): + pass + a = A() + a.a = a + id_a = id(a) + b = B() + b.b = b + gc.collect() + del a + del b + self.assertNotEqual(gc.collect(), 0) + for obj in gc.garbage: + if id(obj) == id_a: + del obj.a + break + else: + self.fail("didn't find obj in garbage (finalizer)") + gc.garbage.remove(obj) + + @suppress_immortalization() + def test_function(self): + # Tricky: f -> d -> f, code should call d.clear() after the exec to + # break the cycle. + d = {} + exec("def f(): pass\n", d) + gc.collect() + del d + # In the free-threaded build, the count returned by `gc.collect()` + # is 3 because it includes f's code object. + self.assertIn(gc.collect(), (2, 3)) + + def test_function_tp_clear_leaves_consistent_state(self): + # https://github.com/python/cpython/issues/91636 + code = """if 1: + + import gc + import weakref + + class LateFin: + __slots__ = ('ref',) + + def __del__(self): + + # 8. Now `latefin`'s finalizer is called. Here we + # obtain a reference to `func`, which is currently + # undergoing `tp_clear`. + global func + func = self.ref() + + class Cyclic(tuple): + __slots__ = () + + # 4. The finalizers of all garbage objects are called. In + # this case this is only us as `func` doesn't have a + # finalizer. + def __del__(self): + + # 5. Create a weakref to `func` now. If we had created + # it earlier, it would have been cleared by the + # garbage collector before calling the finalizers. + self[1].ref = weakref.ref(self[0]) + + # 6. Drop the global reference to `latefin`. The only + # remaining reference is the one we have. + global latefin + del latefin + + # 7. Now `func` is `tp_clear`-ed. This drops the last + # reference to `Cyclic`, which gets `tp_dealloc`-ed. + # This drops the last reference to `latefin`. + + latefin = LateFin() + def func(): + pass + cyc = tuple.__new__(Cyclic, (func, latefin)) + + # 1. Create a reference cycle of `cyc` and `func`. + func.__module__ = cyc + + # 2. Make the cycle unreachable, but keep the global reference + # to `latefin` so that it isn't detected as garbage. This + # way its finalizer will not be called immediately. + del func, cyc + + # 3. Invoke garbage collection, + # which will find `cyc` and `func` as garbage. + gc.collect() + + # 9. Previously, this would crash because `func_qualname` + # had been NULL-ed out by func_clear(). + print(f"{func=}") + """ + # We're mostly just checking that this doesn't crash. + rc, stdout, stderr = assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertRegex(stdout, rb"""\A\s*func=\s*\Z""") + self.assertFalse(stderr) + + @refcount_test + def test_frame(self): + def f(): + frame = sys._getframe() + gc.collect() + f() + self.assertEqual(gc.collect(), 1) + + def test_saveall(self): + # Verify that cyclic garbage like lists show up in gc.garbage if the + # SAVEALL option is enabled. + + # First make sure we don't save away other stuff that just happens to + # be waiting for collection. + gc.collect() + # if this fails, someone else created immortal trash + self.assertEqual(gc.garbage, []) + + L = [] + L.append(L) + id_L = id(L) + + debug = gc.get_debug() + gc.set_debug(debug | gc.DEBUG_SAVEALL) + del L + gc.collect() + gc.set_debug(debug) + + self.assertEqual(len(gc.garbage), 1) + obj = gc.garbage.pop() + self.assertEqual(id(obj), id_L) + + def test_del(self): + # __del__ methods can trigger collection, make this to happen + thresholds = gc.get_threshold() + gc.enable() + gc.set_threshold(1) + + class A: + def __del__(self): + dir(self) + a = A() + del a + + gc.disable() + gc.set_threshold(*thresholds) + + def test_del_newclass(self): + # __del__ methods can trigger collection, make this to happen + thresholds = gc.get_threshold() + gc.enable() + gc.set_threshold(1) + + class A(object): + def __del__(self): + dir(self) + a = A() + del a + + gc.disable() + gc.set_threshold(*thresholds) + + # The following two tests are fragile: + # They precisely count the number of allocations, + # which is highly implementation-dependent. + # For example, disposed tuples are not freed, but reused. + # To minimize variations, though, we first store the get_count() results + # and check them at the end. + @refcount_test + @requires_gil_enabled('needs precise allocation counts') + def test_get_count(self): + gc.collect() + a, b, c = gc.get_count() + x = [] + d, e, f = gc.get_count() + self.assertEqual((b, c), (0, 0)) + self.assertEqual((e, f), (0, 0)) + # This is less fragile than asserting that a equals 0. + self.assertLess(a, 5) + # Between the two calls to get_count(), at least one object was + # created (the list). + self.assertGreater(d, a) + + @refcount_test + def test_collect_generations(self): + gc.collect() + # This object will "trickle" into generation N + 1 after + # each call to collect(N) + x = [] + gc.collect(0) + # x is now in gen 1 + a, b, c = gc.get_count() + gc.collect(1) + # x is now in gen 2 + d, e, f = gc.get_count() + gc.collect(2) + # x is now in gen 3 + g, h, i = gc.get_count() + # We don't check a, d, g since their exact values depends on + # internal implementation details of the interpreter. + self.assertEqual((b, c), (1, 0)) + self.assertEqual((e, f), (0, 1)) + self.assertEqual((h, i), (0, 0)) + + def test_trashcan(self): + class Ouch: + n = 0 + def __del__(self): + Ouch.n = Ouch.n + 1 + if Ouch.n % 17 == 0: + gc.collect() + + # "trashcan" is a hack to prevent stack overflow when deallocating + # very deeply nested tuples etc. It works in part by abusing the + # type pointer and refcount fields, and that can yield horrible + # problems when gc tries to traverse the structures. + # If this test fails (as it does in 2.0, 2.1 and 2.2), it will + # most likely die via segfault. + + # Note: In 2.3 the possibility for compiling without cyclic gc was + # removed, and that in turn allows the trashcan mechanism to work + # via much simpler means (e.g., it never abuses the type pointer or + # refcount fields anymore). Since it's much less likely to cause a + # problem now, the various constants in this expensive (we force a lot + # of full collections) test are cut back from the 2.2 version. + gc.enable() + N = 150 + for count in range(2): + t = [] + for i in range(N): + t = [t, Ouch()] + u = [] + for i in range(N): + u = [u, Ouch()] + v = {} + for i in range(N): + v = {1: v, 2: Ouch()} + gc.disable() + + @threading_helper.requires_working_threading() + def test_trashcan_threads(self): + # Issue #13992: trashcan mechanism should be thread-safe + NESTING = 60 + N_THREADS = 2 + + def sleeper_gen(): + """A generator that releases the GIL when closed or dealloc'ed.""" + try: + yield + finally: + time.sleep(0.000001) + + class C(list): + # Appending to a list is atomic, which avoids the use of a lock. + inits = [] + dels = [] + def __init__(self, alist): + self[:] = alist + C.inits.append(None) + def __del__(self): + # This __del__ is called by subtype_dealloc(). + C.dels.append(None) + # `g` will release the GIL when garbage-collected. This + # helps assert subtype_dealloc's behaviour when threads + # switch in the middle of it. + g = sleeper_gen() + next(g) + # Now that __del__ is finished, subtype_dealloc will proceed + # to call list_dealloc, which also uses the trashcan mechanism. + + def make_nested(): + """Create a sufficiently nested container object so that the + trashcan mechanism is invoked when deallocating it.""" + x = C([]) + for i in range(NESTING): + x = [C([x])] + del x + + def run_thread(): + """Exercise make_nested() in a loop.""" + while not exit: + make_nested() + + old_switchinterval = sys.getswitchinterval() + support.setswitchinterval(1e-5) + try: + exit = [] + threads = [] + for i in range(N_THREADS): + t = threading.Thread(target=run_thread) + threads.append(t) + with threading_helper.start_threads(threads, lambda: exit.append(1)): + time.sleep(1.0) + finally: + sys.setswitchinterval(old_switchinterval) + gc.collect() + self.assertEqual(len(C.inits), len(C.dels)) + + def test_boom(self): + class Boom: + def __getattr__(self, someattribute): + del self.attr + raise AttributeError + + a = Boom() + b = Boom() + a.attr = b + b.attr = a + + gc.collect() + garbagelen = len(gc.garbage) + del a, b + # a<->b are in a trash cycle now. Collection will invoke + # Boom.__getattr__ (to see whether a and b have __del__ methods), and + # __getattr__ deletes the internal "attr" attributes as a side effect. + # That causes the trash cycle to get reclaimed via refcounts falling to + # 0, thus mutating the trash graph as a side effect of merely asking + # whether __del__ exists. This used to (before 2.3b1) crash Python. + # Now __getattr__ isn't called. + self.assertEqual(gc.collect(), 2) + self.assertEqual(len(gc.garbage), garbagelen) + + def test_boom2(self): + class Boom2: + def __init__(self): + self.x = 0 + + def __getattr__(self, someattribute): + self.x += 1 + if self.x > 1: + del self.attr + raise AttributeError + + a = Boom2() + b = Boom2() + a.attr = b + b.attr = a + + gc.collect() + garbagelen = len(gc.garbage) + del a, b + # Much like test_boom(), except that __getattr__ doesn't break the + # cycle until the second time gc checks for __del__. As of 2.3b1, + # there isn't a second time, so this simply cleans up the trash cycle. + # We expect a, b, a.__dict__ and b.__dict__ (4 objects) to get + # reclaimed this way. + self.assertEqual(gc.collect(), 2) + self.assertEqual(len(gc.garbage), garbagelen) + + def test_get_referents(self): + alist = [1, 3, 5] + got = gc.get_referents(alist) + got.sort() + self.assertEqual(got, alist) + + atuple = tuple(alist) + got = gc.get_referents(atuple) + got.sort() + self.assertEqual(got, alist) + + adict = {1: 3, 5: 7} + expected = [1, 3, 5, 7] + got = gc.get_referents(adict) + got.sort() + self.assertEqual(got, expected) + + got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0)) + got.sort() + self.assertEqual(got, [0, 0] + list(range(5))) + + self.assertEqual(gc.get_referents(1, 'a', 4j), []) + + @suppress_immortalization() + def test_is_tracked(self): + # Atomic built-in types are not tracked, user-defined objects and + # mutable containers are. + # NOTE: types with special optimizations (e.g. tuple) have tests + # in their own test files instead. + self.assertFalse(gc.is_tracked(None)) + self.assertFalse(gc.is_tracked(1)) + self.assertFalse(gc.is_tracked(1.0)) + self.assertFalse(gc.is_tracked(1.0 + 5.0j)) + self.assertFalse(gc.is_tracked(True)) + self.assertFalse(gc.is_tracked(False)) + self.assertFalse(gc.is_tracked(b"a")) + self.assertFalse(gc.is_tracked("a")) + self.assertFalse(gc.is_tracked(bytearray(b"a"))) + self.assertFalse(gc.is_tracked(type)) + self.assertFalse(gc.is_tracked(int)) + self.assertFalse(gc.is_tracked(object)) + self.assertFalse(gc.is_tracked(object())) + + class UserClass: + pass + + class UserInt(int): + pass + + # Base class is object; no extra fields. + class UserClassSlots: + __slots__ = () + + # Base class is fixed size larger than object; no extra fields. + class UserFloatSlots(float): + __slots__ = () + + # Base class is variable size; no extra fields. + class UserIntSlots(int): + __slots__ = () + + if not Py_GIL_DISABLED: + # gh-117783: modules may be immortalized in free-threaded build + self.assertTrue(gc.is_tracked(gc)) + self.assertTrue(gc.is_tracked(UserClass)) + self.assertTrue(gc.is_tracked(UserClass())) + self.assertTrue(gc.is_tracked(UserInt())) + self.assertTrue(gc.is_tracked([])) + self.assertTrue(gc.is_tracked(set())) + self.assertTrue(gc.is_tracked(UserClassSlots())) + self.assertTrue(gc.is_tracked(UserFloatSlots())) + self.assertTrue(gc.is_tracked(UserIntSlots())) + + def test_is_finalized(self): + # Objects not tracked by the always gc return false + self.assertFalse(gc.is_finalized(3)) + + storage = [] + class Lazarus: + def __del__(self): + storage.append(self) + + lazarus = Lazarus() + self.assertFalse(gc.is_finalized(lazarus)) + + del lazarus + gc.collect() + + lazarus = storage.pop() + self.assertTrue(gc.is_finalized(lazarus)) + + def test_bug1055820b(self): + # Corresponds to temp2b.py in the bug report. + + ouch = [] + def callback(ignored): + ouch[:] = [wr() for wr in WRs] + + Cs = [C1055820(i) for i in range(2)] + WRs = [weakref.ref(c, callback) for c in Cs] + c = None + + gc.collect() + self.assertEqual(len(ouch), 0) + # Make the two instances trash, and collect again. The bug was that + # the callback materialized a strong reference to an instance, but gc + # cleared the instance's dict anyway. + Cs = None + gc.collect() + self.assertEqual(len(ouch), 2) # else the callbacks didn't run + for x in ouch: + # If the callback resurrected one of these guys, the instance + # would be damaged, with an empty __dict__. + self.assertEqual(x, None) + + def test_bug21435(self): + # This is a poor test - its only virtue is that it happened to + # segfault on Tim's Windows box before the patch for 21435 was + # applied. That's a nasty bug relying on specific pieces of cyclic + # trash appearing in exactly the right order in finalize_garbage()'s + # input list. + # But there's no reliable way to force that order from Python code, + # so over time chances are good this test won't really be testing much + # of anything anymore. Still, if it blows up, there's _some_ + # problem ;-) + gc.collect() + + class A: + pass + + class B: + def __init__(self, x): + self.x = x + + def __del__(self): + self.attr = None + + def do_work(): + a = A() + b = B(A()) + + a.attr = b + b.attr = a + + do_work() + gc.collect() # this blows up (bad C pointer) when it fails + + @cpython_only + @requires_subprocess() + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_garbage_at_shutdown(self): + import subprocess + code = """if 1: + import gc + import _testcapi + @_testcapi.with_tp_del + class X: + def __init__(self, name): + self.name = name + def __repr__(self): + return "" %% self.name + def __tp_del__(self): + pass + + x = X('first') + x.x = x + x.y = X('second') + del x + gc.set_debug(%s) + """ + def run_command(code): + p = subprocess.Popen([sys.executable, "-Wd", "-c", code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + p.stdout.close() + p.stderr.close() + self.assertEqual(p.returncode, 0) + self.assertEqual(stdout, b"") + return stderr + + stderr = run_command(code % "0") + self.assertIn(b"ResourceWarning: gc: 2 uncollectable objects at " + b"shutdown; use", stderr) + self.assertNotIn(b"", stderr) + # With DEBUG_UNCOLLECTABLE, the garbage list gets printed + stderr = run_command(code % "gc.DEBUG_UNCOLLECTABLE") + self.assertIn(b"ResourceWarning: gc: 2 uncollectable objects at " + b"shutdown", stderr) + self.assertTrue( + (b"[, ]" in stderr) or + (b"[, ]" in stderr), stderr) + # With DEBUG_SAVEALL, no additional message should get printed + # (because gc.garbage also contains normally reclaimable cyclic + # references, and its elements get printed at runtime anyway). + stderr = run_command(code % "gc.DEBUG_SAVEALL") + self.assertNotIn(b"uncollectable objects at shutdown", stderr) + + def test_gc_main_module_at_shutdown(self): + # Create a reference cycle through the __main__ module and check + # it gets collected at interpreter shutdown. + code = """if 1: + class C: + def __del__(self): + print('__del__ called') + l = [C()] + l.append(l) + """ + rc, out, err = assert_python_ok('-c', code) + self.assertEqual(out.strip(), b'__del__ called') + + def test_gc_ordinary_module_at_shutdown(self): + # Same as above, but with a non-__main__ module. + with temp_dir() as script_dir: + module = """if 1: + class C: + def __del__(self): + print('__del__ called') + l = [C()] + l.append(l) + """ + code = """if 1: + import sys + sys.path.insert(0, %r) + import gctest + """ % (script_dir,) + make_script(script_dir, 'gctest', module) + rc, out, err = assert_python_ok('-c', code) + self.assertEqual(out.strip(), b'__del__ called') + + def test_global_del_SystemExit(self): + code = """if 1: + class ClassWithDel: + def __del__(self): + print('__del__ called') + a = ClassWithDel() + a.link = a + raise SystemExit(0)""" + self.addCleanup(unlink, TESTFN) + with open(TESTFN, 'w', encoding="utf-8") as script: + script.write(code) + rc, out, err = assert_python_ok(TESTFN) + self.assertEqual(out.strip(), b'__del__ called') + + def test_get_stats(self): + stats = gc.get_stats() + self.assertEqual(len(stats), 3) + for st in stats: + self.assertIsInstance(st, dict) + self.assertEqual(set(st), + {"collected", "collections", "uncollectable"}) + self.assertGreaterEqual(st["collected"], 0) + self.assertGreaterEqual(st["collections"], 0) + self.assertGreaterEqual(st["uncollectable"], 0) + # Check that collection counts are incremented correctly + if gc.isenabled(): + self.addCleanup(gc.enable) + gc.disable() + old = gc.get_stats() + gc.collect(0) + new = gc.get_stats() + self.assertEqual(new[0]["collections"], old[0]["collections"] + 1) + self.assertEqual(new[1]["collections"], old[1]["collections"]) + self.assertEqual(new[2]["collections"], old[2]["collections"]) + gc.collect(2) + new = gc.get_stats() + self.assertEqual(new[0]["collections"], old[0]["collections"] + 1) + self.assertEqual(new[1]["collections"], old[1]["collections"]) + self.assertEqual(new[2]["collections"], old[2]["collections"] + 1) + + def test_freeze(self): + gc.freeze() + self.assertGreater(gc.get_freeze_count(), 0) + gc.unfreeze() + self.assertEqual(gc.get_freeze_count(), 0) + + def test_get_objects(self): + gc.collect() + l = [] + l.append(l) + self.assertTrue( + any(l is element for element in gc.get_objects()) + ) + + @requires_gil_enabled('need generational GC') + def test_get_objects_generations(self): + gc.collect() + l = [] + l.append(l) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=2)) + ) + gc.collect(generation=0) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=2)) + ) + gc.collect(generation=1) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=2)) + ) + gc.collect(generation=2) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=2)) + ) + del l + gc.collect() + + def test_get_objects_arguments(self): + gc.collect() + self.assertEqual(len(gc.get_objects()), + len(gc.get_objects(generation=None))) + + self.assertRaises(ValueError, gc.get_objects, 1000) + self.assertRaises(ValueError, gc.get_objects, -1000) + self.assertRaises(TypeError, gc.get_objects, "1") + self.assertRaises(TypeError, gc.get_objects, 1.234) + + def test_resurrection_only_happens_once_per_object(self): + class A: # simple self-loop + def __init__(self): + self.me = self + + class Lazarus(A): + resurrected = 0 + resurrected_instances = [] + + def __del__(self): + Lazarus.resurrected += 1 + Lazarus.resurrected_instances.append(self) + + gc.collect() + gc.disable() + + # We start with 0 resurrections + laz = Lazarus() + self.assertEqual(Lazarus.resurrected, 0) + + # Deleting the instance and triggering a collection + # resurrects the object + del laz + gc.collect() + self.assertEqual(Lazarus.resurrected, 1) + self.assertEqual(len(Lazarus.resurrected_instances), 1) + + # Clearing the references and forcing a collection + # should not resurrect the object again. + Lazarus.resurrected_instances.clear() + self.assertEqual(Lazarus.resurrected, 1) + gc.collect() + self.assertEqual(Lazarus.resurrected, 1) + + gc.enable() + + def test_resurrection_is_transitive(self): + class Cargo: + def __init__(self): + self.me = self + + class Lazarus: + resurrected_instances = [] + + def __del__(self): + Lazarus.resurrected_instances.append(self) + + gc.collect() + gc.disable() + + laz = Lazarus() + cargo = Cargo() + cargo_id = id(cargo) + + # Create a cycle between cargo and laz + laz.cargo = cargo + cargo.laz = laz + + # Drop the references, force a collection and check that + # everything was resurrected. + del laz, cargo + gc.collect() + self.assertEqual(len(Lazarus.resurrected_instances), 1) + instance = Lazarus.resurrected_instances.pop() + self.assertTrue(hasattr(instance, "cargo")) + self.assertEqual(id(instance.cargo), cargo_id) + + gc.collect() + gc.enable() + + def test_resurrection_does_not_block_cleanup_of_other_objects(self): + + # When a finalizer resurrects objects, stats were reporting them as + # having been collected. This affected both collect()'s return + # value and the dicts returned by get_stats(). + N = 100 + + class A: # simple self-loop + def __init__(self): + self.me = self + + class Z(A): # resurrecting __del__ + def __del__(self): + zs.append(self) + + zs = [] + + def getstats(): + d = gc.get_stats()[-1] + return d['collected'], d['uncollectable'] + + gc.collect() + gc.disable() + + # No problems if just collecting A() instances. + oldc, oldnc = getstats() + for i in range(N): + A() + t = gc.collect() + c, nc = getstats() + self.assertEqual(t, N) # instance objects + self.assertEqual(c - oldc, N) + self.assertEqual(nc - oldnc, 0) + + # But Z() is not actually collected. + oldc, oldnc = c, nc + Z() + # Nothing is collected - Z() is merely resurrected. + t = gc.collect() + c, nc = getstats() + self.assertEqual(t, 0) + self.assertEqual(c - oldc, 0) + self.assertEqual(nc - oldnc, 0) + + # Z() should not prevent anything else from being collected. + oldc, oldnc = c, nc + for i in range(N): + A() + Z() + t = gc.collect() + c, nc = getstats() + self.assertEqual(t, N) + self.assertEqual(c - oldc, N) + self.assertEqual(nc - oldnc, 0) + + # The A() trash should have been reclaimed already but the + # 2 copies of Z are still in zs (and the associated dicts). + oldc, oldnc = c, nc + zs.clear() + t = gc.collect() + c, nc = getstats() + self.assertEqual(t, 2) + self.assertEqual(c - oldc, 2) + self.assertEqual(nc - oldnc, 0) + + gc.enable() + + @unittest.skipIf(ContainerNoGC is None, + 'requires ContainerNoGC extension type') + def test_trash_weakref_clear(self): + # Test that trash weakrefs are properly cleared (bpo-38006). + # + # Structure we are creating: + # + # Z <- Y <- A--+--> WZ -> C + # ^ | + # +--+ + # where: + # WZ is a weakref to Z with callback C + # Y doesn't implement tp_traverse + # A contains a reference to itself, Y and WZ + # + # A, Y, Z, WZ are all trash. The GC doesn't know that Z is trash + # because Y does not implement tp_traverse. To show the bug, WZ needs + # to live long enough so that Z is deallocated before it. Then, if + # gcmodule is buggy, when Z is being deallocated, C will run. + # + # To ensure WZ lives long enough, we put it in a second reference + # cycle. That trick only works due to the ordering of the GC prev/next + # linked lists. So, this test is a bit fragile. + # + # The bug reported in bpo-38006 is caused because the GC did not + # clear WZ before starting the process of calling tp_clear on the + # trash. Normally, handle_weakrefs() would find the weakref via Z and + # clear it. However, since the GC cannot find Z, WR is not cleared and + # it can execute during delete_garbage(). That can lead to disaster + # since the callback might tinker with objects that have already had + # tp_clear called on them (leaving them in possibly invalid states). + + callback = unittest.mock.Mock() + + class A: + __slots__ = ['a', 'y', 'wz'] + + class Z: + pass + + # setup required object graph, as described above + a = A() + a.a = a + a.y = ContainerNoGC(Z()) + a.wz = weakref.ref(a.y.value, callback) + # create second cycle to keep WZ alive longer + wr_cycle = [a.wz] + wr_cycle.append(wr_cycle) + # ensure trash unrelated to this test is gone + gc.collect() + gc.disable() + # release references and create trash + del a, wr_cycle + gc.collect() + # if called, it means there is a bug in the GC. The weakref should be + # cleared before Z dies. + callback.assert_not_called() + gc.enable() + + @cpython_only + def test_get_referents_on_capsule(self): + # gh-124538: Calling gc.get_referents() on an untracked capsule must not crash. + import _datetime + import _socket + untracked_capsule = _datetime.datetime_CAPI + tracked_capsule = _socket.CAPI + + # For whoever sees this in the future: if this is failing + # after making datetime's capsule tracked, that's fine -- this isn't something + # users are relying on. Just find a different capsule that is untracked. + self.assertFalse(gc.is_tracked(untracked_capsule)) + self.assertTrue(gc.is_tracked(tracked_capsule)) + + self.assertEqual(len(gc.get_referents(untracked_capsule)), 0) + gc.get_referents(tracked_capsule) + + @cpython_only + def test_get_objects_during_gc(self): + # gh-125859: Calling gc.get_objects() or gc.get_referrers() during a + # collection should not crash. + test = self + collected = False + + class GetObjectsOnDel: + def __del__(self): + nonlocal collected + collected = True + objs = gc.get_objects() + # NB: can't use "in" here because some objects override __eq__ + for obj in objs: + test.assertTrue(obj is not self) + test.assertEqual(gc.get_referrers(self), []) + + obj = GetObjectsOnDel() + obj.cycle = obj + del obj + + gc.collect() + self.assertTrue(collected) + + def test_traverse_frozen_objects(self): + # See GH-126312: Objects that were not frozen could traverse over + # a frozen object on the free-threaded build, which would cause + # a negative reference count. + x = [1, 2, 3] + gc.freeze() + y = [x] + y.append(y) + del y + gc.collect() + gc.unfreeze() + + def test_deferred_refcount_frozen(self): + # Also from GH-126312: objects that use deferred reference counting + # weren't ignored if they were frozen. Unfortunately, it's pretty + # difficult to come up with a case that triggers this. + # + # Calling gc.collect() while the garbage collector is frozen doesn't + # trigger this normally, but it *does* if it's inside unittest for whatever + # reason. We can't call unittest from inside a test, so it has to be + # in a subprocess. + source = textwrap.dedent(""" + import gc + import unittest + + + class Test(unittest.TestCase): + def test_something(self): + gc.freeze() + gc.collect() + gc.unfreeze() + + + if __name__ == "__main__": + unittest.main() + """) + assert_python_ok("-c", source) + + +class GCCallbackTests(unittest.TestCase): + def setUp(self): + # Save gc state and disable it. + self.enabled = gc.isenabled() + gc.disable() + self.debug = gc.get_debug() + gc.set_debug(0) + gc.callbacks.append(self.cb1) + gc.callbacks.append(self.cb2) + self.othergarbage = [] + + def tearDown(self): + # Restore gc state + del self.visit + gc.callbacks.remove(self.cb1) + gc.callbacks.remove(self.cb2) + gc.set_debug(self.debug) + if self.enabled: + gc.enable() + # destroy any uncollectables + gc.collect() + for obj in gc.garbage: + if isinstance(obj, Uncollectable): + obj.partner = None + del gc.garbage[:] + del self.othergarbage + gc.collect() + + def preclean(self): + # Remove all fluff from the system. Invoke this function + # manually rather than through self.setUp() for maximum + # safety. + self.visit = [] + gc.collect() + garbage, gc.garbage[:] = gc.garbage[:], [] + self.othergarbage.append(garbage) + self.visit = [] + + def cb1(self, phase, info): + self.visit.append((1, phase, dict(info))) + + def cb2(self, phase, info): + self.visit.append((2, phase, dict(info))) + if phase == "stop" and hasattr(self, "cleanup"): + # Clean Uncollectable from garbage + uc = [e for e in gc.garbage if isinstance(e, Uncollectable)] + gc.garbage[:] = [e for e in gc.garbage + if not isinstance(e, Uncollectable)] + for e in uc: + e.partner = None + + def test_collect(self): + self.preclean() + gc.collect() + # Algorithmically verify the contents of self.visit + # because it is long and tortuous. + + # Count the number of visits to each callback + n = [v[0] for v in self.visit] + n1 = [i for i in n if i == 1] + n2 = [i for i in n if i == 2] + self.assertEqual(n1, [1]*2) + self.assertEqual(n2, [2]*2) + + # Count that we got the right number of start and stop callbacks. + n = [v[1] for v in self.visit] + n1 = [i for i in n if i == "start"] + n2 = [i for i in n if i == "stop"] + self.assertEqual(n1, ["start"]*2) + self.assertEqual(n2, ["stop"]*2) + + # Check that we got the right info dict for all callbacks + for v in self.visit: + info = v[2] + self.assertTrue("generation" in info) + self.assertTrue("collected" in info) + self.assertTrue("uncollectable" in info) + + def test_collect_generation(self): + self.preclean() + gc.collect(2) + for v in self.visit: + info = v[2] + self.assertEqual(info["generation"], 2) + + @cpython_only + def test_collect_garbage(self): + self.preclean() + # Each of these cause two objects to be garbage: + Uncollectable() + Uncollectable() + C1055820(666) + gc.collect() + for v in self.visit: + if v[1] != "stop": + continue + info = v[2] + self.assertEqual(info["collected"], 1) + self.assertEqual(info["uncollectable"], 4) + + # We should now have the Uncollectables in gc.garbage + self.assertEqual(len(gc.garbage), 4) + for e in gc.garbage: + self.assertIsInstance(e, Uncollectable) + + # Now, let our callback handle the Uncollectable instances + self.cleanup=True + self.visit = [] + gc.garbage[:] = [] + gc.collect() + for v in self.visit: + if v[1] != "stop": + continue + info = v[2] + self.assertEqual(info["collected"], 0) + self.assertEqual(info["uncollectable"], 2) + + # Uncollectables should be gone + self.assertEqual(len(gc.garbage), 0) + + + @requires_subprocess() + @unittest.skipIf(BUILD_WITH_NDEBUG, + 'built with -NDEBUG') + def test_refcount_errors(self): + self.preclean() + # Verify the "handling" of objects with broken refcounts + + # Skip the test if ctypes is not available + import_module("ctypes") + + import subprocess + code = textwrap.dedent(''' + from test.support import gc_collect, SuppressCrashReport + + a = [1, 2, 3] + b = [a] + + # Avoid coredump when Py_FatalError() calls abort() + SuppressCrashReport().__enter__() + + # Simulate the refcount of "a" being too low (compared to the + # references held on it by live data), but keeping it above zero + # (to avoid deallocating it): + import ctypes + ctypes.pythonapi.Py_DecRef(ctypes.py_object(a)) + + # The garbage collector should now have a fatal error + # when it reaches the broken object + gc_collect() + ''') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + p.stdout.close() + p.stderr.close() + # Verify that stderr has a useful error message: + self.assertRegex(stderr, + br'gc.*\.c:[0-9]+: .*: Assertion "gc_get_refs\(.+\) .*" failed.') + self.assertRegex(stderr, + br'refcount is too small') + # "address : 0x7fb5062efc18" + # "address : 7FB5062EFC18" + address_regex = br'[0-9a-fA-Fx]+' + self.assertRegex(stderr, + br'object address : ' + address_regex) + self.assertRegex(stderr, + br'object refcount : 1') + self.assertRegex(stderr, + br'object type : ' + address_regex) + self.assertRegex(stderr, + br'object type name: list') + self.assertRegex(stderr, + br'object repr : \[1, 2, 3\]') + + +class GCTogglingTests(unittest.TestCase): + def setUp(self): + gc.enable() + + def tearDown(self): + gc.disable() + + def test_bug1055820c(self): + # Corresponds to temp2c.py in the bug report. This is pretty + # elaborate. + + c0 = C1055820(0) + # Move c0 into generation 2. + gc.collect() + + c1 = C1055820(1) + c1.keep_c0_alive = c0 + del c0.loop # now only c1 keeps c0 alive + + c2 = C1055820(2) + c2wr = weakref.ref(c2) # no callback! + + ouch = [] + def callback(ignored): + ouch[:] = [c2wr()] + + # The callback gets associated with a wr on an object in generation 2. + c0wr = weakref.ref(c0, callback) + + c0 = c1 = c2 = None + + # What we've set up: c0, c1, and c2 are all trash now. c0 is in + # generation 2. The only thing keeping it alive is that c1 points to + # it. c1 and c2 are in generation 0, and are in self-loops. There's a + # global weakref to c2 (c2wr), but that weakref has no callback. + # There's also a global weakref to c0 (c0wr), and that does have a + # callback, and that callback references c2 via c2wr(). + # + # c0 has a wr with callback, which references c2wr + # ^ + # | + # | Generation 2 above dots + #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . . + # | Generation 0 below dots + # | + # | + # ^->c1 ^->c2 has a wr but no callback + # | | | | + # <--v <--v + # + # So this is the nightmare: when generation 0 gets collected, we see + # that c2 has a callback-free weakref, and c1 doesn't even have a + # weakref. Collecting generation 0 doesn't see c0 at all, and c0 is + # the only object that has a weakref with a callback. gc clears c1 + # and c2. Clearing c1 has the side effect of dropping the refcount on + # c0 to 0, so c0 goes away (despite that it's in an older generation) + # and c0's wr callback triggers. That in turn materializes a reference + # to c2 via c2wr(), but c2 gets cleared anyway by gc. + + # We want to let gc happen "naturally", to preserve the distinction + # between generations. + junk = [] + i = 0 + detector = GC_Detector() + if Py_GIL_DISABLED: + # The free-threaded build doesn't have multiple generations, so + # just trigger a GC manually. + gc.collect() + while not detector.gc_happened: + i += 1 + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") + self.assertEqual(len(ouch), 0) + junk.append([]) # this will eventually trigger gc + + self.assertEqual(len(ouch), 1) # else the callback wasn't invoked + for x in ouch: + # If the callback resurrected c2, the instance would be damaged, + # with an empty __dict__. + self.assertEqual(x, None) + + @gc_threshold(1000, 0, 0) + def test_bug1055820d(self): + # Corresponds to temp2d.py in the bug report. This is very much like + # test_bug1055820c, but uses a __del__ method instead of a weakref + # callback to sneak in a resurrection of cyclic trash. + + ouch = [] + class D(C1055820): + def __del__(self): + ouch[:] = [c2wr()] + + d0 = D(0) + # Move all the above into generation 2. + gc.collect() + + c1 = C1055820(1) + c1.keep_d0_alive = d0 + del d0.loop # now only c1 keeps d0 alive + + c2 = C1055820(2) + c2wr = weakref.ref(c2) # no callback! + + d0 = c1 = c2 = None + + # What we've set up: d0, c1, and c2 are all trash now. d0 is in + # generation 2. The only thing keeping it alive is that c1 points to + # it. c1 and c2 are in generation 0, and are in self-loops. There's + # a global weakref to c2 (c2wr), but that weakref has no callback. + # There are no other weakrefs. + # + # d0 has a __del__ method that references c2wr + # ^ + # | + # | Generation 2 above dots + #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . . + # | Generation 0 below dots + # | + # | + # ^->c1 ^->c2 has a wr but no callback + # | | | | + # <--v <--v + # + # So this is the nightmare: when generation 0 gets collected, we see + # that c2 has a callback-free weakref, and c1 doesn't even have a + # weakref. Collecting generation 0 doesn't see d0 at all. gc clears + # c1 and c2. Clearing c1 has the side effect of dropping the refcount + # on d0 to 0, so d0 goes away (despite that it's in an older + # generation) and d0's __del__ triggers. That in turn materializes + # a reference to c2 via c2wr(), but c2 gets cleared anyway by gc. + + # We want to let gc happen "naturally", to preserve the distinction + # between generations. + detector = GC_Detector() + junk = [] + i = 0 + if Py_GIL_DISABLED: + # The free-threaded build doesn't have multiple generations, so + # just trigger a GC manually. + gc.collect() + while not detector.gc_happened: + i += 1 + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") + self.assertEqual(len(ouch), 0) + junk.append([]) # this will eventually trigger gc + + self.assertEqual(len(ouch), 1) # else __del__ wasn't invoked + for x in ouch: + # If __del__ resurrected c2, the instance would be damaged, with an + # empty __dict__. + self.assertEqual(x, None) + + @gc_threshold(1000, 0, 0) + def test_indirect_calls_with_gc_disabled(self): + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 10000: + self.fail("gc didn't happen after 10000 iterations") + junk.append([]) # this will eventually trigger gc + + try: + gc.disable() + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 10000: + break + junk.append([]) # this may eventually trigger gc (if it is enabled) + + self.assertEqual(i, 10001) + finally: + gc.enable() + + # Ensure that setting *threshold0* to zero disables collection. + @gc_threshold(0) + def test_threshold_zero(self): + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 50000: + break + junk.append([]) # this may eventually trigger gc (if it is enabled) + + self.assertEqual(i, 50001) + + +class PythonFinalizationTests(unittest.TestCase): + def test_ast_fini(self): + # bpo-44184: Regression test for subtype_dealloc() when deallocating + # an AST instance also destroy its AST type: subtype_dealloc() must + # not access the type memory after deallocating the instance, since + # the type memory can be freed as well. The test is also related to + # _PyAST_Fini() which clears references to AST types. + code = textwrap.dedent(""" + import ast + import codecs + from test import support + + # Small AST tree to keep their AST types alive + tree = ast.parse("def f(x, y): return 2*x-y") + + # Store the tree somewhere to survive until the last GC collection + support.late_deletion(tree) + """) + assert_python_ok("-c", code) + + +def setUpModule(): + global enabled, debug + enabled = gc.isenabled() + gc.disable() + assert not gc.isenabled() + debug = gc.get_debug() + gc.set_debug(debug & ~gc.DEBUG_LEAK) # this test is supposed to leak + gc.collect() # Delete 2nd generation garbage + + +def tearDownModule(): + gc.set_debug(debug) + # test gc.enable() even if GC is disabled by default + if verbose: + print("restoring automatic collection") + # make sure to always test gc.enable() + gc.enable() + assert gc.isenabled() + if not enabled: + gc.disable() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 8833a102b3..d4c42a24c3 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -176,7 +176,6 @@ def f(): g.send(0) self.assertEqual(next(g), 1) - @unittest.expectedFailure # TODO: RUSTPYTHON; NotImplementedError def test_handle_frame_object_in_creation(self): #Attempt to expose partially constructed frames diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 997c98c2c6..5fbadd5c04 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2449,8 +2449,6 @@ def raise_it(): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=raise_it) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_preexec_gc_module_failure(self): # This tests the code that disables garbage collection if the child # process will execute any Python. diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 76eda799da..5548f4c71a 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -344,7 +344,6 @@ def test_super_argcount(self): with self.assertRaisesRegex(TypeError, "expected at most"): super(int, int, int) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "argument 1 must be a type" does not match "Expected type 'type' but 'int' found." def test_super_argtype(self): with self.assertRaisesRegex(TypeError, "argument 1 must be a type"): super(1, int) @@ -409,7 +408,6 @@ def method(self): with self.assertRaisesRegex(AttributeError, "'super' object has no attribute 'msg'"): C().method() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "argument 1 must be a type" does not match "Expected type 'type' but 'int' found." def test_bad_first_arg(self): class C: def method(self): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 8f86792e82..7e38859311 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1008,8 +1008,6 @@ def test_1_join_on_shutdown(self): @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - # TODO: RUSTPYTHON need to fix test_1_join_on_shutdown then this might work - @unittest.expectedFailure def test_2_join_in_forked_process(self): # Like the test above, but from a forked interpreter script = """if 1: @@ -1730,8 +1728,6 @@ def worker(started, cont, interrupted): class AtexitTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_atexit_output(self): rc, out, err = assert_python_ok("-c", """if True: import threading diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index f47c17b723..1483ed82d4 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -859,13 +859,9 @@ def cb(self, ignore): gc.collect() self.assertEqual(alist, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gc_during_ref_creation(self): self.check_gc_during_creation(weakref.ref) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gc_during_proxy_creation(self): self.check_gc_during_creation(weakref.proxy) @@ -1353,13 +1349,9 @@ def check_len_cycles(self, dict_type, cons): self.assertIn(n1, (0, 1)) self.assertEqual(n2, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_keyed_len_cycles(self): self.check_len_cycles(weakref.WeakKeyDictionary, lambda k: (k, 1)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_valued_len_cycles(self): self.check_len_cycles(weakref.WeakValueDictionary, lambda k: (1, k)) @@ -1387,13 +1379,9 @@ def check_len_race(self, dict_type, cons): self.assertGreaterEqual(n2, 0) self.assertLessEqual(n2, n1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_keyed_len_race(self): self.check_len_race(weakref.WeakKeyDictionary, lambda k: (k, 1)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_valued_len_race(self): self.check_len_race(weakref.WeakValueDictionary, lambda k: (1, k)) diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index b321dac6c6..1fe9ff059b 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -752,7 +752,7 @@ def testEnterReturnsTuple(self): self.assertEqual(10, b1) self.assertEqual(20, b2) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'FrameSummary' object has no attribute 'end_lineno' + @unittest.expectedFailure # TODO: RUSTPYTHON; colno/end_colno not set correctly def testExceptionLocation(self): # The location of an exception raised from # __init__, __enter__ or __exit__ of a context diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 9fd7ea3880..7d16b293bd 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -31,6 +31,12 @@ parking_lot = { workspace = true, optional = true } unicode_names2 = { workspace = true } radium = { workspace = true } +# EBR dependencies +crossbeam-utils = { workspace = true } +memoffset = { workspace = true } +scopeguard = { workspace = true } +atomic = { workspace = true } + lock_api = "0.4" siphasher = "1" num-complex.workspace = true diff --git a/crates/common/src/ebr/collector.rs b/crates/common/src/ebr/collector.rs new file mode 100644 index 0000000000..fa546d81b1 --- /dev/null +++ b/crates/common/src/ebr/collector.rs @@ -0,0 +1,423 @@ +/// Epoch-based garbage collector. +use core::fmt; +use core::sync::atomic::Ordering; +use std::sync::Arc; + +use super::Epoch; +use super::guard::Guard; +use super::internal::{Global, Local}; + +/// A garbage collector based on *epoch-based reclamation* (EBR). +pub struct Collector { + pub(crate) global: Arc, +} + +unsafe impl Send for Collector {} +unsafe impl Sync for Collector {} + +impl Default for Collector { + // https://github.com/rust-lang/rust-clippy/issues/11382 + #[allow(clippy::arc_with_non_send_sync)] + fn default() -> Self { + Self { + global: Arc::new(Global::new()), + } + } +} + +impl Collector { + /// Creates a new collector. + pub fn new() -> Self { + Self::default() + } + + /// Registers a new handle for the collector. + pub fn register(&self) -> LocalHandle { + Local::register(self) + } + + /// Reads the global epoch, without issueing a fence. + #[inline] + pub fn global_epoch(&self) -> Epoch { + self.global.epoch.load(Ordering::Relaxed) + } +} + +impl Clone for Collector { + /// Creates another reference to the same garbage collector. + fn clone(&self) -> Self { + Collector { + global: self.global.clone(), + } + } +} + +impl fmt::Debug for Collector { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Collector { .. }") + } +} + +impl PartialEq for Collector { + /// Checks if both handles point to the same collector. + fn eq(&self, rhs: &Collector) -> bool { + Arc::ptr_eq(&self.global, &rhs.global) + } +} +impl Eq for Collector {} + +/// A handle to a garbage collector. +pub struct LocalHandle { + pub(crate) local: *const Local, +} + +impl LocalHandle { + /// Pins the handle. + #[inline] + pub fn pin(&self) -> Guard { + unsafe { (*self.local).pin() } + } + + /// Returns `true` if the handle is pinned. + #[cfg(test)] + #[inline] + pub(crate) fn is_pinned(&self) -> bool { + unsafe { (*self.local).is_pinned() } + } +} + +impl Drop for LocalHandle { + #[inline] + fn drop(&mut self) { + unsafe { + Local::release_handle(&*self.local); + } + } +} + +impl fmt::Debug for LocalHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("LocalHandle { .. }") + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use crossbeam_utils::thread; + + use crate::ebr::{RawShared, collector::Collector}; + + const NUM_THREADS: usize = 8; + + #[test] + fn pin_reentrant() { + let collector = Collector::new(); + let handle = collector.register(); + drop(collector); + + assert!(!handle.is_pinned()); + { + let _guard = &handle.pin(); + assert!(handle.is_pinned()); + { + let _guard = &handle.pin(); + assert!(handle.is_pinned()); + } + assert!(handle.is_pinned()); + } + assert!(!handle.is_pinned()); + } + + #[test] + fn flush_local_bag() { + let collector = Collector::new(); + let handle = collector.register(); + drop(collector); + + for _ in 0..100 { + let guard = &handle.pin(); + unsafe { + let a = RawShared::from_owned(7); + guard.defer_destroy(a); + + let is_empty = || (*(*guard.local).bag.get()).is_empty(); + assert!(!is_empty()); + + while !is_empty() { + guard.flush(); + } + } + } + } + + #[test] + fn garbage_buffering() { + let collector = Collector::new(); + let handle = collector.register(); + drop(collector); + + let guard = &handle.pin(); + unsafe { + for _ in 0..10 { + let a = RawShared::from_owned(7); + guard.defer_destroy(a); + } + assert!(!(*(*guard.local).bag.get()).is_empty()); + } + } + + #[test] + fn pin_holds_advance() { + #[cfg(miri)] + const N: usize = 500; + #[cfg(not(miri))] + const N: usize = 500_000; + + let collector = Collector::new(); + + thread::scope(|scope| { + for _ in 0..NUM_THREADS { + scope.spawn(|_| { + let handle = collector.register(); + for _ in 0..N { + let guard = &handle.pin(); + + let before = collector.global.epoch.load(Ordering::Relaxed); + collector.global.collect(guard); + let after = collector.global.epoch.load(Ordering::Relaxed); + + assert!(after.wrapping_sub(before) <= 2); + } + }); + } + }) + .unwrap(); + } + + #[test] + fn buffering() { + const COUNT: usize = 10; + #[cfg(miri)] + const N: usize = 500; + #[cfg(not(miri))] + const N: usize = 100_000; + static DESTROYS: AtomicUsize = AtomicUsize::new(0); + + let collector = Collector::new(); + let handle = collector.register(); + + unsafe { + let guard = &handle.pin(); + for _ in 0..COUNT { + let a = RawShared::from_owned(7); + guard.defer_unchecked(move || { + a.drop(); + DESTROYS.fetch_add(1, Ordering::Relaxed); + }); + } + } + + for _ in 0..N { + collector.global.collect(&handle.pin()); + } + assert!(DESTROYS.load(Ordering::Relaxed) < COUNT); + + handle.pin().flush(); + + while DESTROYS.load(Ordering::Relaxed) < COUNT { + let guard = &handle.pin(); + collector.global.collect(guard); + } + assert_eq!(DESTROYS.load(Ordering::Relaxed), COUNT); + } + + #[test] + fn count_drops() { + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 100_000; + static DROPS: AtomicUsize = AtomicUsize::new(0); + + #[allow(dead_code)] + struct Elem(i32); + + impl Drop for Elem { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::Relaxed); + } + } + + let collector = Collector::new(); + let handle = collector.register(); + + unsafe { + let guard = &handle.pin(); + + for _ in 0..COUNT { + let a = RawShared::from_owned(Elem(7)); + guard.defer_destroy(a); + } + guard.flush(); + } + + while DROPS.load(Ordering::Relaxed) < COUNT { + let guard = &handle.pin(); + collector.global.collect(guard); + } + assert_eq!(DROPS.load(Ordering::Relaxed), COUNT); + } + + #[test] + fn count_destroy() { + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 100_000; + static DESTROYS: AtomicUsize = AtomicUsize::new(0); + + let collector = Collector::new(); + let handle = collector.register(); + + unsafe { + let guard = &handle.pin(); + + for _ in 0..COUNT { + let a = RawShared::from_owned(7); + guard.defer_unchecked(move || { + a.drop(); + DESTROYS.fetch_add(1, Ordering::Relaxed); + }); + } + guard.flush(); + } + + while DESTROYS.load(Ordering::Relaxed) < COUNT { + let guard = &handle.pin(); + collector.global.collect(guard); + } + assert_eq!(DESTROYS.load(Ordering::Relaxed), COUNT); + } + + #[test] + fn drop_array() { + const COUNT: usize = 700; + static DROPS: AtomicUsize = AtomicUsize::new(0); + + #[allow(dead_code)] + struct Elem(i32); + + impl Drop for Elem { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::Relaxed); + } + } + + let collector = Collector::new(); + let handle = collector.register(); + + let mut guard = handle.pin(); + + let mut v = Vec::with_capacity(COUNT); + for i in 0..COUNT { + v.push(Elem(i as i32)); + } + + { + let a = RawShared::from_owned(v); + unsafe { + guard.defer_destroy(a); + } + guard.flush(); + } + + while DROPS.load(Ordering::Relaxed) < COUNT { + guard.reactivate(); + collector.global.collect(&guard); + } + assert_eq!(DROPS.load(Ordering::Relaxed), COUNT); + } + + #[test] + fn destroy_array() { + use std::mem::ManuallyDrop; + + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 100_000; + static DESTROYS: AtomicUsize = AtomicUsize::new(0); + + let collector = Collector::new(); + let handle = collector.register(); + + unsafe { + let guard = &handle.pin(); + + let mut v = Vec::with_capacity(COUNT); + for i in 0..COUNT { + v.push(i as i32); + } + + let len = v.len(); + let ptr = ManuallyDrop::new(v).as_mut_ptr() as usize; + guard.defer_unchecked(move || { + drop(Vec::from_raw_parts(ptr as *const i32 as *mut i32, len, len)); + DESTROYS.fetch_add(len, Ordering::Relaxed); + }); + guard.flush(); + } + + while DESTROYS.load(Ordering::Relaxed) < COUNT { + let guard = &handle.pin(); + collector.global.collect(guard); + } + assert_eq!(DESTROYS.load(Ordering::Relaxed), COUNT); + } + + #[test] + fn stress() { + const THREADS: usize = 8; + #[cfg(miri)] + const COUNT: usize = 500; + #[cfg(not(miri))] + const COUNT: usize = 100_000; + static DROPS: AtomicUsize = AtomicUsize::new(0); + + #[allow(dead_code)] + struct Elem(i32); + + impl Drop for Elem { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::Relaxed); + } + } + + let collector = Collector::new(); + + thread::scope(|scope| { + for _ in 0..THREADS { + scope.spawn(|_| { + let handle = collector.register(); + for _ in 0..COUNT { + let guard = &handle.pin(); + unsafe { + let a = RawShared::from_owned(Elem(7i32)); + guard.defer_destroy(a); + } + } + }); + } + }) + .unwrap(); + + let handle = collector.register(); + while DROPS.load(Ordering::Relaxed) < COUNT * THREADS { + let guard = &handle.pin(); + collector.global.collect(guard); + } + assert_eq!(DROPS.load(Ordering::Relaxed), COUNT * THREADS); + } +} diff --git a/crates/common/src/ebr/default.rs b/crates/common/src/ebr/default.rs new file mode 100644 index 0000000000..5275bf9cdf --- /dev/null +++ b/crates/common/src/ebr/default.rs @@ -0,0 +1,77 @@ +//! The default garbage collector. +//! +//! For each thread, a participant is lazily initialized on its first use, when the current thread +//! is registered in the default collector. If initialized, the thread's participant will get +//! destructed on thread exit, which in turn unregisters the thread. + +use super::collector::{Collector, LocalHandle}; +use super::guard::Guard; +use super::sync::once_lock::OnceLock; + +fn collector() -> &'static Collector { + /// The global data for the default garbage collector. + static COLLECTOR: OnceLock = OnceLock::new(); + COLLECTOR.get_or_init(Collector::new) +} + +thread_local! { + /// The per-thread participant for the default garbage collector. + static HANDLE: LocalHandle = collector().register(); +} + +/// Enters EBR critical section. +#[inline] +pub fn cs() -> Guard { + with_handle(|handle| handle.pin()) +} + +/// Returns the default global collector. +pub fn default_collector() -> &'static Collector { + collector() +} + +#[inline] +fn with_handle(mut f: F) -> R +where + F: FnMut(&LocalHandle) -> R, +{ + HANDLE + .try_with(|h| f(h)) + .unwrap_or_else(|_| f(&collector().register())) +} + +#[inline] +pub fn global_epoch() -> usize { + default_collector().global_epoch().value() +} + +#[cfg(test)] +mod tests { + use crossbeam_utils::thread; + + #[test] + fn pin_while_exiting() { + struct Foo; + + impl Drop for Foo { + fn drop(&mut self) { + // Pin after `HANDLE` has been dropped. This must not panic. + super::cs(); + } + } + + thread_local! { + static FOO: Foo = const { Foo }; + } + + thread::scope(|scope| { + scope.spawn(|_| { + // Initialize `FOO` and then `HANDLE`. + FOO.with(|_| ()); + super::cs(); + // At thread exit, `HANDLE` gets dropped first and `FOO` second. + }); + }) + .unwrap(); + } +} diff --git a/crates/common/src/ebr/deferred.rs b/crates/common/src/ebr/deferred.rs new file mode 100644 index 0000000000..500aa88b98 --- /dev/null +++ b/crates/common/src/ebr/deferred.rs @@ -0,0 +1,138 @@ +use core::fmt; +use core::marker::PhantomData; +use core::mem::{self, MaybeUninit}; +use core::ptr; + +/// Number of words a piece of `Data` can hold. +/// +/// Three words should be enough for the majority of cases. For example, you can fit inside it the +/// function pointer together with a fat pointer representing an object that needs to be destroyed. +const DATA_WORDS: usize = 3; + +/// Some space to keep a `FnOnce()` object on the stack. +type Data = [usize; DATA_WORDS]; + +/// A `FnOnce()` that is stored inline if small, or otherwise boxed on the heap. +/// +/// This is a handy way of keeping an unsized `FnOnce()` within a sized structure. +pub(crate) struct Deferred { + call: unsafe fn(*mut u8), + data: MaybeUninit, + _marker: PhantomData<*mut ()>, // !Send + !Sync +} + +impl fmt::Debug for Deferred { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + f.pad("Deferred { .. }") + } +} + +impl Deferred { + /// Constructs a new `Deferred` from a `FnOnce()`. + pub(crate) fn new(f: F) -> Self { + let size = mem::size_of::(); + let align = mem::align_of::(); + + unsafe { + if size <= mem::size_of::() && align <= mem::align_of::() { + let mut data = MaybeUninit::::uninit(); + ptr::write(data.as_mut_ptr().cast::(), f); + + unsafe fn call(raw: *mut u8) { + let f: F = unsafe { ptr::read(raw.cast::()) }; + f(); + } + + Deferred { + call: call::, + data, + _marker: PhantomData, + } + } else { + let b: Box = Box::new(f); + let mut data = MaybeUninit::::uninit(); + ptr::write(data.as_mut_ptr().cast::>(), b); + + unsafe fn call(raw: *mut u8) { + // It's safe to cast `raw` from `*mut u8` to `*mut Box`, because `raw` is + // originally derived from `*mut Box`. + let b: Box = unsafe { ptr::read(raw.cast::>()) }; + (*b)(); + } + + Deferred { + call: call::, + data, + _marker: PhantomData, + } + } + } + } + + /// Calls the function. + #[inline] + pub(crate) fn call(mut self) { + let call = self.call; + unsafe { call(self.data.as_mut_ptr().cast::()) }; + } +} + +#[cfg(test)] +mod tests { + #![allow(dropping_copy_types)] + + use super::Deferred; + use core::hint::black_box; + use std::cell::Cell; + + #[test] + fn on_stack() { + let fired = &Cell::new(false); + let a = [0usize; 1]; + + let d = Deferred::new(move || { + black_box(a); + fired.set(true); + }); + + assert!(!fired.get()); + d.call(); + assert!(fired.get()); + } + + #[test] + fn on_heap() { + let fired = &Cell::new(false); + let a = [0usize; 10]; + + let d = Deferred::new(move || { + black_box(a); + fired.set(true); + }); + + assert!(!fired.get()); + d.call(); + assert!(fired.get()); + } + + #[test] + fn string() { + let a = "hello".to_string(); + let d = Deferred::new(move || assert_eq!(a, "hello")); + d.call(); + } + + #[test] + fn boxed_slice_i32() { + let a: Box<[i32]> = vec![2, 3, 5, 7].into_boxed_slice(); + let d = Deferred::new(move || assert_eq!(*a, [2, 3, 5, 7])); + d.call(); + } + + #[test] + fn long_slice_usize() { + let a: [usize; 5] = [2, 3, 5, 7, 11]; + let d = Deferred::new(move || assert_eq!(a, [2, 3, 5, 7, 11])); + d.call(); + } +} diff --git a/crates/common/src/ebr/epoch.rs b/crates/common/src/ebr/epoch.rs new file mode 100644 index 0000000000..a0743053a9 --- /dev/null +++ b/crates/common/src/ebr/epoch.rs @@ -0,0 +1,138 @@ +//! The global epoch +//! +//! The last bit in this number is unused and is always zero. Every so often the global epoch is +//! incremented, i.e. we say it "advances". A pinned participant may advance the global epoch only +//! if all currently pinned participants have been pinned in the current epoch. +//! +//! If an object became garbage in some epoch, then we can be sure that after two advancements no +//! participant will hold a reference to it. That is the crux of safe memory reclamation. + +use core::sync::atomic::{AtomicUsize, Ordering}; + +/// An epoch that can be marked as pinned or unpinned. +/// +/// Internally, the epoch is represented as an integer that wraps around at some unspecified point +/// and a flag that represents whether it is pinned or unpinned. +#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)] +pub struct Epoch { + /// The least significant bit is set if pinned. The rest of the bits hold the epoch. + data: usize, +} + +impl Epoch { + /// Returns the starting epoch in unpinned state. + #[inline] + pub(crate) fn starting() -> Self { + Self::default() + } + + /// Returns the number of epochs `self` is ahead of `rhs`. + /// + /// Internally, epochs are represented as numbers in the range `(isize::MIN / 2) .. (isize::MAX + /// / 2)`, so the returned distance will be in the same interval. + pub fn wrapping_sub(self, rhs: Self) -> isize { + // The result is the same with `(self.data & !1).wrapping_sub(rhs.data & !1) as isize >> 1`, + // because the possible difference of LSB in `(self.data & !1).wrapping_sub(rhs.data & !1)` + // will be ignored in the shift operation. + self.data.wrapping_sub(rhs.data & !1) as isize >> 1 + } + + /// Returns `true` if the epoch is marked as pinned. + #[inline] + pub(crate) fn is_pinned(self) -> bool { + (self.data & 1) == 1 + } + + /// Returns the same epoch, but marked as pinned. + #[inline] + pub(crate) fn pinned(self) -> Epoch { + Epoch { + data: self.data | 1, + } + } + + /// Returns the same epoch, but marked as unpinned. + #[inline] + pub(crate) fn unpinned(self) -> Epoch { + Epoch { + data: self.data & !1, + } + } + + /// Returns the successor epoch. + /// + /// The returned epoch will be marked as pinned only if the previous one was as well. + #[inline] + pub(crate) fn successor(self) -> Epoch { + Epoch { + data: self.data.wrapping_add(2), + } + } + + /// Returns the epoch value. + #[inline] + pub fn value(self) -> usize { + self.unpinned().data >> 1 + } +} + +/// An atomic value that holds an `Epoch`. +#[derive(Default, Debug)] +pub(crate) struct AtomicEpoch { + /// Since `Epoch` is just a wrapper around `usize`, an `AtomicEpoch` is similarly represented + /// using an `AtomicUsize`. + data: AtomicUsize, +} + +impl AtomicEpoch { + /// Creates a new atomic epoch. + #[inline] + pub(crate) fn new(epoch: Epoch) -> Self { + let data = AtomicUsize::new(epoch.data); + AtomicEpoch { data } + } + + /// Loads a value from the atomic epoch. + #[inline] + pub(crate) fn load(&self, ord: Ordering) -> Epoch { + Epoch { + data: self.data.load(ord), + } + } + + /// Stores a value into the atomic epoch. + #[inline] + pub(crate) fn store(&self, epoch: Epoch, ord: Ordering) { + self.data.store(epoch.data, ord); + } + + /// Stores a value into the atomic epoch if the current value is the same as `current`. + /// + /// The return value is a result indicating whether the new value was written and containing + /// the previous value. On success this value is guaranteed to be equal to `current`. + /// + /// This method takes two `Ordering` arguments to describe the memory + /// ordering of this operation. `success` describes the required ordering for the + /// read-modify-write operation that takes place if the comparison with `current` succeeds. + /// `failure` describes the required ordering for the load operation that takes place when + /// the comparison fails. Using `Acquire` as success ordering makes the store part + /// of this operation `Relaxed`, and using `Release` makes the successful load + /// `Relaxed`. The failure ordering can only be `SeqCst`, `Acquire` or `Relaxed` + /// and must be equivalent to or weaker than the success ordering. + #[inline] + pub(crate) fn compare_exchange( + &self, + current: Epoch, + new: Epoch, + success: Ordering, + failure: Ordering, + ) -> Result { + match self + .data + .compare_exchange(current.data, new.data, success, failure) + { + Ok(data) => Ok(Epoch { data }), + Err(data) => Err(Epoch { data }), + } + } +} diff --git a/crates/common/src/ebr/guard.rs b/crates/common/src/ebr/guard.rs new file mode 100644 index 0000000000..7b7b2ce93b --- /dev/null +++ b/crates/common/src/ebr/guard.rs @@ -0,0 +1,177 @@ +use core::fmt; +use core::mem; + +use scopeguard::defer; + +use super::RawShared; +use super::deferred::Deferred; +use super::internal::Local; + +/// A RAII-style guard that keeps the current thread in an EBR critical section. +pub struct Guard { + pub(crate) local: *const Local, +} + +impl Guard { + /// Stores a function so that it can be executed at some point after all currently pinned + /// threads get unpinned. + /// + /// This method first stores `f` into the thread-local (or handle-local) cache. If this cache + /// becomes full, some functions are moved into the global cache. At the same time, some + /// functions from both local and global caches may get executed in order to incrementally + /// clean up the caches as they fill up. + /// + /// There is no guarantee when exactly `f` will be executed. The only guarantee is that it + /// won't be executed until all currently pinned threads get unpinned. In theory, `f` might + /// never run, but the epoch-based garbage collection will make an effort to execute it + /// reasonably soon. + /// + /// If this method is called from an [`unprotected`] guard, the function will simply be + /// executed immediately. + /// + /// # Safety + /// + /// The given function must not hold reference onto the stack. It is highly recommended that + /// the passed function is **always** marked with `move` in order to prevent accidental + /// borrows. + /// + /// Apart from that, keep in mind that another thread may execute `f`, so anything accessed by + /// the closure must be `Send`. + /// # Safety + /// + /// The given function must not hold reference onto the stack. It is highly recommended that + /// the passed function is **always** marked with `move` in order to prevent accidental + /// borrows. The closure must be `Send` as another thread may execute it. + pub unsafe fn defer_unchecked(&self, f: F) + where + F: FnOnce() -> R, + { + // SAFETY: caller must ensure this is safe + if let Some(local) = unsafe { self.local.as_ref() } { + unsafe { local.defer(Deferred::new(move || drop(f())), self) }; + } else { + drop(f()); + } + } + + /// Stores a destructor for an object so that it can be deallocated and dropped at some point + /// after all currently pinned threads get unpinned. + /// + /// This method first stores the destructor into the thread-local (or handle-local) cache. If + /// this cache becomes full, some destructors are moved into the global cache. At the same + /// time, some destructors from both local and global caches may get executed in order to + /// incrementally clean up the caches as they fill up. + /// + /// There is no guarantee when exactly the destructor will be executed. The only guarantee is + /// that it won't be executed until all currently pinned threads get unpinned. In theory, the + /// destructor might never run, but the epoch-based garbage collection will make an effort to + /// execute it reasonably soon. + /// + /// If this method is called from an [`unprotected`] guard, the destructor will simply be + /// executed immediately. + /// + /// # Safety + /// + /// The object must not be reachable by other threads anymore, otherwise it might be still in + /// use when the destructor runs. + /// + /// Apart from that, keep in mind that another thread may execute the destructor, so the object + /// must be sendable to other threads. + pub(crate) unsafe fn defer_destroy(&self, ptr: RawShared<'_, T>) { + // SAFETY: caller guarantees ptr is valid and no longer reachable + unsafe { self.defer_unchecked(move || ptr.drop()) }; + } + + /// Clears up the thread-local cache of deferred functions by executing them or moving into the + /// global cache. + /// + /// Call this method after deferring execution of a function if you want to get it executed as + /// soon as possible. Flushing will make sure it is residing in in the global cache, so that + /// any thread has a chance of taking the function and executing it. + pub fn flush(&self) { + if let Some(local) = unsafe { self.local.as_ref() } { + local.flush(self); + } + } + + /// Deactivate and reactivate the critical section. + /// + /// This method is useful when you don't want delay the advancement of the global epoch by + /// holding an old epoch. For safety, you should not maintain any guard-based reference across + /// the call (the latter is enforced by `&mut self`). The thread will only be repinned if this + /// is the only active guard for the current thread. + pub fn reactivate(&mut self) { + if let Some(local) = unsafe { self.local.as_ref() } { + local.repin(); + } + } + + /// Temporarily deactivate the critical section, executes the given function and then + /// reactivates the critical section. + /// + /// This method is useful when you need to perform a long-running operation (e.g. sleeping) + /// and don't need to maintain any guard-based reference across the call (the latter is enforced + /// by `&mut self`). The thread will only be unpinned if this is the only active guard for the + /// current thread. + pub fn reactivate_after(&mut self, f: F) -> R + where + F: FnOnce() -> R, + { + if let Some(local) = unsafe { self.local.as_ref() } { + // We need to acquire a handle here to ensure the Local doesn't + // disappear from under us. + local.acquire_handle(); + local.unpin(); + } + + // Ensure the Guard is re-pinned even if the function panics + defer! { + if let Some(local) = unsafe { self.local.as_ref() } { + mem::forget(local.pin()); + local.release_handle(); + } + } + + f() + } + + /// Increases the manual collection counter, and perform collection if the counter reaches + /// the threshold which is set by `set_manual_collection_interval`. + #[allow(dead_code)] + pub(crate) fn incr_manual_collection(&self) { + if let Some(local) = unsafe { self.local.as_ref() } { + local.incr_manual_collection(self); + } + } +} + +impl Drop for Guard { + #[inline] + fn drop(&mut self) { + if let Some(local) = unsafe { self.local.as_ref() } { + local.unpin(); + } + } +} + +impl fmt::Debug for Guard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Guard { .. }") + } +} + +/// Returns a reference to a dummy guard that allows unprotected access to atomic pointers. +/// +/// This guard should be used in special occasions only. Note that it doesn't actually keep any +/// thread pinned - it's just a fake guard that allows loading from atomics unsafely. +/// +/// # Safety +/// +/// Loading and dereferencing data from atomic shared pointers using this guard is safe only if +/// the pointers are not being concurrently modified by other threads. +#[inline] +pub unsafe fn unprotected() -> Guard { + Guard { + local: core::ptr::null(), + } +} diff --git a/crates/common/src/ebr/internal.rs b/crates/common/src/ebr/internal.rs new file mode 100644 index 0000000000..72d67147bd --- /dev/null +++ b/crates/common/src/ebr/internal.rs @@ -0,0 +1,638 @@ +//! The global data and participant for garbage collection. +//! +//! # Registration +//! +//! In order to track all participants in one place, we need some form of participant +//! registration. When a participant is created, it is registered to a global lock-free +//! singly-linked list of registries; and when a participant is leaving, it is unregistered from the +//! list. +//! +//! # Pinning +//! +//! Every participant contains an integer that tells whether the participant is pinned and if so, +//! what was the global epoch at the time it was pinned. Participants also hold a pin counter that +//! aids in periodic global epoch advancement. +//! +//! When a participant is pinned, a `Guard` is returned as a witness that the participant is pinned. +//! Guards are necessary for performing atomic operations, and for freeing/dropping locations. +//! +//! # Thread-local bag +//! +//! Objects that get unlinked from concurrent data structures must be stashed away until the global +//! epoch sufficiently advances so that they become safe for destruction. Pointers to such objects +//! are pushed into a thread-local bag, and when it becomes full, the bag is marked with the current +//! global epoch and pushed into the global queue of bags. We store objects in thread-local storages +//! for amortizing the synchronization cost of pushing the garbages to a global queue. +//! +//! # Global queue +//! +//! Whenever a bag is pushed into a queue, the objects in some bags in the queue are collected and +//! destroyed along the way. This design reduces contention on data structures. The global queue +//! cannot be explicitly accessed: the only way to interact with it is by calling functions +//! `defer()` that adds an object to the thread-local bag, or `collect()` that manually triggers +//! garbage collection. +//! +//! Ideally each instance of concurrent data structure may have its own queue that gets fully +//! destroyed as soon as the data structure gets dropped. + +use super::RawShared; +use core::cell::{Cell, UnsafeCell}; +use core::mem::{ManuallyDrop, forget, replace}; +use core::sync::atomic::{Ordering, compiler_fence}; +use core::{fmt, ptr}; + +use crossbeam_utils::CachePadded; +use memoffset::offset_of; + +use super::collector::{Collector, LocalHandle}; +use super::deferred::Deferred; +use super::epoch::{AtomicEpoch, Epoch}; +use super::guard::{Guard, unprotected}; +use super::sync::list::{Entry, IsElement, IterError, List}; +use super::sync::queue::Queue; + +/// Maximum number of objects a bag can contain. +static mut MAX_OBJECTS: usize = 64; + +#[allow(dead_code)] +static mut MANUAL_EVENTS_BETWEEN_COLLECT: usize = 64; + +/// A bag of deferred functions. +pub(crate) struct Bag(Vec); + +/// `Bag::try_push()` requires that it is safe for another thread to execute the given functions. +unsafe impl Send for Bag {} + +impl Bag { + /// Returns a new, empty bag. + pub(crate) fn new() -> Self { + Self::default() + } + + /// Returns `true` if the bag is empty. + pub(crate) fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Attempts to insert a deferred function into the bag. + /// + /// Returns `Ok(())` if successful, and `Err(deferred)` for the given `deferred` if the bag is + /// full. + /// + /// # Safety + /// + /// It should be safe for another thread to execute the given function. + pub(crate) unsafe fn try_push(&mut self, deferred: Deferred) -> Result<(), Deferred> { + if self.0.len() < self.0.capacity() { + self.0.push(deferred); + Ok(()) + } else { + Err(deferred) + } + } + + /// Seals the bag with the given epoch. + fn seal(self, epoch: Epoch) -> SealedBag { + SealedBag { epoch, _bag: self } + } +} + +impl Default for Bag { + fn default() -> Self { + Bag(Vec::with_capacity(unsafe { MAX_OBJECTS })) + } +} + +impl Drop for Bag { + fn drop(&mut self) { + // Call all deferred functions. + for deferred in self.0.drain(..) { + deferred.call(); + } + } +} + +// can't #[derive(Debug)] because Debug is not implemented for arrays 64 items long +impl fmt::Debug for Bag { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Bag").field("deferreds", &self.0).finish() + } +} + +/// A pair of an epoch and a bag. +#[derive(Default, Debug)] +struct SealedBag { + epoch: Epoch, + _bag: Bag, +} + +/// It is safe to share `SealedBag` because `is_expired` only inspects the epoch. +unsafe impl Sync for SealedBag {} + +impl SealedBag { + /// Checks if it is safe to drop the bag w.r.t. the given global epoch. + fn is_expired(&self, global_epoch: Epoch) -> bool { + // A pinned participant can witness at most one epoch advancement. Therefore, any bag that + // is within one epoch of the current one cannot be destroyed yet. + // NOTE: This version of EBR maintain epoch skew of threads ≤ 1 and reclaim garbages + // three epochs ago. + global_epoch.wrapping_sub(self.epoch) >= 3 + } +} + +/// The global data for a garbage collector. +pub(crate) struct Global { + /// The intrusive linked list of `Local`s. + locals: List, + + /// The global queue of bags of deferred functions. + queue: Queue, + + /// The global epoch. + pub(crate) epoch: CachePadded, +} + +impl Global { + const COLLECTS_TRIALS: usize = 16; + + /// Creates a new global data for garbage collection. + #[inline] + pub(crate) fn new() -> Self { + Self { + locals: List::new(), + queue: Queue::new(), + epoch: CachePadded::new(AtomicEpoch::new(Epoch::starting())), + } + } + + /// Pushes the bag into the global queue and replaces the bag with a new empty bag. + pub(crate) fn push_bag(&self, bag: &mut Bag, guard: &Guard) { + let bag = replace(bag, Bag::new()); + + atomic::fence(Ordering::SeqCst); + + let epoch = self.epoch.load(Ordering::Relaxed); + self.queue.push(bag.seal(epoch), guard); + } + + /// Collects several bags from the global queue and executes deferred functions in them. + /// + /// Note: This may itself produce garbage and in turn allocate new bags. + /// + /// `pin()` rarely calls `collect()`, so we want the compiler to place that call on a cold + /// path. In other words, we want the compiler to optimize branching for the case when + /// `collect()` is not called. + #[cold] + pub(crate) fn collect(&self, guard: &Guard) { + if let Some(local) = unsafe { guard.local.as_ref() } { + local.manual_count.set(0); + local.pin_count.set(0); + } + self.try_advance(guard); + + debug_assert!( + !guard.local.is_null(), + "An unprotected guard cannot be used to collect global garbages." + ); + + for _ in 0..Self::COLLECTS_TRIALS { + match self.queue.try_pop_if( + |sealed_bag: &SealedBag| sealed_bag.is_expired(self.epoch.load(Ordering::Relaxed)), + guard, + ) { + None => break, + Some(sealed_bag) => { + drop(sealed_bag); + } + } + } + } + + /// Attempts to advance the global epoch. + /// + /// The global epoch can advance only if all currently pinned participants have been pinned in + /// the current epoch. + /// + /// Returns the current global epoch. + /// + /// `try_advance()` is annotated `#[cold]` because it is rarely called. + #[cold] + pub(crate) fn try_advance(&self, guard: &Guard) -> Epoch { + let global_epoch = self.epoch.load(Ordering::Relaxed); + atomic::fence(Ordering::SeqCst); + + // `Local`s are stored in a linked list because linked lists are fairly + // easy to implement in a lock-free manner. However, traversal can be slow due to cache + // misses and data dependencies. We should experiment with other data structures as well. + for local in self.locals.iter(guard) { + match local { + Err(IterError::Stalled) => { + // A concurrent thread stalled this iteration. That thread might also try to + // advance the epoch, in which case we leave the job to it. Otherwise, the + // epoch will not be advanced. + return global_epoch; + } + Ok(local) => { + let local_epoch = local.epoch.load(Ordering::Relaxed); + + // If the participant was pinned in a different epoch, we cannot advance the + // global epoch just yet. + if local_epoch.is_pinned() && local_epoch.unpinned() != global_epoch { + return global_epoch; + } + } + } + } + atomic::fence(Ordering::Acquire); + + // All pinned participants were pinned in the current global epoch. + // Now let's advance the global epoch... + // + // Note that if another thread already advanced it before us, this store will simply + // overwrite the global epoch with the same value. This is true because `try_advance` was + // called from a thread that was pinned in `global_epoch`, and the global epoch cannot be + // advanced two steps ahead of it. + let new_epoch = global_epoch.successor(); + self.epoch.store(new_epoch, Ordering::Release); + new_epoch + } +} + +/// Participant for garbage collection. +pub(crate) struct Local { + /// A node in the intrusive linked list of `Local`s. + entry: Entry, + + /// A reference to the global data. + /// + /// When all guards and handles get dropped, this reference is destroyed. + collector: UnsafeCell>, + + /// The local bag of deferred functions. + /// + /// Note that removing the global garbage queue and using only the thread local bags + /// will increase the memory consumption in a queue workload. + pub(crate) bag: UnsafeCell, + + /// The number of guards keeping this participant pinned. + guard_count: Cell, + + /// The number of active handles. + handle_count: Cell, + + /// This is just an auxilliary counter that sometimes kicks off collection. + advance_count: Cell, + prev_epoch: Cell, + pin_count: Cell, + pub(crate) manual_count: Cell, + + must_collect: Cell, + collecting: Cell, + + /// The local epoch. + pub(crate) epoch: CachePadded, +} + +impl Local { + const COUNTS_BETWEEN_ADVANCE: usize = 64; + + /// Registers a new `Local` in the provided `Global`. + pub(crate) fn register(collector: &Collector) -> LocalHandle { + unsafe { + // Since we dereference no pointers in this block, it is safe to use `unprotected`. + + let local = RawShared::from_owned(Local { + entry: Entry::default(), + collector: UnsafeCell::new(ManuallyDrop::new(collector.clone())), + bag: UnsafeCell::new(Bag::new()), + guard_count: Cell::new(0), + handle_count: Cell::new(1), + advance_count: Cell::new(0), + prev_epoch: Cell::new(Epoch::starting()), + pin_count: Cell::new(0), + manual_count: Cell::new(0), + must_collect: Cell::new(false), + collecting: Cell::new(false), + epoch: CachePadded::new(AtomicEpoch::new(Epoch::starting())), + }); + collector.global.locals.insert(local, &unprotected()); + LocalHandle { + local: local.as_raw(), + } + } + } + + /// Returns a reference to the `Global` in which this `Local` resides. + #[inline] + pub(crate) fn global(&self) -> &Global { + &self.collector().global + } + + /// Returns a reference to the `Collector` in which this `Local` resides. + #[inline] + pub(crate) fn collector(&self) -> &Collector { + unsafe { &*self.collector.get() } + } + + /// Returns `true` if the current participant is pinned. + #[inline] + #[cfg(test)] + pub(crate) fn is_pinned(&self) -> bool { + self.guard_count.get() > 0 + } + + /// Adds `deferred` to the thread-local bag. + /// + /// # Safety + /// + /// It should be safe for another thread to execute the given function. + pub(crate) unsafe fn defer(&self, mut deferred: Deferred, guard: &Guard) { + // SAFETY: we have exclusive access via guard + let bag = unsafe { &mut *self.bag.get() }; + + while let Err(d) = unsafe { bag.try_push(deferred) } { + self.global().push_bag(bag, guard); + deferred = d; + self.schedule_collection(); + } + self.incr_advance(guard); + } + + pub(crate) fn flush(&self, guard: &Guard) { + self.push_to_global(guard); + self.schedule_collection(); + } + + pub(crate) fn push_to_global(&self, guard: &Guard) { + let bag = unsafe { &mut *self.bag.get() }; + + if !bag.is_empty() { + self.global().push_bag(bag, guard); + } + } + + pub(crate) fn schedule_collection(&self) { + self.must_collect.set(true); + if self.collecting.get() { + self.repin_without_collect(); + } + } + + pub(crate) fn incr_advance(&self, guard: &Guard) { + let advance_count = self.advance_count.get().wrapping_add(1); + self.advance_count.set(advance_count); + + if advance_count.is_multiple_of(Self::COUNTS_BETWEEN_ADVANCE) { + self.global().try_advance(guard); + } + } + + /// Pins the `Local`. + #[inline] + pub(crate) fn pin(&self) -> Guard { + let guard = Guard { local: self }; + + let guard_count = self.guard_count.get(); + self.guard_count.set(guard_count.checked_add(1).unwrap()); + + if guard_count == 0 { + let new_epoch = loop { + let global_epoch = self.global().epoch.load(Ordering::Relaxed); + let new_epoch = global_epoch.pinned(); + + // Now we must store `new_epoch` into `self.epoch` and execute a `SeqCst` fence. + // The fence makes sure that any future loads from `Atomic`s will not happen before + // this store. + if cfg!(all( + any(target_arch = "x86", target_arch = "x86_64"), + not(miri) + )) { + // HACK(stjepang): On x86 architectures there are two different ways of executing + // a `SeqCst` fence. + // + // 1. `atomic::fence(SeqCst)`, which compiles into a `mfence` instruction. + // 2. `_.compare_exchange(_, _, SeqCst, SeqCst)`, which compiles into a `lock cmpxchg` + // instruction. + // + // Both instructions have the effect of a full barrier, but benchmarks have shown + // that the second one makes pinning faster in this particular case. It is not + // clear that this is permitted by the C++ memory model (SC fences work very + // differently from SC accesses), but experimental evidence suggests that this + // works fine. Using inline assembly would be a viable (and correct) alternative, + // but alas, that is not possible on stable Rust. + let current = Epoch::starting(); + let res = self.epoch.compare_exchange( + current, + new_epoch, + Ordering::SeqCst, + Ordering::SeqCst, + ); + debug_assert!(res.is_ok(), "participant was expected to be unpinned"); + // We add a compiler fence to make it less likely for LLVM to do something wrong + // here. Formally, this is not enough to get rid of data races; practically, + // it should go a long way. + compiler_fence(Ordering::SeqCst); + } else { + self.epoch.store(new_epoch, Ordering::Relaxed); + atomic::fence(Ordering::SeqCst); + } + + if new_epoch.value() == self.global().epoch.load(Ordering::Acquire).value() { + break new_epoch; + } + self.epoch.store(Epoch::starting(), Ordering::Release); + }; + + // Reset the advance couter if epoch has advanced. + if new_epoch != self.prev_epoch.get() { + self.prev_epoch.set(new_epoch); + self.advance_count.set(0); + } + } + + guard + } + + /// Unpins the `Local`. + #[inline] + pub(crate) fn unpin(&self) { + let guard_count = self.guard_count.get(); + if guard_count == 1 && !self.collecting.get() { + self.collecting.set(true); + while self.must_collect.get() { + self.must_collect.set(false); + debug_assert!(self.epoch.load(Ordering::Relaxed).is_pinned()); + let guard = ManuallyDrop::new(Guard { local: self }); + self.global().collect(&guard); + self.repin_without_collect(); + } + self.collecting.set(false); + } + + self.guard_count.set(guard_count - 1); + if guard_count == 1 { + self.epoch.store(Epoch::starting(), Ordering::Release); + + if self.handle_count.get() == 0 { + self.finalize(); + } + } + } + + /// Unpins and then pins the `Local`. + #[inline] + pub(crate) fn repin(&self) { + self.acquire_handle(); + self.unpin(); + compiler_fence(Ordering::SeqCst); + forget(self.pin()); + self.release_handle(); + } + + /// Repins the local epoch without checking a scheduled collection. + #[inline] + pub(crate) fn repin_without_collect(&self) -> Epoch { + let epoch = self.epoch.load(Ordering::Relaxed); + let global_epoch = self.global().epoch.load(Ordering::Relaxed).pinned(); + + // Update the local epoch only if the global epoch is greater than the local epoch. + if epoch != global_epoch { + // We store the new epoch with `Release` because we need to ensure any memory + // accesses from the previous epoch do not leak into the new one. + self.epoch.store(global_epoch, Ordering::Release); + } + global_epoch + } + + /// Increments the handle count. + #[inline] + pub(crate) fn acquire_handle(&self) { + let handle_count = self.handle_count.get(); + debug_assert!(handle_count >= 1); + self.handle_count.set(handle_count + 1); + } + + /// Decrements the handle count. + #[inline] + pub(crate) fn release_handle(&self) { + let guard_count = self.guard_count.get(); + let handle_count = self.handle_count.get(); + debug_assert!(handle_count >= 1); + self.handle_count.set(handle_count - 1); + + if guard_count == 0 && handle_count == 1 { + self.finalize(); + } + } + + /// Removes the `Local` from the global linked list. + #[cold] + fn finalize(&self) { + debug_assert_eq!(self.guard_count.get(), 0); + debug_assert_eq!(self.handle_count.get(), 0); + + // Temporarily increment handle count. This is required so that the following call to `pin` + // doesn't call `finalize` again. + self.handle_count.set(1); + { + // Pin and move the local bag into the global queue. It's important that `push_bag` + // doesn't defer destruction on any new garbage. + let guard = &self.pin(); + self.push_to_global(guard); + } + // Revert the handle count back to zero. + self.handle_count.set(0); + + unsafe { + // Take the reference to the `Global` out of this `Local`. Since we're not protected + // by a guard at this time, it's crucial that the reference is read before marking the + // `Local` as deleted. + let collector: Collector = ptr::read(&**self.collector.get()); + + // Mark this node in the linked list as deleted. + self.entry.delete(&unprotected()); + + // Finally, drop the reference to the global. Note that this might be the last reference + // to the `Global`. If so, the global data will be destroyed and all deferred functions + // in its queue will be executed. + drop(collector); + } + } + + #[allow(dead_code)] + pub(crate) fn incr_manual_collection(&self, guard: &Guard) { + let manual_count = self.manual_count.get().wrapping_add(1); + self.manual_count.set(manual_count); + + if manual_count.is_multiple_of(unsafe { MANUAL_EVENTS_BETWEEN_COLLECT }) { + self.flush(guard); + } + } +} + +impl IsElement for Local { + fn entry_of(local: &Local) -> &Entry { + let entry_ptr = (local as *const Local as usize + offset_of!(Local, entry)) as *const Entry; + unsafe { &*entry_ptr } + } + + unsafe fn element_of(entry: &Entry) -> &Local { + // offset_of! macro uses unsafe, but it's unnecessary in this context. + #[allow(unused_unsafe)] + let local_ptr = (entry as *const Entry as usize - offset_of!(Local, entry)) as *const Local; + // SAFETY: caller guarantees entry is from a Local + unsafe { &*local_ptr } + } + + unsafe fn finalize(entry: &Entry, guard: &Guard) { + // SAFETY: element_of is safe per the trait contract + unsafe { + guard.defer_destroy(RawShared::from(Self::element_of(entry) as *const Local)); + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + + #[test] + fn check_defer() { + static FLAG: AtomicUsize = AtomicUsize::new(0); + fn set() { + FLAG.store(42, Ordering::Relaxed); + } + + let d = Deferred::new(set); + assert_eq!(FLAG.load(Ordering::Relaxed), 0); + d.call(); + assert_eq!(FLAG.load(Ordering::Relaxed), 42); + } + + #[test] + fn check_bag() { + static FLAG: AtomicUsize = AtomicUsize::new(0); + fn incr() { + FLAG.fetch_add(1, Ordering::Relaxed); + } + + let mut bag = Bag::new(); + assert!(bag.is_empty()); + + for _ in 0..unsafe { MAX_OBJECTS } { + assert!(unsafe { bag.try_push(Deferred::new(incr)).is_ok() }); + assert!(!bag.is_empty()); + assert_eq!(FLAG.load(Ordering::Relaxed), 0); + } + + let result = unsafe { bag.try_push(Deferred::new(incr)) }; + assert!(result.is_err()); + assert!(!bag.is_empty()); + assert_eq!(FLAG.load(Ordering::Relaxed), 0); + + drop(bag); + assert_eq!(FLAG.load(Ordering::Relaxed), unsafe { MAX_OBJECTS }); + } +} diff --git a/crates/common/src/ebr/mod.rs b/crates/common/src/ebr/mod.rs new file mode 100644 index 0000000000..cb841484df --- /dev/null +++ b/crates/common/src/ebr/mod.rs @@ -0,0 +1,30 @@ +//! Epoch-based memory reclamation (EBR). +//! +//! This module provides safe memory reclamation for lock-free data structures +//! using epoch-based reclamation. It is based on crossbeam-epoch. +//! +//! # Overview +//! +//! When an element gets removed from a concurrent collection, it is inserted into +//! a pile of garbage and marked with the current epoch. Every time a thread accesses +//! a collection, it checks the current epoch, attempts to increment it, and destructs +//! some garbage that became so old that no thread can be referencing it anymore. +//! +//! # Pinning +//! +//! Before accessing shared data, a participant must be pinned using [`cs`]. This +//! returns a [`Guard`] that keeps the thread in a critical section until dropped. + +mod collector; +mod default; +mod deferred; +mod epoch; +mod guard; +pub mod internal; +mod pointers; +mod sync; + +pub use default::*; +pub use epoch::*; +pub use guard::*; +pub use pointers::*; diff --git a/crates/common/src/ebr/pointers.rs b/crates/common/src/ebr/pointers.rs new file mode 100644 index 0000000000..bd47583a3c --- /dev/null +++ b/crates/common/src/ebr/pointers.rs @@ -0,0 +1,317 @@ +use core::hash::Hash; +use core::marker::PhantomData; +use core::mem::align_of; +use core::ptr::null_mut; +use core::sync::atomic::AtomicUsize; +use std::fmt::{Debug, Formatter, Pointer}; + +use atomic::{Atomic, Ordering}; + +use super::Guard; + +pub struct Tagged { + ptr: *mut T, +} + +impl Debug for Tagged { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Pointer::fmt(&self.as_raw(), f) + } +} + +impl Pointer for Tagged { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Pointer::fmt(&self.as_raw(), f) + } +} + +impl Default for Tagged { + fn default() -> Self { + Self { ptr: null_mut() } + } +} + +impl Clone for Tagged { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Tagged {} + +impl Hash for Tagged { + fn hash(&self, state: &mut H) { + self.ptr.hash(state) + } +} + +impl From<*const T> for Tagged { + fn from(value: *const T) -> Self { + Self { + ptr: value.cast_mut(), + } + } +} + +impl From<*mut T> for Tagged { + fn from(value: *mut T) -> Self { + Self { ptr: value } + } +} + +pub const HIGH_TAG_WIDTH: u32 = 4; + +impl Tagged { + const fn high_bits_pos() -> u32 { + usize::BITS - HIGH_TAG_WIDTH + } + + const fn high_bits() -> usize { + ((1 << HIGH_TAG_WIDTH) - 1) << Self::high_bits_pos() + } + + pub fn null() -> Self { + Self { ptr: null_mut() } + } + + pub fn is_null(&self) -> bool { + self.as_raw().is_null() + } + + pub fn tag(&self) -> usize { + let ptr = self.ptr as usize; + ptr & low_bits::() + } + + pub fn high_tag(&self) -> usize { + let ptr = self.ptr as usize; + (ptr & Self::high_bits()) >> Self::high_bits_pos() + } + + /// Converts the pointer to a raw pointer (without the tag). + pub fn as_raw(&self) -> *mut T { + let ptr = self.ptr as usize; + (ptr & !low_bits::() & !Self::high_bits()) as *mut T + } + + pub fn with_tag(&self, tag: usize) -> Self { + Self::from(with_tag(self.ptr, tag)) + } + + pub fn with_high_tag(&self, tag: usize) -> Self { + Self::from( + (self.ptr as usize & !Self::high_bits() + | ((tag & ((1 << HIGH_TAG_WIDTH) - 1)) << Self::high_bits_pos())) + as *mut T, + ) + } + + /// # Safety + /// + /// The pointer (without high and low tag bits) must be a valid location to dereference. + pub unsafe fn deref<'g>(&self) -> &'g T { + // SAFETY: caller guarantees ptr is valid + unsafe { &*self.as_raw() } + } + + /// # Safety + /// + /// The pointer (without high and low tag bits) must be a valid location to dereference. + pub unsafe fn deref_mut<'g>(&mut self) -> &'g mut T { + // SAFETY: caller guarantees ptr is valid and unique + unsafe { &mut *self.as_raw() } + } + + /// # Safety + /// + /// The pointer (without high and low tag bits) must be a valid location to dereference. + pub unsafe fn as_ref<'g>(&self) -> Option<&'g T> { + if self.is_null() { + None + } else { + // SAFETY: checked for null above + Some(unsafe { self.deref() }) + } + } + + /// Returns `true` if the two pointer values, including the tag values set by `with_tag`, + /// are identical. + pub fn ptr_eq(self, other: Self) -> bool { + // Instead of using a direct equality comparison (`==`), we use `ptr_eq`, which ignores + // the epoch tag in the high bits. This is because the epoch tags hold no significance + // for clients; they are only used internally by the EBR engine to track the last + // accessed epoch for the pointer. + self.with_high_tag(0).ptr == other.with_high_tag(0).ptr + } +} + +/// Returns a bitmask containing the unused least significant bits of an aligned pointer to `T`. +const fn low_bits() -> usize { + (1 << align_of::().trailing_zeros()) - 1 +} + +/// Returns the pointer with the given tag +fn with_tag(ptr: *mut T, tag: usize) -> *mut T { + ((ptr as usize & !low_bits::()) | (tag & low_bits::())) as *mut T +} + +pub(crate) struct RawAtomic { + inner: Atomic>, +} + +unsafe impl Send for RawAtomic {} +unsafe impl Sync for RawAtomic {} + +impl RawAtomic { + pub fn null() -> Self { + Self { + inner: Atomic::new(Tagged::null()), + } + } + + pub fn load<'g>(&self, order: Ordering, _: &'g Guard) -> RawShared<'g, T> { + RawShared::from(self.inner.load(order)) + } + + pub fn store(&self, val: RawShared<'_, T>, order: Ordering) { + self.inner.store(val.inner, order); + } + + pub fn compare_exchange<'g>( + &self, + current: RawShared<'g, T>, + new: RawShared<'g, T>, + success: Ordering, + failure: Ordering, + _: &'g Guard, + ) -> Result, RawShared<'g, T>> { + self.inner + .compare_exchange(current.inner, new.inner, success, failure) + .map(RawShared::from) + .map_err(RawShared::from) + } + + pub fn compare_exchange_weak<'g>( + &self, + current: RawShared<'g, T>, + new: RawShared<'g, T>, + success: Ordering, + failure: Ordering, + _: &'g Guard, + ) -> Result, RawShared<'g, T>> { + self.inner + .compare_exchange_weak(current.inner, new.inner, success, failure) + .map(RawShared::from) + .map_err(RawShared::from) + } + + pub fn fetch_or<'g>(&self, tag: usize, order: Ordering, _: &'g Guard) -> RawShared<'g, T> { + // HACK: The size and alignment of `Atomic>` will be same with `AtomicUsize`. + // The equality of the sizes is checked by `const_assert!`. + let inner = unsafe { &*(&self.inner as *const _ as *const AtomicUsize) }; + let prev = inner.fetch_or(low_bits::() & tag, order); + RawShared::from(prev as *const _) + } +} + +// A shared pointer type only for the internal EBR implementation. +pub(crate) struct RawShared<'g, T> { + inner: Tagged, + _marker: PhantomData<&'g T>, +} + +impl<'g, T> Clone for RawShared<'g, T> { + fn clone(&self) -> Self { + *self + } +} + +impl<'g, T> Copy for RawShared<'g, T> {} + +impl<'g, T> From<*const T> for RawShared<'g, T> { + fn from(value: *const T) -> Self { + Self { + inner: Tagged::from(value), + _marker: PhantomData, + } + } +} + +impl<'g, T> From<*mut T> for RawShared<'g, T> { + fn from(value: *mut T) -> Self { + Self { + inner: Tagged::from(value), + _marker: PhantomData, + } + } +} + +impl<'g, T> From> for RawShared<'g, T> { + fn from(inner: Tagged) -> Self { + Self { + inner, + _marker: PhantomData, + } + } +} + +impl<'g, T> RawShared<'g, T> { + pub fn null() -> Self { + Self { + inner: Tagged::null(), + _marker: PhantomData, + } + } + + pub fn from_owned(init: T) -> Self { + Self { + inner: Tagged::from(Box::into_raw(Box::new(init))), + _marker: PhantomData, + } + } + + pub unsafe fn drop(self) { + // SAFETY: caller guarantees pointer is valid and owned + unsafe { drop(Box::from_raw(self.inner.as_raw())) } + } + + pub unsafe fn deref(self) -> &'g T { + // SAFETY: caller guarantees pointer is valid + unsafe { self.inner.deref() } + } + + pub unsafe fn as_ref(self) -> Option<&'g T> { + // SAFETY: caller guarantees pointer is valid + unsafe { self.inner.as_ref() } + } + + pub fn tag(self) -> usize { + self.inner.tag() + } + + pub fn with_tag(self, tag: usize) -> Self { + Self { + inner: self.inner.with_tag(tag), + _marker: PhantomData, + } + } + + pub fn as_raw(self) -> *mut T { + self.inner.as_raw() + } + + #[cfg(test)] + pub fn is_null(self) -> bool { + self.inner.is_null() + } + + /// Returns `true` if the two pointer values, including the tag values set by `with_tag`, + /// are identical. + pub fn ptr_eq(&self, other: Self) -> bool { + // Instead of using a direct equality comparison (`==`), we use `ptr_eq`, which ignores + // the epoch tag in the high bits. This is because the epoch tags hold no significance + // for clients; they are only used internally by the EBR engine to track the last + // accessed epoch for the pointer. + self.inner.ptr_eq(other.inner) + } +} diff --git a/crates/common/src/ebr/sync/list.rs b/crates/common/src/ebr/sync/list.rs new file mode 100644 index 0000000000..c938befde7 --- /dev/null +++ b/crates/common/src/ebr/sync/list.rs @@ -0,0 +1,491 @@ +//! Lock-free intrusive linked list. +//! +//! Ideas from Michael. High Performance Dynamic Lock-Free Hash Tables and List-Based Sets. SPAA +//! 2002. + +use core::marker::PhantomData; +use std::sync::atomic::Ordering::{Acquire, Relaxed, Release}; + +use crate::ebr::{RawAtomic, RawShared}; + +use super::super::{Guard, unprotected}; + +/// An entry in a linked list. +/// +/// An Entry is accessed from multiple threads, so it would be beneficial to put it in a different +/// cache-line than thread-local data in terms of performance. +pub(crate) struct Entry { + /// The next entry in the linked list. + /// If the tag is 1, this entry is marked as deleted. + next: RawAtomic, +} + +/// Implementing this trait asserts that the type `T` can be used as an element in the intrusive +/// linked list defined in this module. `T` has to contain (or otherwise be linked to) an instance +/// of `Entry`. +/// +/// # Example +/// +/// ```ignore +/// struct A { +/// entry: Entry, +/// data: usize, +/// } +/// +/// impl IsElement for A { +/// fn entry_of(a: &A) -> &Entry { +/// let entry_ptr = ((a as usize) + offset_of!(A, entry)) as *const Entry; +/// unsafe { &*entry_ptr } +/// } +/// +/// unsafe fn element_of(entry: &Entry) -> &T { +/// let elem_ptr = ((entry as usize) - offset_of!(A, entry)) as *const T; +/// &*elem_ptr +/// } +/// +/// unsafe fn finalize(entry: &Entry, guard: &Guard) { +/// guard.defer_destroy(Shared::from(Self::element_of(entry) as *const _)); +/// } +/// } +/// ``` +/// +/// This trait is implemented on a type separate from `T` (although it can be just `T`), because +/// one type might be placeable into multiple lists, in which case it would require multiple +/// implementations of `IsElement`. In such cases, each struct implementing `IsElement` +/// represents a distinct `Entry` in `T`. +/// +/// For example, we can insert the following struct into two lists using `entry1` for one +/// and `entry2` for the other: +/// +/// ```ignore +/// struct B { +/// entry1: Entry, +/// entry2: Entry, +/// data: usize, +/// } +/// ``` +/// +pub(crate) trait IsElement { + /// Returns a reference to this element's `Entry`. + fn entry_of(_: &T) -> &Entry; + + /// Given a reference to an element's entry, returns that element. + /// + /// ```ignore + /// let elem = ListElement::new(); + /// assert_eq!(elem.entry_of(), + /// unsafe { ListElement::element_of(elem.entry_of()) } ); + /// ``` + /// + /// # Safety + /// + /// The caller has to guarantee that the `Entry` is called with was retrieved from an instance + /// of the element type (`T`). + unsafe fn element_of(_: &Entry) -> &T; + + /// The function that is called when an entry is unlinked from list. + /// + /// # Safety + /// + /// The caller has to guarantee that the `Entry` is called with was retrieved from an instance + /// of the element type (`T`). + unsafe fn finalize(_: &Entry, _: &Guard); +} + +/// A lock-free, intrusive linked list of type `T`. +pub(crate) struct List = T> { + /// The head of the linked list. + head: RawAtomic, + + /// The phantom data for using `T` and `C`. + _marker: PhantomData<(T, C)>, +} + +/// An iterator used for retrieving values from the list. +pub(crate) struct Iter<'g, T, C: IsElement> { + /// The guard that protects the iteration. + guard: &'g Guard, + + /// Pointer from the predecessor to the current entry. + pred: &'g RawAtomic, + + /// The current entry. + curr: RawShared<'g, Entry>, + + /// The list head, needed for restarting iteration. + head: &'g RawAtomic, + + /// Logically, we store a borrow of an instance of `T` and + /// use the type information from `C`. + _marker: PhantomData<(&'g T, C)>, +} + +/// An error that occurs during iteration over the list. +#[derive(PartialEq, Debug)] +pub(crate) enum IterError { + /// A concurrent thread modified the state of the list at the same place that this iterator + /// was inspecting. Subsequent iteration will restart from the beginning of the list. + Stalled, +} + +impl Default for Entry { + /// Returns the empty entry. + fn default() -> Self { + Self { + next: RawAtomic::null(), + } + } +} + +impl Entry { + /// Marks this entry as deleted, deferring the actual deallocation to a later iteration. + /// + /// # Safety + /// + /// The entry should be a member of a linked list, and it should not have been deleted. + /// It should be safe to call `C::finalize` on the entry after the `guard` is dropped, where `C` + /// is the associated helper for the linked list. + pub(crate) unsafe fn delete(&self, guard: &Guard) { + self.next.fetch_or(1, Release, guard); + } +} + +impl> List { + /// Returns a new, empty linked list. + pub(crate) fn new() -> Self { + Self { + head: RawAtomic::null(), + _marker: PhantomData, + } + } + + /// Inserts `entry` into the head of the list. + /// + /// # Safety + /// + /// You should guarantee that: + /// + /// - `container` is not null + /// - `container` is immovable, e.g. inside an `Owned` + /// - the same `Entry` is not inserted more than once + /// - the inserted object will be removed before the list is dropped + pub(crate) unsafe fn insert<'g>(&'g self, container: RawShared<'g, T>, guard: &'g Guard) { + // Insert right after head, i.e. at the beginning of the list. + let to = &self.head; + // Get the intrusively stored Entry of the new element to insert. + // SAFETY: container is guaranteed non-null by caller + let entry: &Entry = C::entry_of(unsafe { container.deref() }); + // Make a Shared ptr to that Entry. + let entry_ptr = RawShared::from(entry as *const _); + // Read the current successor of where we want to insert. + let mut next = to.load(Relaxed, guard); + + loop { + // Set the Entry of the to-be-inserted element to point to the previous successor of + // `to`. + entry.next.store(next, Relaxed); + match to.compare_exchange_weak(next, entry_ptr, Release, Relaxed, guard) { + Ok(_) => break, + // We lost the race or weak CAS failed spuriously. Update the successor and try + // again. + Err(curr) => next = curr, + } + } + } + + /// Returns an iterator over all objects. + /// + /// # Caveat + /// + /// Every object that is inserted at the moment this function is called and persists at least + /// until the end of iteration will be returned. Since this iterator traverses a lock-free + /// linked list that may be concurrently modified, some additional caveats apply: + /// + /// 1. If a new object is inserted during iteration, it may or may not be returned. + /// 2. If an object is deleted during iteration, it may or may not be returned. + /// 3. The iteration may be aborted when it lost in a race condition. In this case, the winning + /// thread will continue to iterate over the same list. + pub(crate) fn iter<'g>(&'g self, guard: &'g Guard) -> Iter<'g, T, C> { + Iter { + guard, + pred: &self.head, + curr: self.head.load(Acquire, guard), + head: &self.head, + _marker: PhantomData, + } + } +} + +impl> Drop for List { + fn drop(&mut self) { + unsafe { + let guard = unprotected(); + let mut curr = self.head.load(Relaxed, &guard); + while let Some(c) = curr.as_ref() { + let succ = c.next.load(Relaxed, &guard); + // Verify that all elements have been removed from the list. + assert_eq!(succ.tag(), 1); + + C::finalize(curr.deref(), &guard); + curr = succ; + } + } + } +} + +impl<'g, T: 'g, C: IsElement> Iterator for Iter<'g, T, C> { + type Item = Result<&'g T, IterError>; + + fn next(&mut self) -> Option { + while let Some(c) = unsafe { self.curr.as_ref() } { + let succ = c.next.load(Acquire, self.guard); + + if succ.tag() == 1 { + // This entry was removed. Try unlinking it from the list. + let succ = succ.with_tag(0); + + // The tag should always be zero, because removing a node after a logically deleted + // node leaves the list in an invalid state. + debug_assert!(self.curr.tag() == 0); + + // Try to unlink `curr` from the list, and get the new value of `self.pred`. + let succ = match self + .pred + .compare_exchange(self.curr, succ, Acquire, Acquire, self.guard) + { + Ok(_) => { + // We succeeded in unlinking `curr`, so we have to schedule + // deallocation. Deferred drop is okay, because `list.delete()` can only be + // called if `T: 'static`. + unsafe { + C::finalize(self.curr.deref(), self.guard); + } + + // `succ` is the new value of `self.pred`. + succ + } + Err(curr) => { + // `curr` is the current value of `self.pred`. + curr + } + }; + + // If the predecessor node is already marked as deleted, we need to restart from + // `head`. + if succ.tag() != 0 { + self.pred = self.head; + self.curr = self.head.load(Acquire, self.guard); + + return Some(Err(IterError::Stalled)); + } + + // Move over the removed by only advancing `curr`, not `pred`. + self.curr = succ; + continue; + } + + // Move one step forward. + self.pred = &c.next; + self.curr = succ; + + return Some(Ok(unsafe { C::element_of(c) })); + } + + // We reached the end of the list. + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ebr::collector::Collector; + use crossbeam_utils::thread; + use std::sync::Barrier; + + impl IsElement for Entry { + fn entry_of(entry: &Entry) -> &Entry { + entry + } + + unsafe fn element_of(entry: &Entry) -> &Entry { + entry + } + + unsafe fn finalize(entry: &Entry, guard: &Guard) { + // SAFETY: entry is valid and element_of returns a valid pointer + unsafe { + guard.defer_destroy(RawShared::from(Self::element_of(entry) as *const _)); + } + } + } + + /// Checks whether the list retains inserted elements + /// and returns them in the correct order. + #[test] + fn insert() { + let collector = Collector::new(); + let handle = collector.register(); + let guard = handle.pin(); + + let l: List = List::new(); + + let e1 = RawShared::from_owned(Entry::default()); + let e2 = RawShared::from_owned(Entry::default()); + let e3 = RawShared::from_owned(Entry::default()); + + unsafe { + l.insert(e1, &guard); + l.insert(e2, &guard); + l.insert(e3, &guard); + } + + let mut iter = l.iter(&guard); + let maybe_e3 = iter.next(); + assert!(maybe_e3.is_some()); + assert!(maybe_e3.unwrap().unwrap() as *const Entry == e3.as_raw()); + let maybe_e2 = iter.next(); + assert!(maybe_e2.is_some()); + assert!(maybe_e2.unwrap().unwrap() as *const Entry == e2.as_raw()); + let maybe_e1 = iter.next(); + assert!(maybe_e1.is_some()); + assert!(maybe_e1.unwrap().unwrap() as *const Entry == e1.as_raw()); + assert!(iter.next().is_none()); + + unsafe { + e1.as_ref().unwrap().delete(&guard); + e2.as_ref().unwrap().delete(&guard); + e3.as_ref().unwrap().delete(&guard); + } + } + + /// Checks whether elements can be removed from the list and whether + /// the correct elements are removed. + #[test] + fn delete() { + let collector = Collector::new(); + let handle = collector.register(); + let guard = handle.pin(); + + let l: List = List::new(); + + let e1 = RawShared::from_owned(Entry::default()); + let e2 = RawShared::from_owned(Entry::default()); + let e3 = RawShared::from_owned(Entry::default()); + unsafe { + l.insert(e1, &guard); + l.insert(e2, &guard); + l.insert(e3, &guard); + e2.as_ref().unwrap().delete(&guard); + } + + let mut iter = l.iter(&guard); + let maybe_e3 = iter.next(); + assert!(maybe_e3.is_some()); + assert!(maybe_e3.unwrap().unwrap() as *const Entry == e3.as_raw()); + let maybe_e1 = iter.next(); + assert!(maybe_e1.is_some()); + assert!(maybe_e1.unwrap().unwrap() as *const Entry == e1.as_raw()); + assert!(iter.next().is_none()); + + unsafe { + e1.as_ref().unwrap().delete(&guard); + e3.as_ref().unwrap().delete(&guard); + } + + let mut iter = l.iter(&guard); + assert!(iter.next().is_none()); + } + + const THREADS: usize = 8; + const ITERS: usize = 512; + + /// Contends the list on insert and delete operations to make sure they can run concurrently. + #[test] + fn insert_delete_multi() { + let collector = Collector::new(); + + let l: List = List::new(); + let b = Barrier::new(THREADS); + + thread::scope(|s| { + for _ in 0..THREADS { + s.spawn(|_| { + b.wait(); + + let handle = collector.register(); + let guard: Guard = handle.pin(); + let mut v = Vec::with_capacity(ITERS); + + for _ in 0..ITERS { + let e = RawShared::from_owned(Entry::default()); + v.push(e); + unsafe { + l.insert(e, &guard); + } + } + + for e in v { + unsafe { + e.as_ref().unwrap().delete(&guard); + } + } + }); + } + }) + .unwrap(); + + let handle = collector.register(); + let guard = handle.pin(); + + let mut iter = l.iter(&guard); + assert!(iter.next().is_none()); + } + + /// Contends the list on iteration to make sure that it can be iterated over concurrently. + #[test] + fn iter_multi() { + let collector = Collector::new(); + + let l: List = List::new(); + let b = Barrier::new(THREADS); + + thread::scope(|s| { + for _ in 0..THREADS { + s.spawn(|_| { + b.wait(); + + let handle = collector.register(); + let guard: Guard = handle.pin(); + let mut v = Vec::with_capacity(ITERS); + + for _ in 0..ITERS { + let e = RawShared::from_owned(Entry::default()); + v.push(e); + unsafe { + l.insert(e, &guard); + } + } + + let mut iter = l.iter(&guard); + for _ in 0..ITERS { + assert!(iter.next().is_some()); + } + + for e in v { + unsafe { + e.as_ref().unwrap().delete(&guard); + } + } + }); + } + }) + .unwrap(); + + let handle = collector.register(); + let guard = handle.pin(); + + let mut iter = l.iter(&guard); + assert!(iter.next().is_none()); + } +} diff --git a/crates/common/src/ebr/sync/mod.rs b/crates/common/src/ebr/sync/mod.rs new file mode 100644 index 0000000000..08578495a6 --- /dev/null +++ b/crates/common/src/ebr/sync/mod.rs @@ -0,0 +1,5 @@ +//! Synchronization primitives. + +pub(crate) mod list; +pub(crate) mod once_lock; +pub(crate) mod queue; diff --git a/crates/common/src/ebr/sync/once_lock.rs b/crates/common/src/ebr/sync/once_lock.rs new file mode 100644 index 0000000000..2e0c7e7529 --- /dev/null +++ b/crates/common/src/ebr/sync/once_lock.rs @@ -0,0 +1,104 @@ +// Based on unstable std::sync::OnceLock. +// +// Source: https://github.com/rust-lang/rust/blob/8e9c93df464b7ada3fc7a1c8ccddd9dcb24ee0a0/library/std/src/sync/once_lock.rs + +use core::cell::UnsafeCell; +use core::mem::MaybeUninit; +use core::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Once; + +pub(crate) struct OnceLock { + once: Once, + // Once::is_completed requires Rust 1.43, so use this to track of whether they have been initialized. + is_initialized: AtomicBool, + value: UnsafeCell>, + // Unlike std::sync::OnceLock, we don't need PhantomData here because + // we don't use #[may_dangle]. +} + +unsafe impl Sync for OnceLock {} +unsafe impl Send for OnceLock {} + +impl OnceLock { + /// Creates a new empty cell. + #[must_use] + pub(crate) const fn new() -> Self { + Self { + once: Once::new(), + is_initialized: AtomicBool::new(false), + value: UnsafeCell::new(MaybeUninit::uninit()), + } + } + + /// Gets the contents of the cell, initializing it with `f` if the cell + /// was empty. + /// + /// Many threads may call `get_or_init` concurrently with different + /// initializing functions, but it is guaranteed that only one function + /// will be executed. + /// + /// # Panics + /// + /// If `f` panics, the panic is propagated to the caller, and the cell + /// remains uninitialized. + /// + /// It is an error to reentrantly initialize the cell from `f`. The + /// exact outcome is unspecified. Current implementation deadlocks, but + /// this may be changed to a panic in the future. + pub(crate) fn get_or_init(&self, f: F) -> &T + where + F: FnOnce() -> T, + { + // Fast path check + if self.is_initialized() { + // SAFETY: The inner value has been initialized + return unsafe { self.get_unchecked() }; + } + self.initialize(f); + + debug_assert!(self.is_initialized()); + + // SAFETY: The inner value has been initialized + unsafe { self.get_unchecked() } + } + + #[inline] + fn is_initialized(&self) -> bool { + self.is_initialized.load(Ordering::Acquire) + } + + #[cold] + fn initialize(&self, f: F) + where + F: FnOnce() -> T, + { + let slot = self.value.get().cast::(); + let is_initialized = &self.is_initialized; + + self.once.call_once(|| { + let value = f(); + unsafe { + slot.write(value); + } + is_initialized.store(true, Ordering::Release); + }); + } + + /// # Safety + /// + /// The value must be initialized + unsafe fn get_unchecked(&self) -> &T { + debug_assert!(self.is_initialized()); + // SAFETY: value is initialized per caller contract + unsafe { &*self.value.get().cast::() } + } +} + +impl Drop for OnceLock { + fn drop(&mut self) { + if self.is_initialized() { + // SAFETY: The inner value has been initialized + unsafe { self.value.get().cast::().drop_in_place() }; + } + } +} diff --git a/crates/common/src/ebr/sync/queue.rs b/crates/common/src/ebr/sync/queue.rs new file mode 100644 index 0000000000..4b3d7269bb --- /dev/null +++ b/crates/common/src/ebr/sync/queue.rs @@ -0,0 +1,466 @@ +//! Michael-Scott lock-free queue. +//! +//! Usable with any number of producers and consumers. +//! +//! Michael and Scott. Simple, Fast, and Practical Non-Blocking and Blocking Concurrent Queue +//! Algorithms. PODC 1996. +//! +//! Simon Doherty, Lindsay Groves, Victor Luchangco, and Mark Moir. 2004b. Formal Verification of a +//! Practical Lock-Free Queue Algorithm. + +use core::mem::MaybeUninit; +use core::sync::atomic::Ordering::{Acquire, Relaxed, Release}; + +use crossbeam_utils::{Backoff, CachePadded}; + +use crate::ebr::{RawAtomic, RawShared}; + +use super::super::{Guard, unprotected}; + +// The representation here is a singly-linked list, with a sentinel node at the front. In general +// the `tail` pointer may lag behind the actual tail. Non-sentinel nodes are either all `Data` or +// all `Blocked` (requests for data from blocked threads). +pub(crate) struct Queue { + head: CachePadded>>, + tail: CachePadded>>, +} + +struct Node { + /// The slot in which a value of type `T` can be stored. + /// + /// The type of `data` is `MaybeUninit` because a `Node` doesn't always contain a `T`. + /// For example, the sentinel node in a queue never contains a value: its slot is always empty. + /// Other nodes start their life with a push operation and contain a value until it gets popped + /// out. After that such empty nodes get added to the collector for destruction. + data: MaybeUninit, + + next: RawAtomic>, +} + +// Any particular `T` should never be accessed concurrently, so no need for `Sync`. +unsafe impl Sync for Queue {} +unsafe impl Send for Queue {} + +impl Queue { + /// Create a new, empty queue. + pub(crate) fn new() -> Queue { + let q = Queue { + head: CachePadded::new(RawAtomic::null()), + tail: CachePadded::new(RawAtomic::null()), + }; + let sentinel = RawShared::from_owned(Node { + data: MaybeUninit::uninit(), + next: RawAtomic::null(), + }); + q.head.store(sentinel, Relaxed); + q.tail.store(sentinel, Relaxed); + q + } + + /// Attempts to atomically place `n` into the `next` pointer of `onto`, and returns `true` on + /// success. The queue's `tail` pointer may be updated. + #[inline(always)] + fn push_internal( + &self, + onto: RawShared<'_, Node>, + new: RawShared<'_, Node>, + guard: &Guard, + ) -> bool { + // is `onto` the actual tail? + let o = unsafe { onto.deref() }; + let next = o.next.load(Acquire, guard); + if unsafe { next.as_ref().is_some() } { + // if not, try to "help" by moving the tail pointer forward + let _ = self + .tail + .compare_exchange(onto, next, Release, Relaxed, guard); + false + } else { + // looks like the actual tail; attempt to link in `n` + let result = o + .next + .compare_exchange(RawShared::null(), new, Release, Relaxed, guard) + .is_ok(); + if result { + // try to move the tail pointer forward + let _ = self + .tail + .compare_exchange(onto, new, Release, Relaxed, guard); + } + result + } + } + + /// Adds `t` to the back of the queue, possibly waking up threads blocked on `pop`. + pub(crate) fn push(&self, t: T, guard: &Guard) { + let new = RawShared::from_owned(Node { + data: MaybeUninit::new(t), + next: RawAtomic::null(), + }); + + loop { + // We push onto the tail, so we'll start optimistically by looking there first. + let tail = self.tail.load(Acquire, guard); + + // Attempt to push onto the `tail` snapshot; fails if `tail.next` has changed. + if self.push_internal(tail, new, guard) { + break; + } + } + } + + /// Attempts to pop a data node. `Ok(None)` if queue is empty; `Err(())` if lost race to pop. + #[inline(always)] + fn pop_internal(&self, guard: &Guard) -> Result, ()> { + let head = self.head.load(Acquire, guard); + let h = unsafe { head.deref() }; + let next = h.next.load(Acquire, guard); + match unsafe { next.as_ref() } { + Some(n) => unsafe { + self.head + .compare_exchange(head, next, Release, Relaxed, guard) + .map(|_| { + let tail = self.tail.load(Relaxed, guard); + // Advance the tail so that we don't retire a pointer to a reachable node. + if head.ptr_eq(tail) { + let _ = self + .tail + .compare_exchange(tail, next, Release, Relaxed, guard); + } + guard.defer_destroy(head); + Some(n.data.assume_init_read()) + }) + .map_err(|_| ()) + }, + None => Ok(None), + } + } + + /// Attempts to pop a data node, if the data satisfies the given condition. `Ok(None)` if queue + /// is empty or the data does not satisfy the condition; `Err(())` if lost race to pop. + #[inline(always)] + fn pop_if_internal(&self, condition: F, guard: &Guard) -> Result, ()> + where + T: Sync, + F: Fn(&T) -> bool, + { + let head = self.head.load(Acquire, guard); + let h = unsafe { head.deref() }; + let next = h.next.load(Acquire, guard); + match unsafe { next.as_ref() } { + Some(n) if condition(unsafe { &*n.data.as_ptr() }) => unsafe { + self.head + .compare_exchange(head, next, Release, Relaxed, guard) + .map(|_| { + let tail = self.tail.load(Relaxed, guard); + // Advance the tail so that we don't retire a pointer to a reachable node. + if head.ptr_eq(tail) { + let _ = self + .tail + .compare_exchange(tail, next, Release, Relaxed, guard); + } + guard.defer_destroy(head); + Some(n.data.assume_init_read()) + }) + .map_err(|_| ()) + }, + None | Some(_) => Ok(None), + } + } + + /// Attempts to dequeue from the front. + /// + /// Returns `None` if the queue is observed to be empty. + pub(crate) fn try_pop(&self, guard: &Guard) -> Option { + loop { + if let Ok(head) = self.pop_internal(guard) { + return head; + } + } + } + + /// Attempts to dequeue from the front, if the item satisfies the given condition. + /// + /// Returns `None` if the queue is observed to be empty, or the head does not satisfy the given + /// condition. + pub(crate) fn try_pop_if(&self, condition: F, guard: &Guard) -> Option + where + T: Sync, + F: Fn(&T) -> bool, + { + let backoff = Backoff::new(); + loop { + if let Ok(head) = self.pop_if_internal(&condition, guard) { + return head; + } + backoff.spin(); + } + } +} + +impl Drop for Queue { + fn drop(&mut self) { + unsafe { + let guard = &unprotected(); + + while self.try_pop(guard).is_some() {} + + // Destroy the remaining sentinel node. + let sentinel = self.head.load(Relaxed, guard); + sentinel.drop(); + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::ebr::cs; + use crossbeam_utils::thread; + + struct Queue { + queue: super::Queue, + } + + impl Queue { + pub(crate) fn new() -> Queue { + Queue { + queue: super::Queue::new(), + } + } + + pub(crate) fn push(&self, t: T) { + let guard = &cs(); + self.queue.push(t, guard); + } + + pub(crate) fn is_empty(&self) -> bool { + let guard = &cs(); + let head = self.queue.head.load(Acquire, guard); + let h = unsafe { head.deref() }; + h.next.load(Acquire, guard).is_null() + } + + pub(crate) fn try_pop(&self) -> Option { + let guard = &cs(); + self.queue.try_pop(guard) + } + + pub(crate) fn pop(&self) -> T { + loop { + match self.try_pop() { + None => continue, + Some(t) => return t, + } + } + } + } + + #[cfg(miri)] + const CONC_COUNT: i64 = 1000; + #[cfg(not(miri))] + const CONC_COUNT: i64 = 1000000; + + #[test] + fn push_try_pop_1() { + let q: Queue = Queue::new(); + assert!(q.is_empty()); + q.push(37); + assert!(!q.is_empty()); + assert_eq!(q.try_pop(), Some(37)); + assert!(q.is_empty()); + } + + #[test] + fn push_try_pop_2() { + let q: Queue = Queue::new(); + assert!(q.is_empty()); + q.push(37); + q.push(48); + assert_eq!(q.try_pop(), Some(37)); + assert!(!q.is_empty()); + assert_eq!(q.try_pop(), Some(48)); + assert!(q.is_empty()); + } + + #[test] + fn push_try_pop_many_seq() { + let q: Queue = Queue::new(); + assert!(q.is_empty()); + for i in 0..200 { + q.push(i) + } + assert!(!q.is_empty()); + for i in 0..200 { + assert_eq!(q.try_pop(), Some(i)); + } + assert!(q.is_empty()); + } + + #[test] + fn push_pop_1() { + let q: Queue = Queue::new(); + assert!(q.is_empty()); + q.push(37); + assert!(!q.is_empty()); + assert_eq!(q.pop(), 37); + assert!(q.is_empty()); + } + + #[test] + fn push_pop_2() { + let q: Queue = Queue::new(); + q.push(37); + q.push(48); + assert_eq!(q.pop(), 37); + assert_eq!(q.pop(), 48); + } + + #[test] + fn push_pop_many_seq() { + let q: Queue = Queue::new(); + assert!(q.is_empty()); + for i in 0..200 { + q.push(i) + } + assert!(!q.is_empty()); + for i in 0..200 { + assert_eq!(q.pop(), i); + } + assert!(q.is_empty()); + } + + #[test] + fn push_try_pop_many_spsc() { + let q: Queue = Queue::new(); + assert!(q.is_empty()); + + thread::scope(|scope| { + scope.spawn(|_| { + let mut next = 0; + + while next < CONC_COUNT { + if let Some(elem) = q.try_pop() { + assert_eq!(elem, next); + next += 1; + } + } + }); + + for i in 0..CONC_COUNT { + q.push(i) + } + }) + .unwrap(); + } + + #[test] + fn push_try_pop_many_spmc() { + fn recv(_t: i32, q: &Queue) { + let mut cur = -1; + for _i in 0..CONC_COUNT { + if let Some(elem) = q.try_pop() { + assert!(elem > cur); + cur = elem; + + if cur == CONC_COUNT - 1 { + break; + } + } + } + } + + let q: Queue = Queue::new(); + assert!(q.is_empty()); + thread::scope(|scope| { + for i in 0..3 { + let q = &q; + scope.spawn(move |_| recv(i, q)); + } + + scope.spawn(|_| { + for i in 0..CONC_COUNT { + q.push(i); + } + }); + }) + .unwrap(); + } + + #[test] + fn push_try_pop_many_mpmc() { + enum LR { + Left(i64), + Right(i64), + } + + let q: Queue = Queue::new(); + assert!(q.is_empty()); + + thread::scope(|scope| { + for _t in 0..2 { + scope.spawn(|_| { + for i in CONC_COUNT - 1..CONC_COUNT { + q.push(LR::Left(i)) + } + }); + scope.spawn(|_| { + for i in CONC_COUNT - 1..CONC_COUNT { + q.push(LR::Right(i)) + } + }); + scope.spawn(|_| { + let mut vl = vec![]; + let mut vr = vec![]; + for _i in 0..CONC_COUNT { + match q.try_pop() { + Some(LR::Left(x)) => vl.push(x), + Some(LR::Right(x)) => vr.push(x), + _ => {} + } + } + + let mut vl2 = vl.clone(); + let mut vr2 = vr.clone(); + vl2.sort_unstable(); + vr2.sort_unstable(); + + assert_eq!(vl, vl2); + assert_eq!(vr, vr2); + }); + } + }) + .unwrap(); + } + + #[test] + fn push_pop_many_spsc() { + let q: Queue = Queue::new(); + + thread::scope(|scope| { + scope.spawn(|_| { + let mut next = 0; + while next < CONC_COUNT { + assert_eq!(q.pop(), next); + next += 1; + } + }); + + for i in 0..CONC_COUNT { + q.push(i) + } + }) + .unwrap(); + assert!(q.is_empty()); + } + + #[test] + fn is_empty_dont_pop() { + let q: Queue = Queue::new(); + q.push(20); + q.push(20); + assert!(!q.is_empty()); + assert!(!q.is_empty()); + assert!(q.try_pop().is_some()); + } +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 0181562d04..b6712dda44 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -14,6 +14,7 @@ pub mod boxvec; pub mod cformat; #[cfg(any(unix, windows, target_os = "wasi"))] pub mod crt_fd; +pub mod ebr; pub mod encodings; #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] pub mod fileutils; diff --git a/crates/common/src/refcount.rs b/crates/common/src/refcount.rs index a5fbfa8fc3..3c038d1374 100644 --- a/crates/common/src/refcount.rs +++ b/crates/common/src/refcount.rs @@ -1,14 +1,243 @@ -use crate::atomic::{Ordering::*, PyAtomic, Radium}; +//! Reference counting implementation based on EBR (Epoch-Based Reclamation). +//! +//! This module provides a RefCount type that is compatible with EBR's memory reclamation +//! system while maintaining the original API for backward compatibility. -/// from alloc::sync -/// A soft limit on the amount of references that may be made to an `Arc`. -/// -/// Going above this limit will abort your program (although not -/// necessarily) at _exactly_ `MAX_REFCOUNT + 1` references. -const MAX_REFCOUNT: usize = isize::MAX as usize; +use std::cell::{Cell, RefCell}; +use std::sync::atomic::{AtomicU64, Ordering}; + +// Re-export EBR types +pub use crate::ebr::{Guard, HIGH_TAG_WIDTH, cs, global_epoch}; + +// ============================================================================ +// Constants from circ::utils - now defined locally +// ============================================================================ + +pub const EPOCH_WIDTH: u32 = HIGH_TAG_WIDTH; +pub const EPOCH_MASK_HEIGHT: u32 = u64::BITS - EPOCH_WIDTH; +pub const EPOCH: u64 = ((1 << EPOCH_WIDTH) - 1) << EPOCH_MASK_HEIGHT; +pub const DESTRUCTED: u64 = 1 << (EPOCH_MASK_HEIGHT - 1); +pub const WEAKED: u64 = 1 << (EPOCH_MASK_HEIGHT - 2); +pub const TOTAL_COUNT_WIDTH: u32 = u64::BITS - EPOCH_WIDTH - 2; +pub const WEAK_WIDTH: u32 = TOTAL_COUNT_WIDTH / 2; +pub const STRONG_WIDTH: u32 = TOTAL_COUNT_WIDTH - WEAK_WIDTH; +pub const STRONG: u64 = (1 << STRONG_WIDTH) - 1; +pub const WEAK: u64 = ((1 << WEAK_WIDTH) - 1) << STRONG_WIDTH; +pub const COUNT: u64 = 1; +pub const WEAK_COUNT: u64 = 1 << STRONG_WIDTH; + +/// LEAKED bit for interned objects (never deallocated) +/// Position: just after WEAKED bit +pub const LEAKED: u64 = 1 << (EPOCH_MASK_HEIGHT - 3); + +/// State wraps reference count + flags in a single 64-bit word +#[derive(Clone, Copy)] +pub struct State { + inner: u64, +} + +impl State { + #[inline] + pub fn from_raw(inner: u64) -> Self { + Self { inner } + } + + #[inline] + pub fn as_raw(self) -> u64 { + self.inner + } + + #[inline] + pub fn strong(self) -> u32 { + ((self.inner & STRONG) / COUNT) as u32 + } + + #[inline] + pub fn weak(self) -> u32 { + ((self.inner & WEAK) / WEAK_COUNT) as u32 + } + + #[inline] + pub fn destructed(self) -> bool { + (self.inner & DESTRUCTED) != 0 + } + + #[inline] + pub fn weaked(self) -> bool { + (self.inner & WEAKED) != 0 + } + + #[inline] + pub fn leaked(self) -> bool { + (self.inner & LEAKED) != 0 + } + + #[inline] + pub fn epoch(self) -> u32 { + ((self.inner & EPOCH) >> EPOCH_MASK_HEIGHT) as u32 + } + + #[inline] + pub fn with_epoch(self, epoch: usize) -> Self { + Self::from_raw((self.inner & !EPOCH) | (((epoch as u64) << EPOCH_MASK_HEIGHT) & EPOCH)) + } + + #[inline] + pub fn add_strong(self, val: u32) -> Self { + Self::from_raw(self.inner + (val as u64) * COUNT) + } + + #[inline] + pub fn sub_strong(self, val: u32) -> Self { + debug_assert!(self.strong() >= val); + Self::from_raw(self.inner - (val as u64) * COUNT) + } + #[inline] + pub fn add_weak(self, val: u32) -> Self { + Self::from_raw(self.inner + (val as u64) * WEAK_COUNT) + } + + #[inline] + pub fn with_destructed(self, dest: bool) -> Self { + Self::from_raw((self.inner & !DESTRUCTED) | if dest { DESTRUCTED } else { 0 }) + } + + #[inline] + pub fn with_weaked(self, weaked: bool) -> Self { + Self::from_raw((self.inner & !WEAKED) | if weaked { WEAKED } else { 0 }) + } + + #[inline] + pub fn with_leaked(self, leaked: bool) -> Self { + Self::from_raw((self.inner & !LEAKED) | if leaked { LEAKED } else { 0 }) + } +} + +/// Modular arithmetic for epoch comparisons +pub struct Modular { + max: isize, +} + +impl Modular { + /// Creates a modular space where `max` is the maximum. + pub fn new(max: isize) -> Self { + Self { max } + } + + // Sends a number to a modular space. + pub fn trans(&self, val: isize) -> isize { + debug_assert!(val <= self.max); + (val - (self.max + 1)) % (1 << WIDTH) + } + + // Receives a number from a modular space. + pub fn inver(&self, val: isize) -> isize { + (val + (self.max + 1)) % (1 << WIDTH) + } + + pub fn max(&self, nums: &[isize]) -> isize { + self.inver(nums.iter().fold(isize::MIN, |acc, val| { + acc.max(self.trans(val % (1 << WIDTH))) + })) + } + + // Checks if `a` is less than or equal to `b` in the modular space. + pub fn le(&self, a: isize, b: isize) -> bool { + self.trans(a) <= self.trans(b) + } +} + +/// PyState extends State with LEAKED support for RustPython +#[derive(Clone, Copy)] +pub struct PyState { + inner: u64, +} + +impl PyState { + #[inline] + pub fn from_raw(inner: u64) -> Self { + Self { inner } + } + + #[inline] + pub fn as_raw(self) -> u64 { + self.inner + } + + #[inline] + pub fn strong(self) -> u32 { + ((self.inner & STRONG) / COUNT) as u32 + } + + #[inline] + pub fn weak(self) -> u32 { + ((self.inner & WEAK) / WEAK_COUNT) as u32 + } + + #[inline] + pub fn destructed(self) -> bool { + (self.inner & DESTRUCTED) != 0 + } + + #[inline] + pub fn weaked(self) -> bool { + (self.inner & WEAKED) != 0 + } + + #[inline] + pub fn leaked(self) -> bool { + (self.inner & LEAKED) != 0 + } + + #[inline] + pub fn epoch(self) -> u32 { + ((self.inner & EPOCH) >> EPOCH_MASK_HEIGHT) as u32 + } + + #[inline] + pub fn with_epoch(self, epoch: usize) -> Self { + Self::from_raw((self.inner & !EPOCH) | (((epoch as u64) << EPOCH_MASK_HEIGHT) & EPOCH)) + } + + #[inline] + pub fn add_strong(self, val: u32) -> Self { + Self::from_raw(self.inner + (val as u64) * COUNT) + } + + #[inline] + pub fn sub_strong(self, val: u32) -> Self { + debug_assert!(self.strong() >= val); + Self::from_raw(self.inner - (val as u64) * COUNT) + } + + #[inline] + pub fn add_weak(self, val: u32) -> Self { + Self::from_raw(self.inner + (val as u64) * WEAK_COUNT) + } + + #[inline] + pub fn with_destructed(self, dest: bool) -> Self { + Self::from_raw((self.inner & !DESTRUCTED) | if dest { DESTRUCTED } else { 0 }) + } + + #[inline] + pub fn with_weaked(self, weaked: bool) -> Self { + Self::from_raw((self.inner & !WEAKED) | if weaked { WEAKED } else { 0 }) + } + + #[inline] + pub fn with_leaked(self, leaked: bool) -> Self { + Self::from_raw((self.inner & !LEAKED) | if leaked { LEAKED } else { 0 }) + } +} + +/// Reference count using state layout with LEAKED support. +/// +/// State layout (64 bits): +/// [4 bits: epoch] [1 bit: destructed] [1 bit: weaked] [1 bit: leaked] [28 bits: weak_count] [29 bits: strong_count] pub struct RefCount { - strong: PyAtomic, + state: AtomicU64, } impl Default for RefCount { @@ -18,61 +247,222 @@ impl Default for RefCount { } impl RefCount { - const MASK: usize = MAX_REFCOUNT; - + /// Create a new RefCount with strong count = 1 pub fn new() -> Self { + // Initial state: strong=1, weak=1 (implicit weak for strong refs) Self { - strong: Radium::new(1), + state: AtomicU64::new(COUNT + WEAK_COUNT), } } + /// Get current strong count #[inline] pub fn get(&self) -> usize { - self.strong.load(SeqCst) + PyState::from_raw(self.state.load(Ordering::SeqCst)).strong() as usize } + /// Increment strong count #[inline] pub fn inc(&self) { - let old_size = self.strong.fetch_add(1, Relaxed); - - if old_size & Self::MASK == Self::MASK { + let val = PyState::from_raw(self.state.fetch_add(COUNT, Ordering::SeqCst)); + if val.destructed() { + // Already marked for destruction, but we're incrementing + // This shouldn't happen in normal usage std::process::abort(); } + if val.strong() == 0 { + // The previous fetch_add created a permission to run decrement again + self.state.fetch_add(COUNT, Ordering::SeqCst); + } } - /// Returns true if successful + /// Try to increment strong count. Returns true if successful. + /// Returns false if the object is already being destructed. #[inline] pub fn safe_inc(&self) -> bool { - self.strong - .fetch_update(AcqRel, Acquire, |prev| (prev != 0).then_some(prev + 1)) - .is_ok() + let mut old = PyState::from_raw(self.state.load(Ordering::SeqCst)); + loop { + if old.destructed() { + return false; + } + let new_state = old.add_strong(1); + match self.state.compare_exchange( + old.as_raw(), + new_state.as_raw(), + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => return true, + Err(curr) => old = PyState::from_raw(curr), + } + } } - /// Decrement the reference count. Returns true when the refcount drops to 0. + /// Decrement strong count. Returns true when count drops to 0. #[inline] pub fn dec(&self) -> bool { - if self.strong.fetch_sub(1, Release) != 1 { + let old = PyState::from_raw(self.state.fetch_sub(COUNT, Ordering::SeqCst)); + + // LEAKED objects never reach 0 + if old.leaked() { return false; } - PyAtomic::::fence(Acquire); - - true + old.strong() == 1 } -} - -impl RefCount { - // move these functions out and give separated type once type range is stabilized + /// Mark this object as leaked (interned). It will never be deallocated. pub fn leak(&self) { debug_assert!(!self.is_leaked()); - const BIT_MARKER: usize = (isize::MAX as usize) + 1; - debug_assert_eq!(BIT_MARKER.count_ones(), 1); - debug_assert_eq!(BIT_MARKER.leading_zeros(), 0); - self.strong.fetch_add(BIT_MARKER, Relaxed); + let mut old = PyState::from_raw(self.state.load(Ordering::SeqCst)); + loop { + let new_state = old.with_leaked(true); + match self.state.compare_exchange( + old.as_raw(), + new_state.as_raw(), + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => return, + Err(curr) => old = PyState::from_raw(curr), + } + } } + /// Check if this object is leaked (interned). pub fn is_leaked(&self) -> bool { - (self.strong.load(Acquire) as isize) < 0 + PyState::from_raw(self.state.load(Ordering::Acquire)).leaked() + } + + /// Get the raw state for advanced operations + #[inline] + pub fn state(&self) -> &AtomicU64 { + &self.state } + + /// Get PyState from current value + #[inline] + pub fn py_state(&self) -> PyState { + PyState::from_raw(self.state.load(Ordering::SeqCst)) + } + + /// Mark as destructed. Returns true if successful. + #[inline] + pub fn mark_destructed(&self) -> bool { + let mut old = PyState::from_raw(self.state.load(Ordering::SeqCst)); + loop { + if old.destructed() || old.leaked() { + return false; + } + if old.strong() > 0 { + return false; + } + let new_state = old.with_destructed(true); + match self.state.compare_exchange( + old.as_raw(), + new_state.as_raw(), + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => return true, + Err(curr) => old = PyState::from_raw(curr), + } + } + } +} + +// ============================================================================ +// Deferred Drop Infrastructure +// ============================================================================ +// +// This mechanism allows untrack_object() calls to be deferred until after +// the GC collection phase completes, preventing deadlocks that occur when +// pop_edges() triggers object destruction while holding the tracked_objects lock. + +thread_local! { + /// Flag indicating if we're inside a deferred drop context. + /// When true, drop operations should defer untrack calls. + static IN_DEFERRED_CONTEXT: Cell = const { Cell::new(false) }; + + /// Queue of deferred untrack operations. + /// No Send bound needed - this is thread-local and only accessed from the same thread. + static DEFERRED_QUEUE: RefCell>> = const { RefCell::new(Vec::new()) }; +} + +/// Execute a function within a deferred drop context. +/// Any calls to `try_defer_drop` within this context will be queued +/// and executed when the context exits. +#[inline] +pub fn with_deferred_drops(f: F) -> R +where + F: FnOnce() -> R, +{ + IN_DEFERRED_CONTEXT.with(|in_ctx| { + let was_in_context = in_ctx.get(); + in_ctx.set(true); + let result = f(); + in_ctx.set(was_in_context); + + // Only flush if we're the outermost context + if !was_in_context { + flush_deferred_drops(); + } + + result + }) +} + +/// Try to defer a drop-related operation. +/// If inside a deferred context, the operation is queued. +/// Otherwise, it executes immediately. +/// +/// Note: No `Send` bound - this is thread-local and runs on the same thread. +#[inline] +pub fn try_defer_drop(f: F) +where + F: FnOnce() + 'static, +{ + let should_defer = IN_DEFERRED_CONTEXT.with(|in_ctx| in_ctx.get()); + + if should_defer { + DEFERRED_QUEUE.with(|q| { + q.borrow_mut().push(Box::new(f)); + }); + } else { + f(); + } +} + +/// Flush all deferred drop operations. +/// This is automatically called when exiting a deferred context. +#[inline] +pub fn flush_deferred_drops() { + DEFERRED_QUEUE.with(|q| { + // Take all queued operations + let ops: Vec<_> = q.borrow_mut().drain(..).collect(); + // Execute them outside the borrow + for op in ops { + op(); + } + }); +} + +/// Defer a closure execution using EBR until all pinned threads unpin. +/// +/// This function queues a closure to be executed only after all currently +/// pinned threads (those in EBR critical sections) have exited their +/// critical sections. This is the 3-epoch guarantee of EBR. +/// +/// # Safety +/// +/// - The closure must not hold references to the stack +/// - The closure must be `Send` (may execute on a different thread) +/// - Should only be called within an EBR critical section (with a valid Guard) +#[inline] +pub unsafe fn defer_destruction(guard: &Guard, f: F) +where + F: FnOnce() + Send + 'static, +{ + // SAFETY: Caller guarantees the closure is safe to defer + unsafe { guard.defer_unchecked(f) }; } diff --git a/crates/derive-impl/src/pyclass.rs b/crates/derive-impl/src/pyclass.rs index 06bbc06cfb..213493436f 100644 --- a/crates/derive-impl/src/pyclass.rs +++ b/crates/derive-impl/src/pyclass.rs @@ -574,51 +574,72 @@ pub(crate) fn impl_pyclass(attr: PunctuatedNestedMeta, item: Item) -> Result) { + #try_pop_edges_body } - assert_eq!(s, "manual"); - quote! {} - } else { - quote! {#[derive(Traverse)]} - }; - (maybe_trace_code, derive_trace) - } else { - ( - // a dummy impl, which do nothing - // #attrs - quote! { - impl ::rustpython_vm::object::MaybeTraverse for #ident { - fn try_traverse(&self, tracer_fn: &mut ::rustpython_vm::object::TraverseFn) { - // do nothing - } - } - }, - quote! {}, - ) + } } }; @@ -678,7 +699,7 @@ pub(crate) fn impl_pyclass(attr: PunctuatedNestedMeta, item: Item) -> Result) { self.0.try_traverse(traverse_fn) } + + fn try_pop_edges(&mut self, _out: &mut ::std::vec::Vec<::rustpython_vm::PyObjectRef>) { + // Struct sequences don't need pop_edges + } } // PySubclass for proper inheritance diff --git a/crates/derive-impl/src/util.rs b/crates/derive-impl/src/util.rs index 6be1fcdf7a..ccfce8c461 100644 --- a/crates/derive-impl/src/util.rs +++ b/crates/derive-impl/src/util.rs @@ -372,6 +372,7 @@ impl ItemMeta for ClassItemMeta { "ctx", "impl", "traverse", + "pop_edges", ]; fn from_inner(inner: ItemMetaInner) -> Self { diff --git a/crates/stdlib/src/fcntl.rs b/crates/stdlib/src/fcntl.rs index 822faeeeda..477c1f5421 100644 --- a/crates/stdlib/src/fcntl.rs +++ b/crates/stdlib/src/fcntl.rs @@ -92,11 +92,15 @@ mod fcntl { #[pyfunction] fn ioctl( io::Fildes(fd): io::Fildes, - request: u32, + request: i64, arg: OptionalArg, i32>>, mutate_flag: OptionalArg, vm: &VirtualMachine, ) -> PyResult { + // Convert to unsigned - handles both positive u32 values and negative i32 values + // that represent the same bit pattern (e.g., TIOCSWINSZ on some platforms). + // First truncate to u32 (takes lower 32 bits), then zero-extend to c_ulong. + let request = (request as u32) as libc::c_ulong; let arg = arg.unwrap_or_else(|| Either::B(0)); match arg { Either::A(buf_kind) => { diff --git a/crates/stdlib/src/gc.rs b/crates/stdlib/src/gc.rs index 5fc96a302f..648206f53d 100644 --- a/crates/stdlib/src/gc.rs +++ b/crates/stdlib/src/gc.rs @@ -2,75 +2,268 @@ pub(crate) use gc::make_module; #[pymodule] mod gc { - use crate::vm::{PyResult, VirtualMachine, function::FuncArgs}; + use crate::vm::{ + PyObjectRef, PyResult, VirtualMachine, + builtins::PyListRef, + function::{FuncArgs, OptionalArg}, + gc_state, + }; + // Debug flag constants + #[pyattr] + const DEBUG_STATS: u32 = gc_state::DEBUG_STATS; + #[pyattr] + const DEBUG_COLLECTABLE: u32 = gc_state::DEBUG_COLLECTABLE; + #[pyattr] + const DEBUG_UNCOLLECTABLE: u32 = gc_state::DEBUG_UNCOLLECTABLE; + #[pyattr] + const DEBUG_SAVEALL: u32 = gc_state::DEBUG_SAVEALL; + #[pyattr] + const DEBUG_LEAK: u32 = gc_state::DEBUG_LEAK; + + /// Enable automatic garbage collection. + #[pyfunction] + fn enable() { + gc_state::gc_state().enable(); + } + + /// Disable automatic garbage collection. + #[pyfunction] + fn disable() { + gc_state::gc_state().disable(); + } + + /// Return true if automatic gc is enabled. + #[pyfunction] + fn isenabled() -> bool { + gc_state::gc_state().is_enabled() + } + + /// Run a garbage collection. Returns the number of unreachable objects found. + #[derive(FromArgs)] + struct CollectArgs { + #[pyarg(any, optional)] + generation: OptionalArg, + } + + #[pyfunction] + fn collect(args: CollectArgs, vm: &VirtualMachine) -> PyResult { + let generation = args.generation; + let generation_num = generation.unwrap_or(2); + if generation_num < 0 || generation_num > 2 { + return Err(vm.new_value_error("invalid generation".to_owned())); + } + + // Invoke callbacks with "start" phase + invoke_callbacks(vm, "start", generation_num as usize, 0, 0); + + // Manual gc.collect() should run even if GC is disabled + let gc = gc_state::gc_state(); + let (collected, uncollectable) = gc.collect_force(generation_num as usize); + + // Move objects from gc_state.garbage to vm.ctx.gc_garbage (for DEBUG_SAVEALL) + { + let mut state_garbage = gc.garbage.lock(); + if !state_garbage.is_empty() { + let py_garbage = &vm.ctx.gc_garbage; + let mut garbage_vec = py_garbage.borrow_vec_mut(); + for obj in state_garbage.drain(..) { + garbage_vec.push(obj); + } + } + } + + // Invoke callbacks with "stop" phase + invoke_callbacks( + vm, + "stop", + generation_num as usize, + collected, + uncollectable, + ); + + Ok(collected as i32) + } + + /// Return the current collection thresholds as a tuple. #[pyfunction] - fn collect(_args: FuncArgs, _vm: &VirtualMachine) -> i32 { - 0 + fn get_threshold(vm: &VirtualMachine) -> PyObjectRef { + let (t0, t1, t2) = gc_state::gc_state().get_threshold(); + vm.ctx + .new_tuple(vec![ + vm.ctx.new_int(t0).into(), + vm.ctx.new_int(t1).into(), + vm.ctx.new_int(t2).into(), + ]) + .into() } + /// Set the collection thresholds. #[pyfunction] - fn isenabled(_args: FuncArgs, _vm: &VirtualMachine) -> bool { - false + fn set_threshold(threshold0: u32, threshold1: OptionalArg, threshold2: OptionalArg) { + gc_state::gc_state().set_threshold( + threshold0, + threshold1.into_option(), + threshold2.into_option(), + ); } + /// Return the current collection counts as a tuple. #[pyfunction] - fn enable(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) + fn get_count(vm: &VirtualMachine) -> PyObjectRef { + let (c0, c1, c2) = gc_state::gc_state().get_count(); + vm.ctx + .new_tuple(vec![ + vm.ctx.new_int(c0).into(), + vm.ctx.new_int(c1).into(), + vm.ctx.new_int(c2).into(), + ]) + .into() } + /// Return the current debugging flags. #[pyfunction] - fn disable(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) + fn get_debug() -> u32 { + gc_state::gc_state().get_debug() } + /// Set the debugging flags. #[pyfunction] - fn get_count(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) + fn set_debug(flags: u32) { + gc_state::gc_state().set_debug(flags); } + /// Return a list of per-generation gc stats. #[pyfunction] - fn get_debug(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) + fn get_stats(vm: &VirtualMachine) -> PyResult { + let stats = gc_state::gc_state().get_stats(); + let mut result = Vec::with_capacity(3); + + for stat in stats.iter() { + let dict = vm.ctx.new_dict(); + dict.set_item("collections", vm.ctx.new_int(stat.collections).into(), vm)?; + dict.set_item("collected", vm.ctx.new_int(stat.collected).into(), vm)?; + dict.set_item( + "uncollectable", + vm.ctx.new_int(stat.uncollectable).into(), + vm, + )?; + result.push(dict.into()); + } + + Ok(vm.ctx.new_list(result)) + } + + /// Return the list of objects tracked by the collector. + #[derive(FromArgs)] + struct GetObjectsArgs { + #[pyarg(any, optional)] + generation: OptionalArg>, } #[pyfunction] - fn get_objects(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) + fn get_objects(args: GetObjectsArgs, vm: &VirtualMachine) -> PyResult { + let generation_opt = args.generation.flatten(); + if let Some(g) = generation_opt { + if g < 0 || g > 2 { + return Err( + vm.new_value_error(format!("generation must be in range(0, 3), not {}", g)) + ); + } + } + let objects = gc_state::gc_state().get_objects(generation_opt); + Ok(vm.ctx.new_list(objects)) } + /// Return the list of objects directly referred to by any of the arguments. #[pyfunction] - fn get_referents(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) + fn get_referents(args: FuncArgs, vm: &VirtualMachine) -> PyListRef { + let mut result = Vec::new(); + + for obj in args.args { + // Use the gc_get_referents method to get references + result.extend(obj.gc_get_referents()); + } + + vm.ctx.new_list(result) } + /// Return the list of objects that directly refer to any of the arguments. #[pyfunction] - fn get_referrers(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) + fn get_referrers(args: FuncArgs, vm: &VirtualMachine) -> PyListRef { + // This is expensive: we need to scan all tracked objects + // For now, return an empty list (would need full object tracking to implement) + let _ = args; + vm.ctx.new_list(vec![]) } + /// Return True if the object is tracked by the garbage collector. #[pyfunction] - fn get_stats(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) + fn is_tracked(obj: PyObjectRef) -> bool { + // An object is tracked if it has IS_TRACE = true (has a trace function) + obj.is_gc_tracked() } + /// Return True if the object has been finalized by the garbage collector. #[pyfunction] - fn get_threshold(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) + fn is_finalized(obj: PyObjectRef) -> bool { + use core::ptr::NonNull; + let ptr = NonNull::from(obj.as_ref()); + gc_state::gc_state().is_finalized(ptr) } + /// Freeze all objects tracked by gc. #[pyfunction] - fn is_tracked(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) + fn freeze() { + gc_state::gc_state().freeze(); } + /// Unfreeze all objects in the permanent generation. #[pyfunction] - fn set_debug(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) + fn unfreeze() { + gc_state::gc_state().unfreeze(); } + /// Return the number of objects in the permanent generation. #[pyfunction] - fn set_threshold(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("")) + fn get_freeze_count() -> usize { + gc_state::gc_state().get_freeze_count() + } + + /// gc.garbage - list of uncollectable objects + #[pyattr] + fn garbage(vm: &VirtualMachine) -> PyListRef { + vm.ctx.gc_garbage.clone() + } + + /// gc.callbacks - list of callbacks to be invoked + #[pyattr] + fn callbacks(vm: &VirtualMachine) -> PyListRef { + vm.ctx.gc_callbacks.clone() + } + + /// Helper function to invoke GC callbacks + fn invoke_callbacks( + vm: &VirtualMachine, + phase: &str, + generation: usize, + collected: usize, + uncollectable: usize, + ) { + let callbacks_list = &vm.ctx.gc_callbacks; + let callbacks: Vec = callbacks_list.borrow_vec().to_vec(); + if callbacks.is_empty() { + return; + } + + let phase_str: PyObjectRef = vm.ctx.new_str(phase).into(); + let info = vm.ctx.new_dict(); + let _ = info.set_item("generation", vm.ctx.new_int(generation).into(), vm); + let _ = info.set_item("collected", vm.ctx.new_int(collected).into(), vm); + let _ = info.set_item("uncollectable", vm.ctx.new_int(uncollectable).into(), vm); + + for callback in callbacks { + let _ = callback.call((phase_str.clone(), info.clone()), vm); + } } } diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index b90176a62f..c22dd303c5 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -2293,7 +2293,7 @@ mod _ssl { // SSLSocket - represents a TLS-wrapped socket #[pyattr] - #[pyclass(name = "_SSLSocket", module = "ssl")] + #[pyclass(name = "_SSLSocket", module = "ssl", traverse)] #[derive(Debug, PyPayload)] pub(crate) struct PySSLSocket { // Underlying socket @@ -2301,14 +2301,19 @@ mod _ssl { // SSL context context: PyRwLock>, // Server-side or client-side + #[pytraverse(skip)] server_side: bool, // Server hostname for SNI + #[pytraverse(skip)] server_hostname: PyRwLock>, // TLS connection state + #[pytraverse(skip)] connection: PyMutex>, // Handshake completed flag + #[pytraverse(skip)] handshake_done: PyMutex, // Session was reused (for session resumption tracking) + #[pytraverse(skip)] session_was_reused: PyMutex, // Owner (SSLSocket instance that owns this _SSLSocket) owner: PyRwLock>, @@ -2316,22 +2321,27 @@ mod _ssl { session: PyRwLock>, // Verified certificate chain (built during verification) #[allow(dead_code)] + #[pytraverse(skip)] verified_chain: PyRwLock>>>, // MemoryBIO mode (optional) incoming_bio: Option>, outgoing_bio: Option>, // SNI certificate resolver state (for server-side only) + #[pytraverse(skip)] sni_state: PyRwLock>>>, // Pending context change (for SNI callback deferred handling) pending_context: PyRwLock>>, // Buffer to store ClientHello for connection recreation + #[pytraverse(skip)] client_hello_buffer: PyMutex>>, // Shutdown state for tracking close-notify exchange + #[pytraverse(skip)] shutdown_state: PyMutex, // Deferred client certificate verification error (for TLS 1.3) // Stores error message if client cert verification failed during handshake // Error is raised on first I/O operation after handshake // Using Arc to share with the certificate verifier + #[pytraverse(skip)] deferred_cert_error: Arc>>, } diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index 41f6779c21..6480a1d283 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -2,6 +2,7 @@ use super::{ IterStatus, PositionIterInternal, PyBaseExceptionRef, PyGenericAlias, PyMappingProxy, PySet, PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, set::PySetInner, }; +use crate::object::{Traverse, TraverseFn}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, TryFromObject, atomic_func, @@ -29,13 +30,34 @@ use std::sync::LazyLock; pub type DictContentType = dict_inner::Dict; -#[pyclass(module = false, name = "dict", unhashable = true, traverse)] +#[pyclass( + module = false, + name = "dict", + unhashable = true, + traverse = "manual", + pop_edges +)] #[derive(Default)] pub struct PyDict { entries: DictContentType, } pub type PyDictRef = PyRef; +// SAFETY: Traverse properly visits all owned PyObjectRefs +unsafe impl Traverse for PyDict { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + self.entries.traverse(traverse_fn); + } + + fn pop_edges(&mut self, out: &mut Vec) { + // Pop all entries and collect both keys and values + for (key, value) in self.entries.pop_all_entries() { + out.push(key); + out.push(value); + } + } +} + impl fmt::Debug for PyDict { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // TODO: implement more detailed, non-recursive Debug formatter diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index 3e5048e133..80ed5802ff 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -25,7 +25,7 @@ use itertools::Itertools; #[cfg(feature = "jit")] use rustpython_jit::CompiledCode; -#[pyclass(module = false, name = "function", traverse = "manual")] +#[pyclass(module = false, name = "function", traverse = "manual", pop_edges)] #[derive(Debug)] pub struct PyFunction { code: PyMutex>, @@ -50,6 +50,49 @@ unsafe impl Traverse for PyFunction { closure.as_untyped().traverse(tracer_fn); } self.defaults_and_kwdefaults.traverse(tracer_fn); + // Traverse additional fields that may contain references + self.type_params.lock().traverse(tracer_fn); + self.annotations.lock().traverse(tracer_fn); + self.module.lock().traverse(tracer_fn); + self.doc.lock().traverse(tracer_fn); + } + + fn pop_edges(&mut self, out: &mut Vec) { + // Pop closure if present (equivalent to Py_CLEAR(func_closure)) + if let Some(closure) = self.closure.take() { + out.push(closure.into()); + } + + // Pop defaults and kwdefaults + if let Some(mut guard) = self.defaults_and_kwdefaults.try_lock() { + if let Some(defaults) = guard.0.take() { + out.push(defaults.into()); + } + if let Some(kwdefaults) = guard.1.take() { + out.push(kwdefaults.into()); + } + } + + // Note: We do NOT clear annotations here. + // Unlike CPython which can set func_annotations to NULL, RustPython always + // has a dict reference. Clearing the dict in-place would affect all functions + // that share the same annotations dict (e.g., via functools.update_wrapper). + // The annotations dict typically doesn't create cycles, so skipping it is safe. + + // Replace name and qualname with empty string to break potential str subclass cycles + // This matches CPython's func_clear behavior: "name and qualname could be str + // subclasses, so they could have reference cycles" + if let Some(mut guard) = self.name.try_lock() { + let old_name = std::mem::replace(&mut *guard, Context::genesis().empty_str.to_owned()); + out.push(old_name.into()); + } + if let Some(mut guard) = self.qualname.try_lock() { + let old_qualname = std::mem::replace(&mut *guard, Context::genesis().empty_str.to_owned()); + out.push(old_qualname.into()); + } + + // Note: globals, builtins, code are NOT cleared + // as per CPython's func_clear behavior (they're required to be non-NULL) } } diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index 514b38b6c2..1ba1190562 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -3,6 +3,7 @@ use crate::atomic_func; use crate::common::lock::{ PyMappedRwLockReadGuard, PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, }; +use crate::object::{Traverse, TraverseFn}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, class::PyClassImpl, @@ -23,7 +24,13 @@ use crate::{ use alloc::fmt; use core::ops::DerefMut; -#[pyclass(module = false, name = "list", unhashable = true, traverse)] +#[pyclass( + module = false, + name = "list", + unhashable = true, + traverse = "manual", + pop_edges +)] #[derive(Default)] pub struct PyList { elements: PyRwLock>, @@ -50,6 +57,22 @@ impl FromIterator for PyList { } } +// SAFETY: Traverse properly visits all owned PyObjectRefs +unsafe impl Traverse for PyList { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + self.elements.traverse(traverse_fn); + } + + fn pop_edges(&mut self, out: &mut Vec) { + // During GC, we use interior mutability to access elements. + // This is safe because during GC collection, the object is unreachable + // and no other code should be accessing it. + if let Some(mut guard) = self.elements.try_write() { + out.extend(guard.drain(..)); + } + } +} + impl PyPayload for PyList { #[inline] fn class(ctx: &Context) -> &'static Py { diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index e101ef2a52..a5d1e6e820 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -1926,9 +1926,16 @@ impl fmt::Display for PyUtf8Str { } impl MaybeTraverse for PyUtf8Str { + const IS_TRACE: bool = true; + const HAS_POP_EDGES: bool = false; + fn try_traverse(&self, traverse_fn: &mut TraverseFn<'_>) { self.0.try_traverse(traverse_fn); } + + fn try_pop_edges(&mut self, _out: &mut Vec) { + // No pop_edges needed for PyUtf8Str + } } impl PyPayload for PyUtf8Str { diff --git a/crates/vm/src/builtins/super.rs b/crates/vm/src/builtins/super.rs index f0d873abfb..893509bc6d 100644 --- a/crates/vm/src/builtins/super.rs +++ b/crates/vm/src/builtins/super.rs @@ -61,7 +61,7 @@ impl Constructor for PySuper { #[derive(FromArgs)] pub struct InitArgs { #[pyarg(positional, optional)] - py_type: OptionalArg, + py_type: OptionalArg, #[pyarg(positional, optional)] py_obj: OptionalArg, } @@ -75,7 +75,10 @@ impl Initializer for PySuper { vm: &VirtualMachine, ) -> PyResult<()> { // Get the type: - let (typ, obj) = if let OptionalArg::Present(ty) = py_type { + let (typ, obj) = if let OptionalArg::Present(ty_obj) = py_type { + let ty = ty_obj + .downcast::() + .map_err(|_| vm.new_type_error("super() argument 1 must be a type"))?; (ty, py_obj.unwrap_or_none(vm)) } else { let frame = vm diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index f3da8b2616..f2e073560c 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -3,6 +3,7 @@ use crate::common::{ hash::{PyHash, PyUHash}, lock::PyMutex, }; +use crate::object::{Traverse, TraverseFn}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, atomic_func, @@ -24,7 +25,7 @@ use crate::{ use alloc::fmt; use std::sync::LazyLock; -#[pyclass(module = false, name = "tuple", traverse)] +#[pyclass(module = false, name = "tuple", traverse = "manual", pop_edges)] pub struct PyTuple { elements: Box<[R]>, } @@ -36,6 +37,20 @@ impl fmt::Debug for PyTuple { } } +// SAFETY: Traverse properly visits all owned PyObjectRefs +// Note: Only impl for PyTuple (the default) +unsafe impl Traverse for PyTuple { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + self.elements.traverse(traverse_fn); + } + + fn pop_edges(&mut self, out: &mut Vec) { + // Take ownership of elements and extend out + let elements = std::mem::take(&mut self.elements); + out.extend(elements.into_vec()); + } +} + impl PyPayload for PyTuple { #[inline] fn class(ctx: &Context) -> &'static Py { diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index d57f8be0fe..6507383683 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -708,6 +708,17 @@ impl Dict { + inner.indices.len() * size_of::() + inner.entries.len() * size_of::>() } + + /// Pop all entries from the dict, returning (key, value) pairs. + /// This is used for circular reference resolution in GC. + /// Requires &mut self to avoid lock contention. + pub fn pop_all_entries(&mut self) -> impl Iterator + '_ { + let inner = self.inner.get_mut(); + inner.used = 0; + inner.filled = 0; + inner.indices.iter_mut().for_each(|i| *i = IndexEntry::FREE); + inner.entries.drain(..).flatten().map(|e| (e.key, e.value)) + } } type LookupResult = (IndexEntry, IndexIndex); diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs new file mode 100644 index 0000000000..d5eb96647b --- /dev/null +++ b/crates/vm/src/gc_state.rs @@ -0,0 +1,782 @@ +//! Garbage Collection State and Algorithm +//! +//! This module implements CPython-compatible generational garbage collection +//! for RustPython, using an intrusive doubly-linked list approach. + +use crate::common::lock::PyMutex; +use crate::{PyObject, PyObjectRef}; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use std::collections::HashSet; +use std::sync::{Mutex, RwLock}; + +/// GC debug flags +pub const DEBUG_STATS: u32 = 1; +pub const DEBUG_COLLECTABLE: u32 = 2; +pub const DEBUG_UNCOLLECTABLE: u32 = 4; +pub const DEBUG_SAVEALL: u32 = 8; +pub const DEBUG_LEAK: u32 = DEBUG_COLLECTABLE | DEBUG_UNCOLLECTABLE | DEBUG_SAVEALL; + +/// Default thresholds for each generation +const DEFAULT_THRESHOLD_0: u32 = 700; +const DEFAULT_THRESHOLD_1: u32 = 10; +const DEFAULT_THRESHOLD_2: u32 = 10; + +/// Statistics for a single generation +#[derive(Debug, Default)] +pub struct GcStats { + pub collections: u64, + pub collected: u64, + pub uncollectable: u64, +} + +/// A single GC generation with intrusive linked list +pub struct GcGeneration { + /// Number of objects in this generation + count: AtomicUsize, + /// Threshold for triggering collection + threshold: AtomicU32, + /// Collection statistics + stats: PyMutex, +} + +impl GcGeneration { + pub const fn new(threshold: u32) -> Self { + Self { + count: AtomicUsize::new(0), + threshold: AtomicU32::new(threshold), + stats: PyMutex::new(GcStats { + collections: 0, + collected: 0, + uncollectable: 0, + }), + } + } + + pub fn count(&self) -> usize { + self.count.load(Ordering::SeqCst) + } + + pub fn threshold(&self) -> u32 { + self.threshold.load(Ordering::SeqCst) + } + + pub fn set_threshold(&self, value: u32) { + self.threshold.store(value, Ordering::SeqCst); + } + + pub fn stats(&self) -> GcStats { + let guard = self.stats.lock(); + GcStats { + collections: guard.collections, + collected: guard.collected, + uncollectable: guard.uncollectable, + } + } + + pub fn update_stats(&self, collected: u64, uncollectable: u64) { + let mut guard = self.stats.lock(); + guard.collections += 1; + guard.collected += collected; + guard.uncollectable += uncollectable; + } +} + +/// Wrapper for raw pointer to make it Send + Sync +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct GcObjectPtr(NonNull); + +// SAFETY: We only use this for tracking objects, and proper synchronization is used +unsafe impl Send for GcObjectPtr {} +unsafe impl Sync for GcObjectPtr {} + +/// Global GC state +pub struct GcState { + /// 3 generations (0 = youngest, 2 = oldest) + pub generations: [GcGeneration; 3], + /// Permanent generation (frozen objects) + pub permanent: GcGeneration, + /// GC enabled flag + pub enabled: AtomicBool, + /// Per-generation object tracking (for correct gc_refs algorithm) + /// Objects start in gen0, survivors move to gen1, then gen2 + generation_objects: [RwLock>; 3], + /// Debug flags + pub debug: AtomicU32, + /// gc.garbage list (uncollectable objects with __del__) + pub garbage: PyMutex>, + /// gc.callbacks list + pub callbacks: PyMutex>, + /// Mutex for collection (prevents concurrent collections) + collecting: Mutex<()>, + /// Allocation counter for gen0 + alloc_count: AtomicUsize, + /// Registry of all tracked objects (for cycle detection) + tracked_objects: RwLock>, + /// Objects that have been finalized (__del__ already called) + /// Prevents calling __del__ multiple times on resurrected objects + finalized_objects: RwLock>, +} + +// SAFETY: All fields are either inherently Send/Sync (atomics, RwLock, Mutex) or protected by PyMutex. +// PyMutex> is safe to share/send across threads because access is synchronized. +// PyObjectRef itself is Send, and interior mutability is guarded by the mutex. +unsafe impl Send for GcState {} +unsafe impl Sync for GcState {} + +impl Default for GcState { + fn default() -> Self { + Self::new() + } +} + +impl GcState { + pub fn new() -> Self { + Self { + generations: [ + GcGeneration::new(DEFAULT_THRESHOLD_0), + GcGeneration::new(DEFAULT_THRESHOLD_1), + GcGeneration::new(DEFAULT_THRESHOLD_2), + ], + permanent: GcGeneration::new(0), + enabled: AtomicBool::new(true), + generation_objects: [ + RwLock::new(HashSet::new()), + RwLock::new(HashSet::new()), + RwLock::new(HashSet::new()), + ], + debug: AtomicU32::new(0), + garbage: PyMutex::new(Vec::new()), + callbacks: PyMutex::new(Vec::new()), + collecting: Mutex::new(()), + alloc_count: AtomicUsize::new(0), + tracked_objects: RwLock::new(HashSet::new()), + finalized_objects: RwLock::new(HashSet::new()), + } + } + + /// Check if GC is enabled + pub fn is_enabled(&self) -> bool { + self.enabled.load(Ordering::SeqCst) + } + + /// Enable GC + pub fn enable(&self) { + self.enabled.store(true, Ordering::SeqCst); + } + + /// Disable GC + pub fn disable(&self) { + self.enabled.store(false, Ordering::SeqCst); + } + + /// Get debug flags + pub fn get_debug(&self) -> u32 { + self.debug.load(Ordering::SeqCst) + } + + /// Set debug flags + pub fn set_debug(&self, flags: u32) { + self.debug.store(flags, Ordering::SeqCst); + } + + /// Get thresholds for all generations + pub fn get_threshold(&self) -> (u32, u32, u32) { + ( + self.generations[0].threshold(), + self.generations[1].threshold(), + self.generations[2].threshold(), + ) + } + + /// Set thresholds + pub fn set_threshold(&self, t0: u32, t1: Option, t2: Option) { + self.generations[0].set_threshold(t0); + if let Some(t1) = t1 { + self.generations[1].set_threshold(t1); + } + if let Some(t2) = t2 { + self.generations[2].set_threshold(t2); + } + } + + /// Get counts for all generations + pub fn get_count(&self) -> (usize, usize, usize) { + ( + self.generations[0].count(), + self.generations[1].count(), + self.generations[2].count(), + ) + } + + /// Get statistics for all generations + pub fn get_stats(&self) -> [GcStats; 3] { + [ + self.generations[0].stats(), + self.generations[1].stats(), + self.generations[2].stats(), + ] + } + + /// Track a new object (add to gen0) + /// Called when IS_TRACE objects are created + /// + /// # Safety + /// obj must be a valid pointer to a PyObject + pub unsafe fn track_object(&self, obj: NonNull) { + let gc_ptr = GcObjectPtr(obj); + self.generations[0].count.fetch_add(1, Ordering::SeqCst); + self.alloc_count.fetch_add(1, Ordering::SeqCst); + + // Add to generation 0 tracking (for correct gc_refs algorithm) + if let Ok(mut gen0) = self.generation_objects[0].write() { + gen0.insert(gc_ptr); + } + + // Also add to global tracking (for get_objects, etc.) + if let Ok(mut tracked) = self.tracked_objects.write() { + tracked.insert(gc_ptr); + } + } + + /// Untrack an object (remove from GC lists) + /// Called when objects are deallocated + /// + /// # Safety + /// obj must be a valid pointer to a PyObject + pub unsafe fn untrack_object(&self, obj: NonNull) { + let gc_ptr = GcObjectPtr(obj); + + // Remove from all generation tracking lists + for generation in &self.generation_objects { + if let Ok(mut gen_set) = generation.write() { + if gen_set.remove(&gc_ptr) { + break; // Object can only be in one generation + } + } + } + + // Update counts (simplified - just decrement gen0 for now) + let count = self.generations[0].count.load(Ordering::SeqCst); + if count > 0 { + self.generations[0].count.fetch_sub(1, Ordering::SeqCst); + } + + // Remove from global tracking + if let Ok(mut tracked) = self.tracked_objects.write() { + tracked.remove(&gc_ptr); + } + + // Remove from finalized set + if let Ok(mut finalized) = self.finalized_objects.write() { + finalized.remove(&gc_ptr); + } + } + + /// Check if an object has been finalized + pub fn is_finalized(&self, obj: NonNull) -> bool { + let gc_ptr = GcObjectPtr(obj); + if let Ok(finalized) = self.finalized_objects.read() { + finalized.contains(&gc_ptr) + } else { + false + } + } + + /// Mark an object as finalized + pub fn mark_finalized(&self, obj: NonNull) { + let gc_ptr = GcObjectPtr(obj); + if let Ok(mut finalized) = self.finalized_objects.write() { + finalized.insert(gc_ptr); + } + } + + /// Get tracked objects (for gc.get_objects) + /// If generation is None, returns all tracked objects. + /// If generation is Some(n), returns objects in generation n only. + pub fn get_objects(&self, generation: Option) -> Vec { + match generation { + None => { + // Return all tracked objects + if let Ok(tracked) = self.tracked_objects.read() { + tracked + .iter() + .filter_map(|ptr| { + let obj = unsafe { ptr.0.as_ref() }; + if obj.strong_count() > 0 { + Some(obj.to_owned()) + } else { + None + } + }) + .collect() + } else { + Vec::new() + } + } + Some(g) if g >= 0 && g <= 2 => { + // Return objects in specific generation + let gen_idx = g as usize; + if let Ok(gen_set) = self.generation_objects[gen_idx].read() { + gen_set + .iter() + .filter_map(|ptr| { + let obj = unsafe { ptr.0.as_ref() }; + if obj.strong_count() > 0 { + Some(obj.to_owned()) + } else { + None + } + }) + .collect() + } else { + Vec::new() + } + } + _ => Vec::new(), + } + } + + /// Check if automatic GC should run and run it if needed. + /// Called after object allocation. + /// Returns true if GC was run, false otherwise. + pub fn maybe_collect(&self) -> bool { + if !self.is_enabled() { + return false; + } + + // Check gen0 threshold + let count0 = self.generations[0].count.load(Ordering::SeqCst) as u32; + let threshold0 = self.generations[0].threshold(); + if threshold0 > 0 && count0 >= threshold0 { + self.collect(0); + return true; + } + + false + } + + /// Perform garbage collection on the given generation + /// Returns (collected_count, uncollectable_count) + /// + /// Implements CPython-compatible generational GC algorithm: + /// - Only collects objects from generations 0 to `generation` + /// - Uses gc_refs algorithm: gc_refs = strong_count - internal_refs + /// - Only subtracts references between objects IN THE SAME COLLECTION + /// + /// If `force` is true, collection runs even if GC is disabled (for manual gc.collect() calls) + pub fn collect(&self, generation: usize) -> (usize, usize) { + self.collect_inner(generation, false) + } + + /// Force collection even if GC is disabled (for manual gc.collect() calls) + pub fn collect_force(&self, generation: usize) -> (usize, usize) { + self.collect_inner(generation, true) + } + + fn collect_inner(&self, generation: usize, force: bool) -> (usize, usize) { + if !force && !self.is_enabled() { + return (0, 0); + } + + // Try to acquire the collecting lock + let _guard = match self.collecting.try_lock() { + Ok(g) => g, + Err(_) => return (0, 0), + }; + + // Enter EBR critical section for the entire collection. + // This ensures that any objects being freed by other threads won't have + // their memory actually deallocated until we exit this critical section. + // Other threads' deferred deallocations will wait for us to unpin. + let ebr_guard = rustpython_common::ebr::cs(); + + // Memory barrier to ensure visibility of all reference count updates + // from other threads before we start analyzing the object graph. + std::sync::atomic::fence(Ordering::SeqCst); + + let generation = generation.min(2); + let debug = self.debug.load(Ordering::SeqCst); + + // ================================================================ + // Step 1: Gather objects from generations 0..=generation + // Hold read locks for the entire collection to prevent other threads + // from untracking objects while we're iterating. + // ================================================================ + let gen_locks: Vec<_> = (0..=generation) + .filter_map(|i| self.generation_objects[i].read().ok()) + .collect(); + + let mut collecting: HashSet = HashSet::new(); + for gen_set in &gen_locks { + for &ptr in gen_set.iter() { + let obj = unsafe { ptr.0.as_ref() }; + if obj.strong_count() > 0 { + collecting.insert(ptr); + } + } + } + + if collecting.is_empty() { + // Reset gen0 count even if nothing to collect + self.generations[0].count.store(0, Ordering::SeqCst); + self.generations[generation].update_stats(0, 0); + return (0, 0); + } + + if debug & DEBUG_STATS != 0 { + eprintln!( + "gc: collecting {} objects from generations 0..={}", + collecting.len(), + generation + ); + } + + // ================================================================ + // Step 2: Build gc_refs map (copy reference counts) + // ================================================================ + let mut gc_refs: std::collections::HashMap = + std::collections::HashMap::new(); + for &ptr in &collecting { + let obj = unsafe { ptr.0.as_ref() }; + gc_refs.insert(ptr, obj.strong_count()); + } + + // ================================================================ + // Step 3: Subtract internal references + // CRITICAL: Only subtract refs to objects IN THE COLLECTING SET + // ================================================================ + for &ptr in &collecting { + let obj = unsafe { ptr.0.as_ref() }; + // Double-check object is still alive + if obj.strong_count() == 0 { + continue; + } + let referent_ptrs = unsafe { obj.gc_get_referent_ptrs() }; + for child_ptr in referent_ptrs { + let gc_ptr = GcObjectPtr(child_ptr); + // Only decrement if child is also in the collecting set! + if collecting.contains(&gc_ptr) { + if let Some(refs) = gc_refs.get_mut(&gc_ptr) { + *refs = refs.saturating_sub(1); + } + } + } + } + + // ================================================================ + // Step 4: Find reachable objects (gc_refs > 0) and traverse from them + // Objects with gc_refs > 0 are definitely reachable from outside. + // We need to mark all objects reachable from them as also reachable. + // ================================================================ + let mut reachable: HashSet = HashSet::new(); + let mut worklist: Vec = Vec::new(); + + // Start with objects that have gc_refs > 0 + for (&ptr, &refs) in &gc_refs { + if refs > 0 { + reachable.insert(ptr); + worklist.push(ptr); + } + } + + // Traverse reachable objects to find more reachable ones + while let Some(ptr) = worklist.pop() { + let obj = unsafe { ptr.0.as_ref() }; + if obj.is_gc_tracked() { + let referent_ptrs = unsafe { obj.gc_get_referent_ptrs() }; + for child_ptr in referent_ptrs { + let gc_ptr = GcObjectPtr(child_ptr); + // If child is in collecting set and not yet marked reachable + if collecting.contains(&gc_ptr) && reachable.insert(gc_ptr) { + worklist.push(gc_ptr); + } + } + } + } + + // ================================================================ + // Step 5: Find unreachable objects (in collecting but not in reachable) + // ================================================================ + let unreachable: Vec = collecting.difference(&reachable).copied().collect(); + + if debug & DEBUG_STATS != 0 { + eprintln!( + "gc: {} reachable, {} unreachable", + reachable.len(), + unreachable.len() + ); + } + + if unreachable.is_empty() { + // No cycles found - promote survivors to next generation + drop(gen_locks); // Release read locks before promoting + self.promote_survivors(generation, &collecting); + // Reset gen0 count + self.generations[0].count.store(0, Ordering::SeqCst); + self.generations[generation].update_stats(0, 0); + return (0, 0); + } + + // Release read locks before finalization phase. + // This allows other threads to untrack objects while we finalize. + drop(gen_locks); + + // ================================================================ + // Step 6: Finalize unreachable objects and handle resurrection + // ================================================================ + + // 6a: Get references to all unreachable objects + let unreachable_refs: Vec = unreachable + .iter() + .filter_map(|ptr| { + let obj = unsafe { ptr.0.as_ref() }; + if obj.strong_count() > 0 { + Some(obj.to_owned()) + } else { + None + } + }) + .collect(); + + if unreachable_refs.is_empty() { + self.promote_survivors(generation, &reachable); + // Reset gen0 count + self.generations[0].count.store(0, Ordering::SeqCst); + self.generations[generation].update_stats(0, 0); + return (0, 0); + } + + // 6b: Record initial strong counts (for resurrection detection) + // Each object has +1 from unreachable_refs, so initial count includes that + let initial_counts: std::collections::HashMap = unreachable_refs + .iter() + .map(|obj| { + let ptr = GcObjectPtr(core::ptr::NonNull::from(obj.as_ref())); + (ptr, obj.strong_count()) + }) + .collect(); + + // 6c: Clear existing weakrefs BEFORE calling __del__ + // This invalidates existing weakrefs, but new weakrefs created during __del__ + // will still work (WeakRefList::add restores inner.obj if cleared) + for obj_ref in &unreachable_refs { + obj_ref.gc_clear_weakrefs(); + } + + // 6d: Call __del__ on all unreachable objects + // This allows resurrection to work correctly + // Skip objects that have already been finalized (prevents multiple __del__ calls) + for obj_ref in &unreachable_refs { + let ptr = GcObjectPtr(core::ptr::NonNull::from(obj_ref.as_ref())); + let already_finalized = if let Ok(finalized) = self.finalized_objects.read() { + finalized.contains(&ptr) + } else { + false + }; + + if !already_finalized { + // Mark as finalized BEFORE calling __del__ + // This ensures is_finalized() returns True inside __del__ + if let Ok(mut finalized) = self.finalized_objects.write() { + finalized.insert(ptr); + } + obj_ref.try_call_finalizer(); + } + } + + // 6d: Detect resurrection - strong_count increased means object was resurrected + // Step 1: Find directly resurrected objects (strong_count increased) + let mut resurrected_set: HashSet = HashSet::new(); + let unreachable_set: HashSet = unreachable.iter().copied().collect(); + + for obj in &unreachable_refs { + let ptr = GcObjectPtr(core::ptr::NonNull::from(obj.as_ref())); + let initial = initial_counts.get(&ptr).copied().unwrap_or(1); + if obj.strong_count() > initial { + resurrected_set.insert(ptr); + } + } + + // Step 2: Transitive resurrection - objects reachable from resurrected are also resurrected + // This is critical for cases like: Lazarus resurrects itself, its cargo should also survive + let mut worklist: Vec = resurrected_set.iter().copied().collect(); + while let Some(ptr) = worklist.pop() { + let obj = unsafe { ptr.0.as_ref() }; + let referent_ptrs = unsafe { obj.gc_get_referent_ptrs() }; + for child_ptr in referent_ptrs { + let child_gc_ptr = GcObjectPtr(child_ptr); + // If child is in unreachable set and not yet marked as resurrected + if unreachable_set.contains(&child_gc_ptr) && resurrected_set.insert(child_gc_ptr) { + worklist.push(child_gc_ptr); + } + } + } + + // Step 3: Partition into resurrected and truly dead + let (resurrected, truly_dead): (Vec<_>, Vec<_>) = + unreachable_refs.into_iter().partition(|obj| { + let ptr = GcObjectPtr(core::ptr::NonNull::from(obj.as_ref())); + resurrected_set.contains(&ptr) + }); + + let resurrected_count = resurrected.len(); + + if debug & DEBUG_STATS != 0 { + eprintln!( + "gc: {} resurrected, {} truly dead", + resurrected_count, + truly_dead.len() + ); + } + + // 6e: Break cycles ONLY for truly dead objects (not resurrected) + // Only count objects with pop_edges (containers like list, dict, tuple) + // This matches CPython's behavior where instance objects themselves + // are not counted, only their __dict__ and other container types + let collected = truly_dead + .iter() + .filter(|obj| obj.gc_has_pop_edges()) + .count(); + + // 6e-1: If DEBUG_SAVEALL is set, save truly dead objects to garbage + if debug & DEBUG_SAVEALL != 0 { + let mut garbage_guard = self.garbage.lock(); + for obj_ref in truly_dead.iter() { + if obj_ref.gc_has_pop_edges() { + garbage_guard.push(obj_ref.clone()); + } + } + } + + if !truly_dead.is_empty() { + // 6g: Break cycles by clearing references (tp_clear equivalent) + // Weakrefs were already cleared in step 6c, but new weakrefs created + // during __del__ (step 6d) can still be upgraded. + // + // Pop edges and destroy objects using the ebr_guard from the start of collection. + // The guard ensures deferred deallocations from other threads wait for us. + rustpython_common::refcount::with_deferred_drops(|| { + for obj_ref in truly_dead.iter() { + if obj_ref.gc_has_pop_edges() { + let edges = unsafe { obj_ref.gc_pop_edges() }; + drop(edges); + } + } + // Drop truly_dead references, triggering actual deallocation + drop(truly_dead); + }); + } + + // 6f: Resurrected objects stay in tracked_objects (they're still alive) + // Just drop our references to them + drop(resurrected); + + // Promote survivors (reachable objects) to next generation + self.promote_survivors(generation, &reachable); + + // Reset gen0 count after collection (enables automatic GC to trigger again) + self.generations[0].count.store(0, Ordering::SeqCst); + + self.generations[generation].update_stats(collected as u64, 0); + + // Flush EBR deferred operations before exiting collection. + // This ensures any deferred deallocations from this collection are executed. + ebr_guard.flush(); + + (collected, 0) + } + + /// Promote surviving objects to the next generation + fn promote_survivors(&self, from_gen: usize, survivors: &HashSet) { + if from_gen >= 2 { + return; // Already in oldest generation + } + + let next_gen = from_gen + 1; + + for &ptr in survivors { + // Remove from current generation + for gen_idx in 0..=from_gen { + if let Ok(mut gen_set) = self.generation_objects[gen_idx].write() { + if gen_set.remove(&ptr) { + // Add to next generation + if let Ok(mut next_set) = self.generation_objects[next_gen].write() { + next_set.insert(ptr); + } + break; + } + } + } + } + } + + /// Get count of frozen objects + pub fn get_freeze_count(&self) -> usize { + self.permanent.count() + } + + /// Freeze all tracked objects (move to permanent generation) + pub fn freeze(&self) { + // Move all objects from gen0-2 to permanent + for generation in &self.generations { + let count = generation.count.swap(0, Ordering::SeqCst); + self.permanent.count.fetch_add(count, Ordering::SeqCst); + } + } + + /// Unfreeze all objects (move from permanent to gen2) + pub fn unfreeze(&self) { + let count = self.permanent.count.swap(0, Ordering::SeqCst); + self.generations[2].count.fetch_add(count, Ordering::SeqCst); + } +} + +use std::sync::OnceLock; + +/// Global GC state instance +/// Using a static because GC needs to be accessible from object allocation/deallocation +static GC_STATE: OnceLock = OnceLock::new(); + +/// Get a reference to the global GC state +pub fn gc_state() -> &'static GcState { + GC_STATE.get_or_init(GcState::new) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_gc_state_default() { + let state = GcState::new(); + assert!(state.is_enabled()); + assert_eq!(state.get_debug(), 0); + assert_eq!(state.get_threshold(), (700, 10, 10)); + assert_eq!(state.get_count(), (0, 0, 0)); + } + + #[test] + fn test_gc_enable_disable() { + let state = GcState::new(); + assert!(state.is_enabled()); + state.disable(); + assert!(!state.is_enabled()); + state.enable(); + assert!(state.is_enabled()); + } + + #[test] + fn test_gc_threshold() { + let state = GcState::new(); + state.set_threshold(100, Some(20), Some(30)); + assert_eq!(state.get_threshold(), (100, 20, 30)); + } + + #[test] + fn test_gc_debug_flags() { + let state = GcState::new(); + state.set_debug(DEBUG_STATS | DEBUG_COLLECTABLE); + assert_eq!(state.get_debug(), DEBUG_STATS | DEBUG_COLLECTABLE); + } +} diff --git a/crates/vm/src/lib.rs b/crates/vm/src/lib.rs index 3f0eee278a..9380f55696 100644 --- a/crates/vm/src/lib.rs +++ b/crates/vm/src/lib.rs @@ -77,6 +77,7 @@ pub mod py_io; #[cfg(feature = "serde")] pub mod py_serde; +pub mod gc_state; pub mod readline; pub mod recursion; pub mod scope; diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index e904117b06..916bb66550 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -99,6 +99,12 @@ pub(super) unsafe fn try_trace_obj(x: &PyObject, tracer_fn: &mut T payload.try_traverse(tracer_fn) } +/// Call `try_pop_edges` on payload to extract child references +pub(super) unsafe fn try_pop_edges_obj(x: *mut PyObject, out: &mut Vec) { + let x = unsafe { &mut *(x as *mut PyInner) }; + x.payload.try_pop_edges(out); +} + /// This is an actual python object. It consists of a `typ` which is the /// python class, and carries some rust payload optionally. This rust /// payload can be a rust float or rust int in case of float and int objects. @@ -195,6 +201,11 @@ impl WeakRefList { })) }); let mut inner = unsafe { inner_ptr.as_ref().lock() }; + // If obj was cleared by GC but object is still alive (e.g., new weakref + // created during __del__), restore the obj pointer + if inner.obj.is_none() { + inner.obj = Some(NonNull::from(obj)); + } if is_generic && let Some(generic_weakref) = inner.generic_weakref { let generic_weakref = unsafe { generic_weakref.as_ref() }; if generic_weakref.0.ref_count.get() != 0 { @@ -218,14 +229,33 @@ impl WeakRefList { weak } + /// Clear all weakrefs and call their callbacks. + /// This is the main clear function called when the owner object is being dropped. + /// It decrements ref_count and deallocates if needed. fn clear(&self) { + self.clear_inner(true) + } + + /// Clear all weakrefs and call their callbacks, but don't decrement ref_count. + /// Used by GC when clearing weakrefs before pop_edges. + /// The owner object is not being dropped yet, so we shouldn't decrement ref_count. + fn clear_for_gc(&self) { + self.clear_inner(false) + } + + fn clear_inner(&self, decrement_ref_count: bool) { let to_dealloc = { let ptr = match self.inner.get() { Some(ptr) => ptr, None => return, }; let mut inner = unsafe { ptr.as_ref().lock() }; + + // If already cleared (obj is None), skip the ref_count decrement + // to avoid double decrement when called by both GC and drop_slow_inner + let already_cleared = inner.obj.is_none(); inner.obj = None; + // TODO: can be an arrayvec let mut v = Vec::with_capacity(16); loop { @@ -265,8 +295,14 @@ impl WeakRefList { } }) } - inner.ref_count -= 1; - (inner.ref_count == 0).then_some(ptr) + + // Only decrement ref_count if requested AND not already cleared + if decrement_ref_count && !already_cleared { + inner.ref_count -= 1; + (inner.ref_count == 0).then_some(ptr) + } else { + None + } }; if let Some(ptr) = to_dealloc { unsafe { Self::dealloc(ptr) } @@ -793,15 +829,34 @@ impl PyObject { slot_del: fn(&PyObject, &VirtualMachine) -> PyResult<()>, ) -> Result<(), ()> { let ret = crate::vm::thread::with_vm(zelf, |vm| { + // Note: inc() from 0 does a double increment (0→2) for thread safety. + // This gives us "permission" to decrement twice. zelf.0.ref_count.inc(); + let after_inc = zelf.strong_count(); // Should be 2 + if let Err(e) = slot_del(zelf, vm) { let del_method = zelf.get_class_attr(identifier!(vm, __del__)).unwrap(); vm.run_unraisable(e, None, del_method); } + + let after_del = zelf.strong_count(); + + // First decrement + zelf.0.ref_count.dec(); + + // Check for resurrection: if ref_count increased beyond our expected 2, + // then __del__ created new references (resurrection occurred). + if after_del > after_inc { + // Resurrected - don't do second decrement, leave object alive + return false; + } + + // No resurrection - do second decrement to get back to 0 + // This matches the double increment from inc() zelf.0.ref_count.dec() }); match ret { - // the decref right above set ref_count back to 0 + // the decref set ref_count back to 0 Some(true) => Ok(()), // we've been resurrected by __del__ Some(false) => Err(()), @@ -812,28 +867,76 @@ impl PyObject { } } + // Clear weak refs FIRST (before __del__), consistent with GC behavior. + // GC clears weakrefs before calling finalizers (gc_state.rs:554-559). + // This ensures weakref holders are notified even if __del__ causes resurrection. + if let Some(wrl) = self.weak_ref_list() { + wrl.clear(); + } + // CPython-compatible drop implementation let del = self.class().slots.del.load(); if let Some(slot_del) = del { - call_slot_del(self, slot_del)?; - } - if let Some(wrl) = self.weak_ref_list() { - wrl.clear(); + // Check if already finalized by GC (prevents double __del__ calls) + let ptr = core::ptr::NonNull::from(self); + let gc = crate::gc_state::gc_state(); + if !gc.is_finalized(ptr) { + // Mark as finalized BEFORE calling __del__ + // This ensures is_finalized() returns True even if object is resurrected + gc.mark_finalized(ptr); + call_slot_del(self, slot_del)?; + } } Ok(()) } /// Can only be called when ref_count has dropped to zero. `ptr` must be valid + /// + /// This implements immediate recursive destruction for circular reference resolution: + /// 1. Call __del__ if present + /// 2. Extract child references via pop_edges() + /// 3. Deallocate the object + /// 4. Drop child references (may trigger recursive destruction) #[inline(never)] unsafe fn drop_slow(ptr: NonNull) { if let Err(()) = unsafe { ptr.as_ref().drop_slow_inner() } { - // abort drop for whatever reason + // abort drop for whatever reason (e.g., resurrection in __del__) return; } - let drop_dealloc = unsafe { ptr.as_ref().0.vtable.drop_dealloc }; + + let vtable = unsafe { ptr.as_ref().0.vtable }; + let has_dict = unsafe { ptr.as_ref().0.dict.is_some() }; + + // Untrack object from GC BEFORE deallocation. + // This ensures the object is not in generation_objects when we free its memory. + // Must match the condition in PyRef::new_ref: IS_TRACE || has_dict + if vtable.trace.is_some() || has_dict { + // Try to untrack immediately. If we can't acquire the lock (e.g., GC is running), + // defer the untrack operation. + rustpython_common::refcount::try_defer_drop(move || { + // SAFETY: untrack_object only removes the pointer address from a HashSet. + // It does NOT dereference the pointer, so it's safe even after deallocation. + unsafe { + crate::gc_state::gc_state().untrack_object(ptr); + } + }); + } + + // Extract child references before deallocation to break circular refs + let mut edges = Vec::new(); + if let Some(pop_edges_fn) = vtable.pop_edges { + unsafe { pop_edges_fn(ptr.as_ptr(), &mut edges) }; + } + + // Deallocate the object + let drop_dealloc = vtable.drop_dealloc; // call drop only when there are no references in scope - stacked borrows stuff unsafe { drop_dealloc(ptr.as_ptr()) } + + // Now drop child references - this may trigger recursive destruction + // The object is already deallocated, so circular refs are broken + drop(edges); } /// # Safety @@ -853,6 +956,122 @@ impl PyObject { pub(crate) fn set_slot(&self, offset: usize, value: Option) { *self.0.slots[offset].write() = value; } + + /// Check if this object is tracked by the garbage collector. + /// Returns true if the object has IS_TRACE = true (has a trace function) + /// or has an instance dict (user-defined class instances). + pub fn is_gc_tracked(&self) -> bool { + // Objects with trace function are tracked + if self.0.vtable.trace.is_some() { + return true; + } + // Objects with instance dict are also tracked (user-defined class instances) + self.0.dict.is_some() + } + + /// Call __del__ if present, without triggering object deallocation. + /// Used by GC to call finalizers before breaking cycles. + /// This allows proper resurrection detection. + pub fn try_call_finalizer(&self) { + let del = self.class().slots.del.load(); + if let Some(slot_del) = del { + crate::vm::thread::with_vm(self, |vm| { + if let Err(e) = slot_del(self, vm) { + if let Some(del_method) = self.get_class_attr(identifier!(vm, __del__)) { + vm.run_unraisable(e, None, del_method); + } + } + }); + } + } + + /// Clear weakrefs to this object, calling their callbacks. + /// Used by GC to notify weakref holders before object is collected. + /// Clear weakrefs for GC collection. + /// This calls weakref callbacks but doesn't decrement ref_count, + /// because the owner object is not being dropped yet. + pub fn gc_clear_weakrefs(&self) { + if let Some(wrl) = self.weak_ref_list() { + wrl.clear_for_gc(); + } + } + + /// Get the referents (objects directly referenced) of this object. + /// Uses the full traverse including dict and slots. + pub fn gc_get_referents(&self) -> Vec { + let mut result = Vec::new(); + // Traverse the entire object including dict and slots + self.0.traverse(&mut |child: &PyObject| { + result.push(child.to_owned()); + }); + result + } + + /// Get raw pointers to referents without incrementing reference counts. + /// This is used during GC to avoid reference count manipulation. + /// + /// # Safety + /// The returned pointers are only valid as long as the object is alive + /// and its contents haven't been modified. + pub unsafe fn gc_get_referent_ptrs(&self) -> Vec> { + let mut result = Vec::new(); + // Traverse the entire object including dict and slots + self.0.traverse(&mut |child: &PyObject| { + result.push(NonNull::from(child)); + }); + result + } + + /// This is an internal method for type-specific payload traversal only. + /// Most code should use gc_get_referent_ptrs instead. + #[allow(dead_code)] + pub unsafe fn gc_get_referent_ptrs_payload_only(&self) -> Vec> { + let mut result = Vec::new(); + if let Some(trace_fn) = self.0.vtable.trace { + unsafe { + trace_fn(self, &mut |child: &PyObject| { + result.push(NonNull::from(child)); + }); + } + } + result + } + + /// Pop edges from this object for cycle breaking. + /// Returns extracted child references that were removed from this object. + /// This is used during garbage collection to break circular references. + /// + /// # Safety + /// - ptr must be a valid pointer to a PyObject + /// - The caller must have exclusive access (no other references exist) + /// - This is only safe during GC when the object is unreachable + pub unsafe fn gc_pop_edges_raw(ptr: *mut PyObject) -> Vec { + let mut result = Vec::new(); + let obj = unsafe { &*ptr }; + if let Some(pop_edges_fn) = obj.0.vtable.pop_edges { + unsafe { pop_edges_fn(ptr, &mut result) }; + } + result + } + + /// Pop edges from this object for cycle breaking. + /// This version takes &self but should only be called during GC + /// when exclusive access is guaranteed. + /// + /// # Safety + /// - The caller must guarantee exclusive access (no other references exist) + /// - This is only safe during GC when the object is unreachable + pub unsafe fn gc_pop_edges(&self) -> Vec { + // SAFETY: During GC collection, this object is unreachable (gc_refs == 0), + // meaning no other code has a reference to it. The only references are + // internal cycle references which we're about to break. + unsafe { Self::gc_pop_edges_raw(self as *const _ as *mut PyObject) } + } + + /// Check if this object has pop_edges capability + pub fn gc_has_pop_edges(&self) -> bool { + self.0.vtable.pop_edges.is_some() + } } impl Borrow for PyObjectRef { @@ -1069,10 +1288,22 @@ impl PyRef { impl PyRef { #[inline(always)] pub fn new_ref(payload: T, typ: crate::builtins::PyTypeRef, dict: Option) -> Self { + let has_dict = dict.is_some(); let inner = Box::into_raw(PyInner::new(payload, typ, dict)); - Self { - ptr: unsafe { NonNull::new_unchecked(inner.cast::>()) }, + let ptr = unsafe { NonNull::new_unchecked(inner.cast::>()) }; + + // Track object if IS_TRACE is true OR has instance dict + // (user-defined class instances have dict but may not have IS_TRACE) + if T::IS_TRACE || has_dict { + let gc = crate::gc_state::gc_state(); + unsafe { + gc.track_object(ptr.cast()); + } + // Check if automatic GC should run + gc.maybe_collect(); } + + Self { ptr } } } diff --git a/crates/vm/src/object/traverse.rs b/crates/vm/src/object/traverse.rs index 2ce0db41a5..994685fe26 100644 --- a/crates/vm/src/object/traverse.rs +++ b/crates/vm/src/object/traverse.rs @@ -13,8 +13,12 @@ pub type TraverseFn<'a> = dyn FnMut(&PyObject) + 'a; pub trait MaybeTraverse { /// if is traceable, will be used by vtable to determine const IS_TRACE: bool = false; + /// if has pop_edges implementation for circular reference resolution + const HAS_POP_EDGES: bool = false; // if this type is traceable, then call with tracer_fn, default to do nothing fn try_traverse(&self, traverse_fn: &mut TraverseFn<'_>); + // if this type has pop_edges, extract child refs for circular reference resolution + fn try_pop_edges(&mut self, _out: &mut Vec) {} } /// Type that need traverse it's children should impl [`Traverse`] (not [`MaybeTraverse`]) @@ -28,6 +32,11 @@ pub unsafe trait Traverse { /// /// - _**DO NOT**_ clone a [`PyObjectRef`] or [`PyRef`] in [`Traverse::traverse()`] fn traverse(&self, traverse_fn: &mut TraverseFn<'_>); + + /// Extract all owned child PyObjectRefs for circular reference resolution. + /// Called just before object deallocation to break circular references. + /// Default implementation does nothing. + fn pop_edges(&mut self, _out: &mut Vec) {} } unsafe impl Traverse for PyObjectRef { diff --git a/crates/vm/src/object/traverse_object.rs b/crates/vm/src/object/traverse_object.rs index 075ce5b951..571a1073cb 100644 --- a/crates/vm/src/object/traverse_object.rs +++ b/crates/vm/src/object/traverse_object.rs @@ -1,10 +1,10 @@ use alloc::fmt; use crate::{ - PyObject, + PyObject, PyObjectRef, object::{ Erased, InstanceDict, MaybeTraverse, PyInner, PyObjectPayload, debug_obj, drop_dealloc_obj, - try_trace_obj, + try_pop_edges_obj, try_trace_obj, }, }; @@ -14,6 +14,9 @@ pub(in crate::object) struct PyObjVTable { pub(in crate::object) drop_dealloc: unsafe fn(*mut PyObject), pub(in crate::object) debug: unsafe fn(&PyObject, &mut fmt::Formatter<'_>) -> fmt::Result, pub(in crate::object) trace: Option)>, + /// Pop edges for circular reference resolution. + /// Called just before deallocation to extract child references. + pub(in crate::object) pop_edges: Option)>, } impl PyObjVTable { @@ -28,6 +31,13 @@ impl PyObjVTable { None } }, + pop_edges: const { + if T::HAS_POP_EDGES { + Some(try_pop_edges_obj::) + } else { + None + } + }, } } } diff --git a/crates/vm/src/stdlib/thread.rs b/crates/vm/src/stdlib/thread.rs index eb88191165..be0afc1f8a 100644 --- a/crates/vm/src/stdlib/thread.rs +++ b/crates/vm/src/stdlib/thread.rs @@ -349,6 +349,10 @@ pub(crate) mod _thread { } fn run_thread(func: ArgCallable, args: FuncArgs, vm: &VirtualMachine) { + // Enter EBR critical section for this thread (Coarse-grained pinning) + // This ensures GC won't free objects while this thread might access them + crate::vm::thread::ensure_pinned(); + match func.invoke(args, vm) { Ok(_obj) => {} Err(e) if e.fast_isinstance(vm.ctx.exceptions.system_exit) => {} @@ -366,6 +370,9 @@ pub(crate) mod _thread { } } vm.state.thread_count.fetch_sub(1); + + // Drop EBR guard when thread exits, allowing epoch advancement + crate::vm::thread::drop_guard(); } #[cfg(not(target_arch = "wasm32"))] @@ -400,6 +407,164 @@ pub(crate) mod _thread { vm.state.thread_count.load() } + /// ExceptHookArgs - simple class to hold exception hook arguments + /// This allows threading.py to import _excepthook and _ExceptHookArgs from _thread + #[pyattr] + #[pyclass(module = "_thread", name = "_ExceptHookArgs")] + #[derive(Debug, PyPayload)] + struct ExceptHookArgs { + exc_type: crate::PyObjectRef, + exc_value: crate::PyObjectRef, + exc_traceback: crate::PyObjectRef, + thread: crate::PyObjectRef, + } + + #[pyclass(with(Constructor))] + impl ExceptHookArgs { + #[pygetset] + fn exc_type(&self) -> crate::PyObjectRef { + self.exc_type.clone() + } + + #[pygetset] + fn exc_value(&self) -> crate::PyObjectRef { + self.exc_value.clone() + } + + #[pygetset] + fn exc_traceback(&self) -> crate::PyObjectRef { + self.exc_traceback.clone() + } + + #[pygetset] + fn thread(&self) -> crate::PyObjectRef { + self.thread.clone() + } + } + + impl Constructor for ExceptHookArgs { + // Takes a single iterable argument like namedtuple + type Args = (crate::PyObjectRef,); + + fn py_new( + _cls: &Py, + args: Self::Args, + vm: &VirtualMachine, + ) -> PyResult { + // Convert the argument to a list/tuple and extract elements + let seq: Vec = args.0.try_to_value(vm)?; + if seq.len() != 4 { + return Err(vm.new_type_error(format!( + "_ExceptHookArgs expected 4 arguments, got {}", + seq.len() + ))); + } + Ok(Self { + exc_type: seq[0].clone(), + exc_value: seq[1].clone(), + exc_traceback: seq[2].clone(), + thread: seq[3].clone(), + }) + } + } + + /// Handle uncaught exception in Thread.run() + /// Suppresses exceptions during interpreter shutdown. + #[pyfunction] + fn _excepthook(args: crate::PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + use std::sync::atomic::Ordering; + + // During interpreter finalization, suppress exceptions from daemon threads + if vm.state.finalizing.load(Ordering::Acquire) { + return Ok(()); + } + + // Get exception info from args (ExceptHookArgs namedtuple) + let exc_type = args.get_attr("exc_type", vm)?; + let exc_value = args.get_attr("exc_value", vm).ok(); + let exc_traceback = args.get_attr("exc_traceback", vm).ok(); + let thread = args.get_attr("thread", vm).ok(); + + // Check for SystemExit - should be silently ignored + // exc_type is the exception TYPE (e.g., SystemExit class), not an instance + // We compare by checking if exc_type IS SystemExit (same object identity) + // or if exc_type is a subclass of SystemExit + let is_system_exit = exc_type + .downcast_ref::() + .is_some_and(|ty| ty.fast_issubclass(vm.ctx.exceptions.system_exit)); + if is_system_exit { + return Ok(()); + } + + // Get stderr - fall back to thread._stderr if sys.stderr is None + let stderr = match vm.sys_module.get_attr("stderr", vm) { + Ok(stderr) if !vm.is_none(&stderr) => stderr, + _ => { + // Try to get thread._stderr as fallback + if let Some(ref thread) = thread { + if !vm.is_none(thread) { + if let Ok(thread_stderr) = thread.get_attr("_stderr", vm) { + if !vm.is_none(&thread_stderr) { + thread_stderr + } else { + return Ok(()); + } + } else { + return Ok(()); + } + } else { + return Ok(()); + } + } else { + return Ok(()); + } + } + }; + + // Print "Exception in thread :" + let thread_name = if let Some(thread) = &thread { + if !vm.is_none(thread) { + thread + .get_attr("name", vm) + .ok() + .and_then(|n| n.str(vm).ok()) + .map(|s| s.as_str().to_owned()) + } else { + None + } + } else { + None + }; + let name = thread_name.unwrap_or_else(|| format!("{}", get_ident())); + + let _ = vm.call_method(&stderr, "write", (format!("Exception in thread {}:\n", name),)); + let _ = vm.call_method(&stderr, "flush", ()); + + // Print the traceback using traceback.print_exception + // Pass file=stderr to ensure output goes to the correct stderr + if let Ok(traceback_mod) = vm.import("traceback", 0) { + if let Ok(print_exc) = traceback_mod.get_attr("print_exception", vm) { + use crate::function::KwArgs; + let kwargs: KwArgs = + vec![("file".to_owned(), stderr.clone())].into_iter().collect(); + let _ = print_exc.call_with_args( + crate::function::FuncArgs::new( + vec![ + exc_type.clone(), + exc_value.unwrap_or_else(|| vm.ctx.none()), + exc_traceback.unwrap_or_else(|| vm.ctx.none()), + ], + kwargs, + ), + vm, + ); + } + } + + let _ = vm.call_method(&stderr, "flush", ()); + Ok(()) + } + #[pyattr] #[pyclass(module = "thread", name = "_local")] #[derive(Debug, PyPayload)] diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index b12352f6ee..72ddefceed 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -51,6 +51,10 @@ pub struct Context { pub(crate) string_pool: StringPool, pub(crate) slot_new_wrapper: PyMethodDef, pub names: ConstName, + + // GC module state (callbacks and garbage lists) + pub gc_callbacks: PyListRef, + pub gc_garbage: PyListRef, } macro_rules! declare_const_name { @@ -328,6 +332,11 @@ impl Context { let empty_str = unsafe { string_pool.intern("", types.str_type.to_owned()) }; let empty_bytes = create_object(PyBytes::from(Vec::new()), types.bytes_type); + + // GC callbacks and garbage lists + let gc_callbacks = PyRef::new_ref(PyList::default(), types.list_type.to_owned(), None); + let gc_garbage = PyRef::new_ref(PyList::default(), types.list_type.to_owned(), None); + Self { true_value, false_value, @@ -347,6 +356,9 @@ impl Context { string_pool, slot_new_wrapper, names, + + gc_callbacks, + gc_garbage, } } diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 8d37ad6c84..fe3123536f 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -110,11 +110,14 @@ impl Interpreter { /// Finalize vm and turns an exception to exit code. /// - /// Finalization steps including 4 steps: + /// Finalization steps: /// 1. Flush stdout and stderr. - /// 1. Handle exit exception and turn it to exit code. - /// 1. Run atexit exit functions. - /// 1. Mark vm as finalized. + /// 2. Handle exit exception and turn it to exit code. + /// 3. Set finalizing flag (suppresses unraisable exceptions). + /// 4. Call threading._shutdown() to join non-daemon threads. + /// 5. Run atexit exit functions. + /// 6. GC pass and module cleanup. + /// 7. Final GC pass. /// /// Note that calling `finalize` is not necessary by purpose though. pub fn finalize(self, exc: Option) -> u32 { @@ -128,9 +131,31 @@ impl Interpreter { 0 }; + // Set finalizing flag early to suppress unraisable exceptions from + // daemon threads and __del__ methods during shutdown. + vm.state.finalizing.store(true, Ordering::Release); + + // Call threading._shutdown() to properly join non-daemon threads. + // This must happen before module cleanup to ensure threads can + // finish cleanly while modules are still available. + if let Ok(threading) = vm.import("threading", 0) { + if let Ok(shutdown) = threading.get_attr("_shutdown", vm) { + let _ = shutdown.call((), vm); + } + } + atexit::_run_exitfuncs(vm); - vm.state.finalizing.store(true, Ordering::Release); + // First GC pass - collect cycles before module cleanup + crate::gc_state::gc_state().collect_force(2); + + // Clear modules to break references to objects in module namespaces. + // This allows cyclic garbage created in modules to be collected. + vm.finalize_modules(); + + // Second GC pass - now cyclic garbage in modules can be collected + // and __del__ methods will be called + crate::gc_state::gc_state().collect_force(2); vm.flush_std(); diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 7f7c5ea767..2b488ae337 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -17,8 +17,8 @@ mod vm_ops; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, builtins::{ - PyBaseExceptionRef, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned, PyStrRef, - PyTypeRef, code::PyCode, pystr::AsPyStr, tuple::PyTuple, + PyBaseExceptionRef, PyDict, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned, + PyStrRef, PyTypeRef, code::PyCode, pystr::AsPyStr, tuple::PyTuple, }, codecs::CodecsRegistry, common::{hash::HashSecret, lock::PyMutex, rc::PyRc}, @@ -472,6 +472,13 @@ impl VirtualMachine { #[cold] pub fn run_unraisable(&self, e: PyBaseExceptionRef, msg: Option, object: PyObjectRef) { + // Suppress unraisable exceptions during interpreter finalization. + // This matches CPython behavior where daemon thread exceptions and + // __del__ errors are silently ignored during shutdown. + if self.state.finalizing.load(std::sync::atomic::Ordering::Acquire) { + return; + } + let sys_module = self.import("sys", 0).unwrap(); let unraisablehook = sys_module.get_attr("unraisablehook", self).unwrap(); @@ -523,6 +530,12 @@ impl VirtualMachine { let result = f(frame); // defer dec frame let _popped = self.frames.borrow_mut().pop(); + + // Reactivate EBR guard at frame boundary (safe point) + // This allows GC to advance epochs and free deferred objects + #[cfg(feature = "threading")] + crate::vm::thread::reactivate_guard(); + result }) } @@ -965,6 +978,79 @@ impl VirtualMachine { Ok(()) } + /// Clear module references during shutdown. + /// This breaks references from modules to objects, allowing cyclic garbage + /// to be collected in the subsequent GC pass. + /// + /// Clears __main__ and user-imported modules while preserving stdlib modules + /// needed for __del__ to work correctly (e.g., print, traceback, etc.). + pub fn finalize_modules(&self) { + // Get sys.modules dict + if let Ok(modules) = self.sys_module.get_attr(identifier!(self, modules), self) { + if let Some(modules_dict) = modules.downcast_ref::() { + // First pass: clear __main__ module + if let Ok(main_module) = modules_dict.get_item("__main__", self) { + if let Some(module) = main_module.downcast_ref::() { + module.dict().clear(); + } + } + + // Second pass: clear user modules (non-stdlib) + // A module is considered "user" if it has a __file__ attribute + // that doesn't point to the stdlib location + let module_items: Vec<_> = modules_dict.into_iter().collect(); + for (key, value) in &module_items { + if let Some(key_str) = key.downcast_ref::() { + let name = key_str.as_str(); + // Skip stdlib modules (starting with _ or known stdlib names) + if name.starts_with('_') + || matches!( + name, + "sys" + | "builtins" + | "os" + | "io" + | "traceback" + | "linecache" + | "posixpath" + | "ntpath" + | "genericpath" + | "abc" + | "codecs" + | "encodings" + | "stat" + | "collections" + | "functools" + | "types" + | "importlib" + | "warnings" + | "weakref" + | "gc" + ) + { + continue; + } + } + if let Some(module) = value.downcast_ref::() { + // Check if this is a user module by looking for __file__ + if let Ok(file_attr) = module.dict().get_item("__file__", self) { + if !self.is_none(&file_attr) { + // Has __file__ - check if it's not in stdlib paths + if let Some(file_str) = file_attr.downcast_ref::() { + let file_path = file_str.as_str(); + // Clear if not in pylib (stdlib) + if !file_path.contains("pylib") && !file_path.contains("Lib") { + module.dict().clear(); + } + } + } + } + } + } + } + } + } + pub fn fs_encoding(&self) -> &'static PyStrInterned { identifier!(self, utf_8) } diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 7e8f0f87e5..a460563d7e 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -12,10 +12,52 @@ thread_local! { pub(crate) static COROUTINE_ORIGIN_TRACKING_DEPTH: Cell = const { Cell::new(0) }; pub(crate) static ASYNC_GEN_FINALIZER: RefCell> = const { RefCell::new(None) }; pub(crate) static ASYNC_GEN_FIRSTITER: RefCell> = const { RefCell::new(None) }; + + /// Thread-local EBR guard for Coarse-grained pinning strategy. + /// Holds the EBR critical section guard for this thread. + pub(crate) static EBR_GUARD: RefCell> = + const { RefCell::new(None) }; } scoped_tls::scoped_thread_local!(static VM_CURRENT: VirtualMachine); +/// Ensure the current thread is pinned for EBR. +/// Call this at the start of operations that access Python objects. +/// +/// This is part of the Coarse-grained pinning strategy where threads +/// are pinned at entry and periodically reactivate at safe points. +#[inline] +pub fn ensure_pinned() { + EBR_GUARD.with(|guard| { + if guard.borrow().is_none() { + *guard.borrow_mut() = Some(rustpython_common::ebr::cs()); + } + }); +} + +/// Reactivate the EBR guard to allow epoch advancement. +/// Call this at safe points where no object references are held temporarily. +/// +/// This unblocks GC from advancing epochs, allowing deferred objects to be freed. +/// The guard remains active after reactivation. +#[inline] +pub fn reactivate_guard() { + EBR_GUARD.with(|guard| { + if let Some(ref mut g) = *guard.borrow_mut() { + g.reactivate(); + } + }); +} + +/// Drop the EBR guard, unpinning this thread. +/// Call this when the thread is exiting or no longer needs EBR protection. +#[inline] +pub fn drop_guard() { + EBR_GUARD.with(|guard| { + *guard.borrow_mut() = None; + }); +} + pub fn with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> R { if !VM_CURRENT.is_set() { panic!("call with_current_vm() but VM_CURRENT is null"); diff --git a/crates/vm/src/warn.rs b/crates/vm/src/warn.rs index 09d48078e5..9e783f29e1 100644 --- a/crates/vm/src/warn.rs +++ b/crates/vm/src/warn.rs @@ -404,8 +404,41 @@ fn setup_context( let (globals, filename, lineno) = if let Some(f) = f { (f.globals.clone(), f.code.source_path, f.f_lineno()) + } else if let Some(frame) = vm.current_frame() { + // We have a frame but it wasn't found during stack walking + (frame.globals.clone(), vm.ctx.intern_str("sys"), 1) } else { - (vm.current_globals().clone(), vm.ctx.intern_str("sys"), 1) + // No frames on the stack - we're in interpreter shutdown or similar state + // Use __main__ globals if available, otherwise skip the warning + match vm + .sys_module + .get_attr(identifier!(vm, modules), vm) + .and_then(|modules| modules.get_item(vm.ctx.intern_str("__main__"), vm)) + .and_then(|main_mod| main_mod.get_attr(identifier!(vm, __dict__), vm)) + { + Ok(globals) => { + if let Ok(dict) = globals.downcast::() { + (dict, vm.ctx.intern_str("sys"), 1) + } else { + // Cannot get globals, skip warning + return Ok(( + vm.ctx.intern_str("sys").to_owned(), + 1, + None, + vm.ctx.new_dict().into(), + )); + } + } + Err(_) => { + // Cannot get __main__ globals, skip warning + return Ok(( + vm.ctx.intern_str("sys").to_owned(), + 1, + None, + vm.ctx.new_dict().into(), + )); + } + } }; let registry = if let Ok(registry) = globals.get_item(__warningregistry__, vm) {