diff --git a/GLOSSARY.md b/GLOSSARY.md index 2ccafd6..a58bda1 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -8,7 +8,7 @@ This file is generated from `pyxray/src/pyxray/glossary.py`. Edit that and run ` ## Index -[ASDL](#asdl) | [Argument Clinic](#argument-clinic) | [C API](#c-api) | [C stack](#c-stack) | [C3 linearization](#c3-linearization) | [DEBUG_SAVEALL](#debug_saveall) | [DISABLE](#disable) | [EXTENDED_ARG](#extended_arg) | [GC pre header](#gc-pre-header) | [GIL](#gil) | [JIT](#jit) | [MRO](#mro) | [PEG parser](#peg-parser) | [PYTHONMALLOC](#pythonmalloc) | [PyABIInfo](#pyabiinfo) | [PyAPI_FUNC](#pyapi_func) | [PyVarObject](#pyvarobject) | [Pyodide](#pyodide) | [SOABI](#soabi) | [WebAssembly](#webassembly) | [abi flags](#abi-flags) | [abstract interpreter](#abstract-interpreter) | [abstract syntax tree](#abstract-syntax-tree) | [adaptive counter](#adaptive-counter) | [adaptive instruction](#adaptive-instruction) | [allocator domain](#allocator-domain) | [arena](#arena) | [assembler](#assembler) | [asynchronous exception](#asynchronous-exception) | [atexit callback](#atexit-callback) | [attach and detach](#attach-and-detach) | [backtrace](#backtrace) | [basic block](#basic-block) | [biased reference counting](#biased-reference-counting) | [binding](#binding) | [block](#block) | [blurb](#blurb) | [borrowed reference](#borrowed-reference) | [bound method](#bound-method) | [bytecode](#bytecode) | [cached hash](#cached-hash) | [calling convention](#calling-convention) | [cases generator](#cases-generator) | [cell](#cell) | [class cell](#class-cell) | [closure](#closure) | [code generation](#code-generation) | [code object](#code-object) | [code point](#code-point) | [code unit](#code-unit) | [coding cookie](#coding-cookie) | [cold block](#cold-block) | [collection threshold](#collection-threshold) | [compact dict](#compact-dict) | [compact int](#compact-int) | [compact string](#compact-string) | [computed goto](#computed-goto) | [configure](#configure) | [constant folding](#constant-folding) | [control flow graph](#control-flow-graph) | [copy and patch](#copy-and-patch) | [critical section](#critical-section) | [cross interpreter data](#cross-interpreter-data) | [cycle collector](#cycle-collector) | [daemon thread](#daemon-thread) | [data descriptor](#data-descriptor) | [data stack](#data-stack) | [deallocation](#deallocation) | [debug build](#debug-build) | [debug hooks](#debug-hooks) | [deferred reference counting](#deferred-reference-counting) | [deoptimization](#deoptimization) | [descriptor](#descriptor) | [devguide](#devguide) | [digit array](#digit-array) | [dispatch](#dispatch) | [dispatch table](#dispatch-table) | [environment changed](#environment-changed) | [eval breaker](#eval-breaker) | [eval loop](#eval-loop) | [evaluation order](#evaluation-order) | [exact type check](#exact-type-check) | [exception table](#exception-table) | [executor](#executor) | [extension module](#extension-module) | [f string](#f-string) | [finalized bit](#finalized-bit) | [finalizer](#finalizer) | [finalizing flag](#finalizing-flag) | [frame](#frame) | [frame object](#frame-object) | [free list](#free-list) | [free threaded build](#free-threaded-build) | [free variable](#free-variable) | [frozen module](#frozen-module) | [gdb](#gdb) | [generated file](#generated-file) | [generation](#generation) | [global import lock](#global-import-lock) | [grammar](#grammar) | [guard](#guard) | [header file](#header-file) | [heap type](#heap-type) | [immortal object](#immortal-object) | [immortalization](#immortalization) | [import bootstrap](#import-bootstrap) | [import placeholder](#import-placeholder) | [indent and dedent](#indent-and-dedent) | [inline cache](#inline-cache) | [inline values](#inline-values) | [instance dictionary](#instance-dictionary) | [instruction](#instruction) | [instruction DSL](#instruction-dsl) | [instruction pointer](#instruction-pointer) | [instrumented instruction](#instrumented-instruction) | [interning](#interning) | [interpreter finalisation](#interpreter-finalisation) | [interpreter pool](#interpreter-pool) | [interpreter state](#interpreter-state) | [lazy import](#lazy-import) | [left recursion](#left-recursion) | [limited API](#limited-api) | [line table](#line-table) | [loader state](#loader-state) | [magic number](#magic-number) | [mark alive pass](#mark-alive-pass) | [marshal](#marshal) | [meta path finder](#meta-path-finder) | [metaclass](#metaclass) | [micro operation](#micro-operation) | [mimalloc](#mimalloc) | [mimalloc heap](#mimalloc-heap) | [module alias](#module-alias) | [module lock](#module-lock) | [module spec](#module-spec) | [monitoring events](#monitoring-events) | [new reference](#new-reference) | [object](#object) | [object header](#object-header) | [obmalloc](#obmalloc) | [oparg](#oparg) | [opcode](#opcode) | [optimistic read](#optimistic-read) | [over allocation](#over-allocation) | [ownership](#ownership) | [parser generator](#parser-generator) | [path configuration](#path-configuration) | [path entry finder](#path-entry-finder) | [pdb](#pdb) | [pending call](#pending-call) | [per object lock](#per-object-lock) | [periodic check](#periodic-check) | [permanent generation](#permanent-generation) | [pointer](#pointer) | [pool](#pool) | [pool header](#pool-header) | [probe sequence](#probe-sequence) | [product type](#product-type) | [profile guided optimization](#profile-guided-optimization) | [pseudo instruction](#pseudo-instruction) | [pyc file](#pyc-file) | [pyconfig](#pyconfig) | [quickening](#quickening) | [race condition](#race-condition) | [reentrancy](#reentrancy) | [reference count](#reference-count) | [reference count contention](#reference-count-contention) | [reference cycle](#reference-cycle) | [reference leak](#reference-leak) | [referrer](#referrer) | [regen](#regen) | [regrtest](#regrtest) | [replacement field](#replacement-field) | [resource](#resource) | [resurrection](#resurrection) | [runtime state](#runtime-state) | [safe memory reclamation](#safe-memory-reclamation) | [safe path](#safe-path) | [scope](#scope) | [segment](#segment) | [segmentation fault](#segmentation-fault) | [shareable object](#shareable-object) | [short circuiting](#short-circuiting) | [side exit](#side-exit) | [signal handler](#signal-handler) | [single character cache](#single-character-cache) | [size class](#size-class) | [slot](#slot) | [slot wrapper](#slot-wrapper) | [small int cache](#small-int-cache) | [small integer cache](#small-integer-cache) | [soft keyword](#soft-keyword) | [specialization](#specialization) | [specialization family](#specialization-family) | [split table](#split-table) | [stable ABI](#stable-abi) | [stack depth](#stack-depth) | [stack effect](#stack-effect) | [stack reference](#stack-reference) | [stateless function](#stateless-function) | [static object](#static-object) | [static type](#static-type) | [stencil](#stencil) | [stolen reference](#stolen-reference) | [stop the world](#stop-the-world) | [string kind](#string-kind) | [struct](#struct) | [subinterpreter](#subinterpreter) | [sum type](#sum-type) | [switch interval](#switch-interval) | [symbol table](#symbol-table) | [symbol table pass](#symbol-table-pass) | [t string](#t-string) | [tagged integer](#tagged-integer) | [tagged pointer](#tagged-pointer) | [tail call](#tail-call) | [test case](#test-case) | [thread safety declaration](#thread-safety-declaration) | [thread state](#thread-state) | [tier one](#tier-one) | [tier two](#tier-two) | [token](#token) | [tokenizer](#tokenizer) | [tool id](#tool-id) | [trace](#trace) | [trace function](#trace-function) | [traceback](#traceback) | [transient GIL](#transient-gil) | [two phase initialisation](#two-phase-initialisation) | [type object](#type-object) | [underflow](#underflow) | [unwinding](#unwinding) | [value stack](#value-stack) | [varint](#varint) | [watcher](#watcher) | [weak reference](#weak-reference) | [weakref callback](#weakref-callback) | [weakref offset](#weakref-offset) | [zero cost exceptions](#zero-cost-exceptions) +[ASDL](#asdl) | [Argument Clinic](#argument-clinic) | [C API](#c-api) | [C stack](#c-stack) | [C3 linearization](#c3-linearization) | [DEBUG_SAVEALL](#debug_saveall) | [DISABLE](#disable) | [EXTENDED_ARG](#extended_arg) | [GC pre header](#gc-pre-header) | [GIL](#gil) | [JIT](#jit) | [MRO](#mro) | [PEG parser](#peg-parser) | [PYTHONMALLOC](#pythonmalloc) | [PyABIInfo](#pyabiinfo) | [PyAPI_FUNC](#pyapi_func) | [PyVarObject](#pyvarobject) | [Pyodide](#pyodide) | [SOABI](#soabi) | [WebAssembly](#webassembly) | [abi flags](#abi-flags) | [abstract interpreter](#abstract-interpreter) | [abstract syntax tree](#abstract-syntax-tree) | [adaptive counter](#adaptive-counter) | [adaptive instruction](#adaptive-instruction) | [allocator domain](#allocator-domain) | [arena](#arena) | [assembler](#assembler) | [asynchronous exception](#asynchronous-exception) | [atexit callback](#atexit-callback) | [attach and detach](#attach-and-detach) | [backtrace](#backtrace) | [basic block](#basic-block) | [biased reference counting](#biased-reference-counting) | [binding](#binding) | [block](#block) | [blurb](#blurb) | [borrowed reference](#borrowed-reference) | [bound method](#bound-method) | [bytecode](#bytecode) | [cached hash](#cached-hash) | [calling convention](#calling-convention) | [cases generator](#cases-generator) | [cell](#cell) | [class cell](#class-cell) | [clear function](#clear-function) | [closure](#closure) | [code generation](#code-generation) | [code object](#code-object) | [code point](#code-point) | [code unit](#code-unit) | [coding cookie](#coding-cookie) | [cold block](#cold-block) | [collection threshold](#collection-threshold) | [compact dict](#compact-dict) | [compact int](#compact-int) | [compact string](#compact-string) | [computed goto](#computed-goto) | [configure](#configure) | [constant folding](#constant-folding) | [control flow graph](#control-flow-graph) | [copy and patch](#copy-and-patch) | [critical section](#critical-section) | [cross interpreter data](#cross-interpreter-data) | [cycle collector](#cycle-collector) | [daemon thread](#daemon-thread) | [data descriptor](#data-descriptor) | [data stack](#data-stack) | [deallocation](#deallocation) | [debug build](#debug-build) | [debug hooks](#debug-hooks) | [deferred reference counting](#deferred-reference-counting) | [deoptimization](#deoptimization) | [descriptor](#descriptor) | [devguide](#devguide) | [digit array](#digit-array) | [dispatch](#dispatch) | [dispatch table](#dispatch-table) | [environment changed](#environment-changed) | [eval breaker](#eval-breaker) | [eval loop](#eval-loop) | [evaluation order](#evaluation-order) | [exact type check](#exact-type-check) | [exception table](#exception-table) | [executor](#executor) | [extension module](#extension-module) | [f string](#f-string) | [finalized bit](#finalized-bit) | [finalizer](#finalizer) | [finalizing flag](#finalizing-flag) | [frame](#frame) | [frame object](#frame-object) | [free list](#free-list) | [free threaded build](#free-threaded-build) | [free variable](#free-variable) | [frozen module](#frozen-module) | [gdb](#gdb) | [generated file](#generated-file) | [generation](#generation) | [global import lock](#global-import-lock) | [grammar](#grammar) | [guard](#guard) | [header file](#header-file) | [heap type](#heap-type) | [immortal object](#immortal-object) | [immortalization](#immortalization) | [import bootstrap](#import-bootstrap) | [import placeholder](#import-placeholder) | [indent and dedent](#indent-and-dedent) | [inline cache](#inline-cache) | [inline values](#inline-values) | [instance dictionary](#instance-dictionary) | [instruction](#instruction) | [instruction DSL](#instruction-dsl) | [instruction pointer](#instruction-pointer) | [instrumented instruction](#instrumented-instruction) | [interning](#interning) | [interpreter finalisation](#interpreter-finalisation) | [interpreter pool](#interpreter-pool) | [interpreter state](#interpreter-state) | [lazy import](#lazy-import) | [left recursion](#left-recursion) | [limited API](#limited-api) | [line table](#line-table) | [loader state](#loader-state) | [magic number](#magic-number) | [mark alive pass](#mark-alive-pass) | [marshal](#marshal) | [meta path finder](#meta-path-finder) | [metaclass](#metaclass) | [micro operation](#micro-operation) | [mimalloc](#mimalloc) | [mimalloc heap](#mimalloc-heap) | [module alias](#module-alias) | [module lock](#module-lock) | [module spec](#module-spec) | [module state](#module-state) | [monitoring events](#monitoring-events) | [new reference](#new-reference) | [object](#object) | [object header](#object-header) | [obmalloc](#obmalloc) | [oparg](#oparg) | [opcode](#opcode) | [optimistic read](#optimistic-read) | [over allocation](#over-allocation) | [ownership](#ownership) | [parser generator](#parser-generator) | [path configuration](#path-configuration) | [path entry finder](#path-entry-finder) | [pdb](#pdb) | [pending call](#pending-call) | [per object lock](#per-object-lock) | [periodic check](#periodic-check) | [permanent generation](#permanent-generation) | [pointer](#pointer) | [pool](#pool) | [pool header](#pool-header) | [probe sequence](#probe-sequence) | [product type](#product-type) | [profile guided optimization](#profile-guided-optimization) | [pseudo instruction](#pseudo-instruction) | [pyc file](#pyc-file) | [pyconfig](#pyconfig) | [quickening](#quickening) | [race condition](#race-condition) | [reentrancy](#reentrancy) | [reference count](#reference-count) | [reference count contention](#reference-count-contention) | [reference cycle](#reference-cycle) | [reference leak](#reference-leak) | [referrer](#referrer) | [regen](#regen) | [regrtest](#regrtest) | [replacement field](#replacement-field) | [resource](#resource) | [resurrection](#resurrection) | [runtime state](#runtime-state) | [safe memory reclamation](#safe-memory-reclamation) | [safe path](#safe-path) | [scope](#scope) | [segment](#segment) | [segmentation fault](#segmentation-fault) | [shareable object](#shareable-object) | [short circuiting](#short-circuiting) | [side exit](#side-exit) | [signal handler](#signal-handler) | [single character cache](#single-character-cache) | [size class](#size-class) | [slot](#slot) | [slot wrapper](#slot-wrapper) | [small int cache](#small-int-cache) | [small integer cache](#small-integer-cache) | [soft keyword](#soft-keyword) | [specialization](#specialization) | [specialization family](#specialization-family) | [split table](#split-table) | [stable ABI](#stable-abi) | [stack depth](#stack-depth) | [stack effect](#stack-effect) | [stack reference](#stack-reference) | [stateless function](#stateless-function) | [static object](#static-object) | [static type](#static-type) | [stencil](#stencil) | [stolen reference](#stolen-reference) | [stop the world](#stop-the-world) | [string kind](#string-kind) | [struct](#struct) | [subinterpreter](#subinterpreter) | [sum type](#sum-type) | [switch interval](#switch-interval) | [symbol table](#symbol-table) | [symbol table pass](#symbol-table-pass) | [t string](#t-string) | [tagged integer](#tagged-integer) | [tagged pointer](#tagged-pointer) | [tail call](#tail-call) | [test case](#test-case) | [thread safety declaration](#thread-safety-declaration) | [thread state](#thread-state) | [tier one](#tier-one) | [tier two](#tier-two) | [token](#token) | [tokenizer](#tokenizer) | [tool id](#tool-id) | [trace](#trace) | [trace function](#trace-function) | [traceback](#traceback) | [transient GIL](#transient-gil) | [traverse function](#traverse-function) | [two phase initialisation](#two-phase-initialisation) | [type object](#type-object) | [underflow](#underflow) | [unwinding](#unwinding) | [value stack](#value-stack) | [varint](#varint) | [watcher](#watcher) | [weak reference](#weak-reference) | [weakref callback](#weakref-callback) | [weakref offset](#weakref-offset) | [zero cost exceptions](#zero-cost-exceptions) ## Reading the source @@ -1374,6 +1374,22 @@ It is set before the finalizer runs rather than after, which is what guarantees Also written `gc.is_finalized`. First met in O14. See also [finalizer](#finalizer), [resurrection](#resurrection). In the source: [`Include/internal/pycore_gc.h:166-181@v3.15.0rc1#_PyGC_SET_FINALIZED`](https://github.com/python/cpython/blob/v3.15.0rc1/Include/internal/pycore_gc.h#L166-L181). +### traverse function + +**The slot where a container tells the collector what it is holding.** + +`tp_traverse` is handed a callback and calls it once for every object this one owns a reference to. The collector uses that to subtract the references living inside the set it is examining, so a field you forget to report looks exactly like a reference from somewhere outside and the whole cycle survives. A heap type has to report `Py_TYPE(self)` as well, since an instance holds a reference to its own type. + +Also written `tp_traverse`, `Py_VISIT`, `visitproc`. First met in R09. See also [cycle collector](#cycle-collector), [reference cycle](#reference-cycle), [heap type](#heap-type). In the source: [`Objects/typeobject.c:2607-2645@v3.15.0rc1#subtype_traverse`](https://github.com/python/cpython/blob/v3.15.0rc1/Objects/typeobject.c#L2607-L2645). + +### clear function + +**The slot the collector calls to break a cycle it has decided is garbage.** + +`tp_clear` drops the references `tp_traverse` reported, usually one `Py_CLEAR` per field. It is the only part of a collection that changes anything, and it is called across the whole unreachable set before any of it is freed, so an object still has to behave while the rest of its cycle is being emptied around it. Anything the object owns that is not a Python object belongs in `tp_dealloc` instead. + +Also written `tp_clear`, `Py_CLEAR`. First met in R09. See also [traverse function](#traverse-function), [cycle collector](#cycle-collector), [finalizer](#finalizer). In the source: [`Python/gc.c:1083-1120@v3.15.0rc1#delete_garbage`](https://github.com/python/cpython/blob/v3.15.0rc1/Python/gc.c#L1083-L1120). + ## Threads The words for what happens when more than one thread wants to run Python at the same time. C01 is about the one big lock, and C02 is about the many small ones that took its place. @@ -1834,6 +1850,14 @@ Everything about how it loads is decided in two steps. First the finder looks at Also written `.so`, `.pyd`. First met in R07. See also [stable ABI](#stable-abi), [SOABI](#soabi), [PyABIInfo](#pyabiinfo). In the source: [`Python/dynload_shlib.c:39-60@v3.15.0rc1#_PyImport_DynLoadFiletab`](https://github.com/python/cpython/blob/v3.15.0rc1/Python/dynload_shlib.c#L39-L60). +### module state + +**A struct hanging off a module object, one copy per module rather than per process.** + +A module definition with a positive `m_size` gets that many bytes allocated alongside every module object, reachable with `PyModule_GetState`. It is where an extension keeps what a C file would otherwise keep in statics, which matters because statics are shared by every interpreter in the process and module state is not. A type built from a spec can find its own module's state again through `PyType_GetModuleByDef`. + +Also written `m_size`, `PyModule_GetState`, `PyType_GetModuleByDef`. First met in R09. See also [extension module](#extension-module), [two phase initialisation](#two-phase-initialisation), [subinterpreter](#subinterpreter). In the source: [`Objects/moduleobject.c:1027-1035@v3.15.0rc1#PyModule_GetState`](https://github.com/python/cpython/blob/v3.15.0rc1/Objects/moduleobject.c#L1027-L1035). + ### SOABI **The tag a build stamps into the file names of the extensions it compiles.** diff --git a/README.md b/README.md index bb7f000..ae022b6 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,7 @@ A lesson can also end with a boss fight, which is a problem the text does not so | R06 | [The C API tiers](lessons/r06-the-c-api-tiers/r06.ipynb) | The C API is three directories and two macros. Include is open to any extension, Include/cpython needs Py_LIMITED_API to be undefined, and Include/internal starts nearly every file with three lines that stop the compiler unless you define Py_BUILD_CORE. More than half the header lines are in that third directory. Defining Py_LIMITED_API hides 186 of the 766 functions the public headers declare and all 974 in the other two, and the part that costs is not the functions but the struct layouts, because with no fields to read Py_TYPE becomes a call and Py_DECREF becomes a call to _Py_DecRef. The naming convention nearly matches the directories and the exceptions have a reason: a private name has to be exported when a public macro expands to it, which is what the 17 underscore names in the public tier are. None of it survives the build. Every tier resolves through ctypes.pythonapi, and calling _PyDict_SizeOf by hand gives the same number dict.__sizeof__ does, 16 bytes short of sys.getsizeof because that adds the collector header. Two recordings show the split is deliberate: inside the internal headers 93 percent of the names spelled PyAPI_FUNC resolve against 0.4 percent of the ones spelled plain extern, with 168 comments naming which bundled extension needs each export | M8 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r06-the-c-api-tiers/r06.ipynb) | | R07 | [The stable ABI](lessons/r07-the-stable-abi/r07.ipynb) | An extension has to get past two gates before it is a module, and both of them can be driven from Python. The first is the file name, read before anything is opened: the tag has to be one of the handful in a C array compiled into the interpreter, and a name that is wrong is not rejected but simply never looked at. Put empty files in a directory and ask the finder what it sees and the whole rule falls out, including that a free threaded build takes abi3t and refuses abi3, an ordinary 3.15 build takes both, and 3.14 has never heard of abi3t. The 3 in abi3 is not a Python version, it is PYTHON_ABI_VERSION, a counter that stopped moving in 2010, and sys.api_version has been 1013 since 2006. The second gate is PyABIInfo, twelve bytes new in 3.15, and because PyABIInfo_Check is a plain exported function the struct can be built in ctypes and handed to the real check, which refuses four of eight sample extensions and gives a different reason for each. The stable ABI itself is dated by the preprocessor gates around the declarations, 173 functions added since 3.2 with the busiest releases being the recent ones. Two recordings run both gates on a release build and on a build made with --disable-gil, where six tags become four and four refusals become six | M8 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r07-the-stable-abi/r07.ipynb) | | R08 | [When the interpreter stops](lessons/r08-when-the-interpreter-stops/r08.ipynb) | Shutdown is one C function read top to bottom, and every cell here watches it happen in a child interpreter because a notebook cannot watch its own ending. The first thing _Py_Finalize does is call into Python: threading._shutdown joins your non daemon threads, then the atexit callbacks run, and only then does the finalizing flag go up, which is why sys.is_finalizing() is False in a callback and True in a late __del__. Callbacks come back newest first because register inserts at the front of a list, and because that list is copied and then emptied, a callback registered from inside another one is a silent no op. Once teardown starts sys.meta_path is cleared, so a finalizer can still read its own module globals but any import raises ImportError and says why, and both a failing callback and a failing finalizer are printed and ignored while the exit status stays zero. The case worth knowing is the daemon thread: pass time.sleep and your finalizers run, pass a function of your own and its stack frame holds your module globals, the module dict is never cleared and nothing in it is freed. Two recordings run nine endings on a release build and on a debug build, where that one thread leaves 12680 references alive against zero for the same objects held any other way | M8 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r08-when-the-interpreter-stops/r08.ipynb) | +| R09 | [Writing a C extension properly](lessons/r09-writing-a-c-extension-properly/r09.ipynb) | Four things a C extension owes the runtime, each one broken on purpose and then fixed in front of you. The first is the error path: a function that packs a tuple and then returns NULL without dropping it leaks two references on every failed call, so a thousand failures leave two thousand references behind while every test still passes. The second is the collector: the same little container is compiled three ways, one with no GC flag, one with a tp_traverse that reports its type and forgets its field, and one that reports both, and only the third is ever freed out of a cycle, because the collector subtracts a reference for every visit landing inside its candidate set and treats whatever is still above zero as held from outside. The third is teardown, where tp_finalize runs at most once whether the object died on its count or inside a cycle, tp_clear only breaks links and tp_dealloc frees. The fourth is the single line at the bottom of the file, Py_MOD_GIL_USED or Py_MOD_GIL_NOT_USED, which a free threaded build reads and acts on. Two recordings close it: CPython's own leak hunter run over four tests that all pass an ordinary run and two of which it fails, and one shared object loaded three ways where exactly one of the loads turns the lock back on and says so | M8 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r09-writing-a-c-extension-properly/r09.ipynb) | More are landing in order. [lessons/README.md](lessons/README.md) explains how one is put together and how to run them locally. diff --git a/citations.lock.json b/citations.lock.json index d8acbdd..d8e8fcd 100644 --- a/citations.lock.json +++ b/citations.lock.json @@ -1060,6 +1060,11 @@ "first_line": "", "lines": 6 }, + "Include/object.h:520-530@v3.15.0rc1": { + "digest": "bf1a330c8709fac8", + "first_line": "/* Set while the type is being 'readied', to prevent recursive ready calls */", + "lines": 11 + }, "Include/object.h:580-583@v3.15.0rc1": { "digest": "323df69ccd90ebf9", "first_line": "// Flag values for ob_flags (16 bits available, if SIZEOF_VOID_P > 4).", @@ -1155,6 +1160,11 @@ "first_line": "#define _Py_IMMORTAL_INITIAL_REFCNT (3ULL << 30)", "lines": 4 }, + "Include/refcount.h:483-500@v3.15.0rc1": { + "digest": "54d3f1d1d6989131", + "first_line": "#define Py_CLEAR(op) \\", + "lines": 18 + }, "Include/refcount.h:527-538@v3.15.0rc1": { "digest": "4b12e8e078a88f82", "first_line": "// Create a new strong reference to an object:", @@ -1515,6 +1525,11 @@ "first_line": "def check_rc_deltas(deltas):", "lines": 14 }, + "Lib/test/libregrtest/refleak.py:214-236@v3.15.0rc1": { + "digest": "227d5cb7c44aee81", + "first_line": "failed = False", + "lines": 23 + }, "Lib/test/libregrtest/results.py:18-22@v3.15.0rc1": { "digest": "2e112c114c5a2a06", "first_line": "EXITCODE_BAD_TEST = 2", @@ -2190,6 +2205,11 @@ "first_line": "/* For int multiplication, use the O(N**2) school algorithm unless", "lines": 6 }, + "Objects/moduleobject.c:1027-1035@v3.15.0rc1": { + "digest": "f6011b09ee3ae4fa", + "first_line": "void*", + "lines": 9 + }, "Objects/moduleobject.c:1330-1360@v3.15.0rc1": { "digest": "b890e50fe279a649", "first_line": "PyObject*", @@ -2275,6 +2295,11 @@ "first_line": "int", "lines": 37 }, + "Objects/object.c:595-630@v3.15.0rc1": { + "digest": "d74c897f83ac2a9b", + "first_line": "PyObject_CallFinalizerFromDealloc(PyObject *self)", + "lines": 36 + }, "Objects/obmalloc.c:1396-1427@v3.15.0rc1": { "digest": "8bbbd57375b14355", "first_line": "", @@ -2485,6 +2510,11 @@ "first_line": "_PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems)", "lines": 20 }, + "Objects/typeobject.c:2607-2645@v3.15.0rc1": { + "digest": "b894ed89d3d54d12", + "first_line": "subtype_traverse(PyObject *self, visitproc visit, void *arg)", + "lines": 39 + }, "Objects/typeobject.c:2794-2815@v3.15.0rc1": { "digest": "66d90d9277fe53a4", "first_line": "has_finalizer = type->tp_finalize || type->tp_del;", @@ -2540,6 +2570,11 @@ "first_line": "if (ctx->add_weak) {", "lines": 5 }, + "Objects/typeobject.c:5973-6005@v3.15.0rc1": { + "digest": "e5c739ab6a2a7086", + "first_line": "PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def)", + "lines": 33 + }, "Objects/typeobject.c:6144-6180@v3.15.0rc1": { "digest": "a16a8c9e914d75f2", "first_line": "static int", @@ -4380,6 +4415,11 @@ "first_line": "static void", "lines": 29 }, + "Python/gc.c:1083-1120@v3.15.0rc1": { + "digest": "7663d57dedc7bbab", + "first_line": "delete_garbage(PyThreadState *tstate, GCState *gcstate,", + "lines": 38 + }, "Python/gc.c:1221-1236@v3.15.0rc1": { "digest": "dc27d9361032699d", "first_line": "/* Handle objects that may have resurrected after a call to 'finalize_garbage', moving", @@ -4450,6 +4490,11 @@ "first_line": "void", "lines": 21 }, + "Python/gc.c:2026-2044@v3.15.0rc1": { + "digest": "a6ecf3b2252cc226", + "first_line": "_PyObject_GC_New(PyTypeObject *tp)", + "lines": 19 + }, "Python/gc.c:2118-2123@v3.15.0rc1": { "digest": "35669478a7a4e47e", "first_line": "GCState *gcstate = get_gc_state();", @@ -4460,11 +4505,21 @@ "first_line": "/* Set all gc_refs = ob_refcnt. After this, gc_refs is > 0 and", "lines": 20 }, + "Python/gc.c:438-465@v3.15.0rc1": { + "digest": "5a6bfa74ee2f5f63", + "first_line": "/* A traversal callback for subtract_refs. */", + "lines": 28 + }, "Python/gc.c:485-501@v3.15.0rc1": { "digest": "2e571c593e6b01da", "first_line": "/* Subtract internal references from gc_refs. After this, gc_refs is >= 0", "lines": 17 }, + "Python/gc.c:490-515@v3.15.0rc1": { + "digest": "fd15d95adf539c91", + "first_line": "subtract_refs(PyGC_Head *containers)", + "lines": 26 + }, "Python/gc.c:566-583@v3.15.0rc1": { "digest": "ec0b2854fab4f304", "first_line": "/* Move the unreachable objects from young to unreachable. After this,", diff --git a/experiments/README.md b/experiments/README.md index a0ff7f1..c2670f4 100644 --- a/experiments/README.md +++ b/experiments/README.md @@ -53,6 +53,8 @@ So those programs run somewhere else. They run in the images this project publis | [r07-what-a-build-will-load-without-the-lock](tier1/r07-what-a-build-will-load-without-the-lock.md) | R07 | freethreaded | How much of the stable ABI does dropping the global interpreter lock rule out? | | [r08-what-the-end-still-runs](tier1/r08-what-the-end-still-runs.md) | R08 | release | Which of the things you registered actually run when the interpreter stops? | | [r08-what-the-end-leaves-behind](tier1/r08-what-the-end-leaves-behind.md) | R08 | debug | How much does one daemon thread leave stranded when the interpreter stops? | +| [r09-what-the-leak-hunter-catches](tier1/r09-what-the-leak-hunter-catches.md) | R09 | debug | What does CPython's own leak hunter see that an ordinary test run walks straight past? | +| [r09-what-a-module-must-declare](tier1/r09-what-a-module-must-declare.md) | R09 | freethreaded | What is the line at the bottom of an extension module that declares itself safe worth? | ## The commands diff --git a/experiments/tier1/r09-what-a-module-must-declare.md b/experiments/tier1/r09-what-a-module-must-declare.md new file mode 100644 index 0000000..607cadd --- /dev/null +++ b/experiments/tier1/r09-what-a-module-must-declare.md @@ -0,0 +1,113 @@ +# One shared object, three ways to load it, and what each one does to the lock + +Generated by `just build-tier1`. Do not edit by hand, the change will be overwritten. + +What is the line at the bottom of an extension module that declares itself safe worth? + +- Lesson: R09 +- Build: freethreaded +- Image: ghcr.io/tamnd/cpython-internals/cpython:freethreaded@sha256:db72284e3a49f43c38b96bec2baed1380b8348e27ea6f54f6e8d0810b59c3144 +- Interpreter: 3.15.0rc1 free-threading build (37e98da:37e98da, Aug 29 2026, 09:25:00) [GCC 14.2.0] +- Recorded: 2026-09-06 + +Why this needs the freethreaded build: it needs a free threaded build, because a build that always has the lock has nothing to turn back on. + +## The program + +```python +"""What a module has to declare before a free threaded build will trust it. + +CPython's own test suite ships a shared object that exports several init functions, two +of which differ only in one line: whether they say the module can run without the lock. +Loading each one in a child of its own shows what that line is worth. +""" + +import os +import subprocess +import sys +import textwrap + +CHILD = """ +import importlib.machinery, importlib.util, sys, _testmultiphase + +name = sys.argv[1] +print(f" the lock before the import: {sys._is_gil_enabled()}") +loader = importlib.machinery.ExtensionFileLoader(name, _testmultiphase.__file__) +spec = importlib.util.spec_from_loader(name, loader) +loader.exec_module(importlib.util.module_from_spec(spec)) +print(f" the lock after the import: {sys._is_gil_enabled()}") +""" + +CASES = ( + ("_test_from_modexport", "which declares Py_MOD_GIL_NOT_USED", {}), + ("_test_from_modexport_gil_used", "which declares Py_MOD_GIL_USED", {}), + ("_test_from_modexport_gil_used", "the same one, in a child started with PYTHON_GIL=0", {}), +) + +OVERRIDE = {"PYTHON_GIL": "0"} + + +def load(name, extra): + """Import one init function out of that shared object, in a child of its own.""" + return subprocess.run( + [sys.executable, "-c", CHILD, name], + capture_output=True, + text=True, + timeout=180, + env=os.environ | {"PYTHON_COLORS": "0"} | extra, + ) + + +print("what a compiled module has to declare before this build will trust it") +print() +print(f" the abi flags on this build: {sys.abiflags!r}") +print() + +turned_on = 0 +for position, (name, what, _) in enumerate(CASES): + done = load(name, OVERRIDE if position == 2 else {}) + print(f" {name}, {what}") + print(done.stdout, end="") + turned_on += "the lock after the import: True" in done.stdout + said = [one for one in done.stderr.splitlines() if "RuntimeWarning" in one] + if not said: + print(" it went through without a word") + for one in said: + print(" on the way it warned, at some length:") + text = one.split("RuntimeWarning: ")[1] + print(textwrap.indent(textwrap.fill(text, 74), " ")) + print() + +print(f"~ ways of loading the same shared object: {len(CASES)}") +print(f"~ of those that turned the lock back on: {turned_on}") +``` + +## What it printed + +```text +what a compiled module has to declare before this build will trust it + + the abi flags on this build: 't' + + _test_from_modexport, which declares Py_MOD_GIL_NOT_USED + the lock before the import: False + the lock after the import: False + it went through without a word + + _test_from_modexport_gil_used, which declares Py_MOD_GIL_USED + the lock before the import: False + the lock after the import: True + on the way it warned, at some length: + The global interpreter lock (GIL) has been enabled to load module + '_test_from_modexport_gil_used', which has not declared that it can run + safely without the GIL. To override this behavior and keep the GIL + disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0. + + _test_from_modexport_gil_used, the same one, in a child started with PYTHON_GIL=0 + the lock before the import: False + the lock after the import: False + it went through without a word + +~ ways of loading the same shared object: 3 +~ of those that turned the lock back on: 1 +``` diff --git a/experiments/tier1/r09-what-the-leak-hunter-catches.md b/experiments/tier1/r09-what-the-leak-hunter-catches.md new file mode 100644 index 0000000..d67d664 --- /dev/null +++ b/experiments/tier1/r09-what-the-leak-hunter-catches.md @@ -0,0 +1,144 @@ +# Four small tests, run once the ordinary way and once with the leak hunter watching + +Generated by `just build-tier1`. Do not edit by hand, the change will be overwritten. + +What does CPython's own leak hunter see that an ordinary test run walks straight past? + +- Lesson: R09 +- Build: debug +- Image: ghcr.io/tamnd/cpython-internals/cpython:debug@sha256:7baea8f3dd4de2e4c3b020543729b147e636494ae9758dabffb4675793e37170 +- Interpreter: 3.15.0rc1 (37e98da:37e98da, Aug 29 2026, 09:24:21) [GCC 14.2.0] +- Recorded: 2026-09-06 + +Why this needs the debug build: it needs a debug build, because counting what the interpreter is still holding is something only that build keeps a total of. + +## The program + +```python +"""What the reference leak hunter catches that an ordinary run does not. + +Four small tests, written into a directory of their own, run twice each. Once the way +anybody runs a test suite, and once under the flag CPython's buildbots use, which runs +every test six times and counts what the interpreter is still holding afterwards. +""" + +import os +import pathlib +import re +import subprocess +import sys +import tempfile + +CASES = { + "test_clean": """ +import unittest + + +class Clean(unittest.TestCase): + def test_it(self): + held = [object()] + self.assertTrue(held) +""", + "test_cached": """ +import unittest + +CACHE = [] + + +class Cached(unittest.TestCase): + def test_it(self): + if len(CACHE) < 4: + CACHE.append(object()) + self.assertTrue(CACHE) +""", + "test_leaky": """ +import unittest + +HOARD = [] + + +class Leaky(unittest.TestCase): + def test_it(self): + HOARD.append(object()) + self.assertTrue(HOARD) +""", + "test_handles": """ +import os +import unittest + + +class Handles(unittest.TestCase): + def test_it(self): + copy = os.dup(0) + self.assertGreater(copy, 0) +""", +} + +REPORT = re.compile(r"^\w+ leaked (\[[^]]*\]) ([a-z ]+), sum=(-?\d+)(.*)$") + +ROOT = pathlib.Path(tempfile.mkdtemp()) +for name, body in CASES.items(): + (ROOT / f"{name}.py").write_text(body) + + +def run(name, flags=()): + """Run one of the four tests, with or without the leak hunting flag.""" + return subprocess.run( + [sys.executable, "-m", "test", "--testdir", str(ROOT), *flags, name], + capture_output=True, + text=True, + timeout=600, + env=os.environ | {"PYTHON_COLORS": "0"}, + ) + + +print("four small tests, run the way anybody runs a test suite") +print() +ordinary = 0 +for name in CASES: + done = run(name) + ordinary += done.returncode == 0 + print(f" {name:14} {'failed' if done.returncode else 'passed'}") + +print() +print("the same four, run six times each with the leak hunter watching") +print() +hunted = 0 +for name in CASES: + done = run(name, ("-R", "3:3")) + hunted += done.returncode == 0 + said = [m for m in map(REPORT.match, done.stderr.splitlines()) if m] + counts = ", ".join(f"{m.group(1)} {m.group(2)}" for m in said) + excused = " (which the hunter calls fine)" if said and said[0].group(4) else "" + verdict = "failed" if done.returncode else "passed" + print(f" {name:14} {verdict:7} {counts or 'nothing was left behind'}{excused}") + +print() +print(f"~ tests that pass an ordinary run: {ordinary} of {len(CASES)}") +print(f"~ tests that pass the leak hunter: {hunted} of {len(CASES)}") +print("~ repetitions the hunter ran each test for: 6") +print("~ of those repetitions that were warm ups: 3") +``` + +## What it printed + +```text +four small tests, run the way anybody runs a test suite + + test_clean passed + test_cached passed + test_leaky passed + test_handles passed + +the same four, run six times each with the leak hunter watching + + test_clean passed nothing was left behind + test_cached passed [1, 0, 0] references, [1, 0, 0] memory blocks (which the hunter calls fine) + test_leaky failed [1, 1, 1] references, [1, 1, 1] memory blocks + test_handles failed [1, 1, 1] file descriptors + +~ tests that pass an ordinary run: 4 of 4 +~ tests that pass the leak hunter: 2 of 4 +~ repetitions the hunter ran each test for: 6 +~ of those repetitions that were warm ups: 3 +``` diff --git a/lessons/CLAIMS.md b/lessons/CLAIMS.md index b78f8ac..b000dfc 100644 --- a/lessons/CLAIMS.md +++ b/lessons/CLAIMS.md @@ -12,7 +12,7 @@ header, what the allocator does with a freed block, the shape of the eval loop. marked with the reason, and a lesson is allowed at most 3 of them. The cap is the point. Without it the exception becomes the rule and this goes back to being a book. -625 claims across 79 lessons, 59 of them not observable from Python. +630 claims across 80 lessons, 60 of them not observable from Python. ## B01. Building CPython, and whether you need to @@ -820,6 +820,16 @@ Without it the exception becomes the rule and this goes back to being a book. | One full collection of generation two runs during shutdown, and the finalizing flag is already set when it does. | [`r08-18`](r08-when-the-interpreter-stops/r08.ipynb) | | A subinterpreter you never closed is finalised for you, with a RuntimeWarning, before the main interpreter finishes. | [`r08-20`](r08-when-the-interpreter-stops/r08.ipynb) | +## R09. Writing a C extension properly + +| Claim | Proved by | +| --- | --- | +| Missing one Py_DECREF on an error branch costs two references per failed call, and nothing in the process complains | [`r09-07`](r09-writing-a-c-extension-properly/r09.ipynb) | +| The GC flag on its own is not enough. A tp_traverse that forgets a field leaks just as completely as no tp_traverse at all | [`r09-13`](r09-writing-a-c-extension-properly/r09.ipynb) | +| A tp_finalize runs exactly once per object whether the object died on a reference count or inside a cycle | [`r09-15`](r09-writing-a-c-extension-properly/r09.ipynb) | +| One imported module that has not declared itself safe turns the lock back on for the whole process | not observable from Python: the flip only happens on an interpreter built with --disable-gil, and this notebook is almost certainly not one | +| An ordinary test run cannot see a reference leak at all, which is why CPython runs its own suite a second way | [`r09-17`](r09-writing-a-c-extension-properly/r09.ipynb) | + ## T01. One line, seven stages | Claim | Proved by | diff --git a/lessons/README.md b/lessons/README.md index 3d6ae77..7b3ca8c 100644 --- a/lessons/README.md +++ b/lessons/README.md @@ -83,6 +83,7 @@ Each lesson is a notebook you can run. There is nothing to install and nothing t | [R06. The C API tiers](r06-the-c-api-tiers/r06.ipynb) | The three include directories measured file by file and line by line, the Py_LIMITED_API guard walked as a preprocessor stack to count what a limited build may call and what it may not, the three line #error that guards 140 of the 148 internal headers printed as it appears in the file, the three naming conventions cross tabulated against the three directories, eight names from all three tiers handed to ctypes.pythonapi to show that every one of them resolves, the internal headers split by their two spellings so the deliberate half of the export shows up, _PyDict_SizeOf called by hand and checked against the dunder and against sys.getsizeof, seven names asked whether they are a macro or an exported function or both, and every shared object on sys.path sorted by what its file name says it was built against | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r06-the-c-api-tiers/r06.ipynb) | | [R07. The stable ABI](r07-the-stable-abi/r07.ipynb) | The six file name tags this build will load an extension from and the one it stamps on its own output, the SOABI tag taken apart into implementation and version and threading and platform, five candidate file names each dropped into a directory of its own so the finder can say which it would even look at, all of them plus a plain demo.py in one directory to show which wins and that a compiled file shadows a Python one, the PyABIInfo struct rebuilt field by field in ctypes and measured at twelve bytes, eight hypothetical extensions handed to the real PyABIInfo_Check so the four refusals print with CPython's own wording, every public header walked as a preprocessor stack to date each of the 173 gated functions by the release that added it, four named functions with the file and line and stable ABI version read straight out of the headers, and both gates run again in containers on a release build and a free threaded one | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r07-the-stable-abi/r07.ipynb) | | [R08. When the interpreter stops](r08-when-the-interpreter-stops/r08.ipynb) | One child interpreter printing your last line, a thread finishing, two atexit callbacks and a finalizer in the order they actually happen with sys.is_finalizing() next to each, three callbacks registered in order to show the list comes back reversed and a fourth registered during shutdown that never runs, a __del__ that reports what it can still reach and finds its own module globals intact, sys.modules empty and an import of an already imported module raising, an atexit callback and a finalizer that both raise so you can read CPython's own ignored exception wording and see the exit status stay at zero, the same daemon thread started two ways so one run frees everything and the other frees nothing, gc.callbacks catching the single generation two collection that runs after the flag is already set, a subinterpreter left open so the RuntimeWarning and its own atexit callback both turn up, and the whole set of nine endings recorded again on a release build and on a debug build that counts what was left alive | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r08-when-the-interpreter-stops/r08.ipynb) | +| [R09. Writing a C extension properly](r09-writing-a-c-extension-properly/r09.ipynb) | A leaky C function and a careful one compiled side by side so that a thousand failing calls leave two thousand references behind in one and none in the other, a 154 line module holding the same box type written three ways, a table putting gc.get_referents next to whether the collector ever frees that box out of a cycle, the whole short list of what a tp_traverse handler is allowed to call including the _DuringGC family that is new in 3.15, a tp_finalize that reports itself exactly once whether the object died on its count or in a cycle, a miniature version of CPython's -R flag written in a dozen lines that catches both broken boxes and clears the correct one, a recording of the real leak hunter failing two of four tests that an ordinary run passes, and a recording of one shared object loaded three ways on a free threaded build where only the middle load turns the lock back on | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r09-writing-a-c-extension-properly/r09.ipynb) | ## The three programs diff --git a/lessons/r09-writing-a-c-extension-properly/build.py b/lessons/r09-writing-a-c-extension-properly/build.py new file mode 100644 index 0000000..008ded4 --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/build.py @@ -0,0 +1,707 @@ +#!/usr/bin/env python +"""R09. Writing a C extension properly. + +The last lesson of the runtime part, and the one that turns the book around. Everything up to +here watched CPython from Python. This one writes the C on the other side of the boundary and +finds out what the runtime expects from it. + +Two small extension modules, compiled in the notebook where there is a compiler. The first is +one function written twice, differing by a single `Py_DECREF` on a branch that only runs on bad +input. The second is a container written three ways, one that never tells the collector it +exists, one that tells it and then lies about what it holds, and one that gets it right. + +The two Tier 1 recordings are the parts a notebook cannot do. A debug build runs CPython's own +reference leak hunter over four tests that all pass an ordinary run, and a free threaded build +loads the same shared object three ways to show what one declaration at the bottom of the file +is worth. + +Run this file to regenerate the notebook, or `just build-lessons` to regenerate all of them. +`just lessons` checks that the committed notebook still matches this file. +""" + +from nbbuild import BANNER, Lesson +from nbdiagram import Diagrams +from tier1 import show as recording + +lesson = Lesson("r09-writing-a-c-extension-properly", "r09") +badge = lesson.badge +cite = lesson.cite +term = lesson.term +figure = Diagrams("r09-writing-a-c-extension-properly").figure + +ON_DEBUG = "r09-what-the-leak-hunter-catches" +ON_FREETHREADED = "r09-what-a-module-must-declare" + + +lesson.md(f""" +# R09. Writing a C extension properly + +{badge} + +Every lesson so far has watched CPython from the Python side. This one crosses over and writes the C, which is where all the rules you have been reading about stop being descriptions and start being your job. + +There are only four of them, and none is hard. Hand back what you own and let go of what you borrowed. If your object can hold another object, tell the collector. Do your cleanup in the slot meant for it. Say honestly whether your code is safe without the lock. What makes them worth a lesson is that breaking any of the four produces no error, no warning and no crash. It produces a program that works, and slowly stops working, in a way that only a tool built for the purpose can see. + +{figure("the-branch-nobody-tests", "two error paths side by side, one missing a single Py_DECREF")} +""") + + +lesson.md(""" +## About the source references + +Now and then this lesson points at CPython's own source, like this: `Python/gc.c:490-515@v3.15.0rc1`. + +Read it as three parts: the file, the lines, and the release those line numbers belong to. Sometimes there is a fourth part after a `#`, which is the name of the thing those lines are inside. + +Every reference is a link, and every one is checked against the pinned source on each change, so a stale reference fails the build instead of sending you somewhere wrong. You never have to read any of it. The references are there so you can go deeper when you want to, and so you can check that this lesson is not making things up. + +## Setup + +Colab does not come with the small package these lessons use, so the next cell installs it. If you are running this from a checkout of the repository it is already installed and the cell does nothing. +""") + + +lesson.code(""" +import sys + +if sys.version_info < (3, 14): + print("This lesson needs CPython 3.14 or newer.") + print(f"This runtime is {sys.version.split()[0]}, and the cells below will not run on it.") +else: + try: + import pyxray + except ImportError: + %pip install -q "pyxray @ git+https://github.com/tamnd/cpython-internals@main#subdirectory=pyxray" + import pyxray +""") + + +lesson.md(""" +## Which Python is this + +This lesson compiles C. Colab has a compiler and so does any machine with a working development setup, and a browser tab has neither a compiler nor the headers, so the cells check first and say so rather than failing. If you are reading this in a browser, everything below still reads, and the two recordings at the end are real output from real builds. + +## Which interpreter is this +""") + + +lesson.code( + """ +import pyxray + +pyxray.show() +""", + differs=BANNER, + quiet=True, +) + + +lesson.md(f""" +## The branch nobody tests + +Start with the smallest of the four rules, because everything else is a version of it. + +Every C API function that hands you an object hands you one of two things. A {term("new reference")} is yours, and you owe the runtime a `Py_DECREF` before you lose track of it. A {term("borrowed reference")} is not yours, and decrefing it is a bug of the other kind. The documentation says which for every function, and there is no way to tell by looking at the call. + +Here is a function that gets it right on the path you test and wrong on the path you do not. It packs its argument into a tuple twice over and then hashes the tuple, which works for a number and fails for a list. `PyTuple_Pack` returns a new reference, and that tuple is holding two references to your argument. On the way out through the error the tuple is simply dropped. + +{lesson.claim("Missing one Py_DECREF on an error branch costs two references per failed call, and nothing in the process complains")} +""") + + +lesson.code(""" +import os +import subprocess +import sysconfig +import tempfile + +ERRPATH = \"\"\" +#include + +/* pair(x) hands back (x, x), but only if that tuple can be hashed. */ + +static PyObject * +leaky(PyObject *module, PyObject *arg) +{ + PyObject *pair = PyTuple_Pack(2, arg, arg); + if (pair == NULL) { + return NULL; + } + if (PyObject_Hash(pair) == -1) { + return NULL; + } + return pair; +} + +static PyObject * +careful(PyObject *module, PyObject *arg) +{ + PyObject *pair = PyTuple_Pack(2, arg, arg); + if (pair == NULL) { + return NULL; + } + if (PyObject_Hash(pair) == -1) { + Py_DECREF(pair); + return NULL; + } + return pair; +} + +static PyMethodDef methods[] = { + {"leaky", leaky, METH_O, "Build a pair, and get the error path wrong."}, + {"careful", careful, METH_O, "Build a pair, and get the error path right."}, + {NULL, NULL, 0, NULL}, +}; + +static PyModuleDef_Slot slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static struct PyModuleDef errpath = { + PyModuleDef_HEAD_INIT, "errpath", NULL, 0, methods, slots, NULL, NULL, NULL, +}; + +PyMODINIT_FUNC +PyInit_errpath(void) +{ + return PyModuleDef_Init(&errpath); +} +\"\"\" + +SUFFIX = sysconfig.get_config_var("EXT_SUFFIX") or ".so" +WORK = tempfile.mkdtemp() +sys.path.insert(0, WORK) +NOTHING = " no compiler and headers here, so there is nothing to build" + + +def build(name, source): + \"\"\"Compile one C file into something this interpreter can import. Empty means it worked.\"\"\" + written = os.path.join(WORK, name + ".c") + with open(written, "w") as handle: + handle.write(source) + line = [*(sysconfig.get_config_var("CC") or "cc").split(), "-shared", "-fPIC"] + if sys.platform == "darwin": + line += ["-undefined", "dynamic_lookup"] + line += ["-I", sysconfig.get_path("include"), written] + line += ["-o", os.path.join(WORK, name + SUFFIX)] + try: + done = subprocess.run(line, capture_output=True, text=True, timeout=300) + except OSError: + return "there is no C compiler on this runtime" + return "" if done.returncode == 0 else done.stderr.strip().splitlines()[-1] + + +TROUBLE = build("errpath", ERRPATH) +HAVE_C = not TROUBLE +if not HAVE_C: + print(NOTHING) +else: + import errpath + + for name in ("leaky", "careful"): + work = getattr(errpath, name) + victim = [] + before = sys.getrefcount(victim) + failed = 0 + for _ in range(1000): + try: + work(victim) + except TypeError: + failed += 1 + left = sys.getrefcount(victim) - before + print(f" {name:8} {failed} calls raised, and left {left} references behind") +""") + + +lesson.md(f""" +Two thousand references to a list nobody can reach any more, and the process is perfectly happy. It will stay happy until the machine runs out of memory, which on a web server handling bad input all day is a Tuesday. + +The fix is one line, and the habit that produces it is worth more than the line. C has no `finally`, so CPython's own source uses a single exit: every function that owns something has one cleanup block at the bottom, and every failure jumps to it with `goto error`. Once a function has more than one owned object that is the only shape that stays correct, because the alternative is repeating the right sequence of decrefs at every `return NULL` and getting one of them wrong. `Py_CLEAR` {cite("Include/refcount.h:483-500@v3.15.0rc1#Py_CLEAR")} exists for the same reason: it sets the field to `NULL` before dropping the reference, so nothing can see a half freed pointer if the deallocation runs code that comes back around. + +## A box the collector cannot see + +Now the second rule, which needs a container to show. + +The {term("cycle collector")} finds garbage that reference counting cannot: a group of objects pointing at each other with nothing pointing in from outside. It does that by asking every candidate what it is holding, subtracting those references from the counts, and seeing what is left {cite("Python/gc.c:490-515@v3.15.0rc1#subtract_refs")} {cite("Python/gc.c:438-465@v3.15.0rc1#visit_decref")}. + +{figure("how-the-collector-uses-traverse", "four steps from counting references to deciding what is held from outside")} + +Asking is the {term("traverse function")} slot, `tp_traverse`, and there is no way for the runtime to check your answer. Report too little and the collector concludes your object is held from outside and leaves the whole cycle alone. That is the failure mode: not a crash, a leak. + +Two lines of C decide whether any of that machinery applies to your type at all. `Py_TPFLAGS_HAVE_GC` {cite("Include/object.h:520-530@v3.15.0rc1#Py_TPFLAGS_HAVE_GC")} in the flags, and allocation through `PyObject_GC_New` {cite("Python/gc.c:2026-2044@v3.15.0rc1#_PyObject_GC_New")}, which puts a small header in front of your object so the collector has somewhere to keep its bookkeeping. + +Here is one container, a box holding a single object, written three ways. The next cell is only the types. +""") + + +lesson.code(""" +TYPES = \"\"\" +#include +#include + +typedef struct { + PyObject_HEAD + PyObject *item; +} BoxObject; + +typedef struct { + PyObject *loose; + PyObject *half; + PyObject *tracked; + long freed; + long finalized; +} boxes_state; + +static struct PyModuleDef boxes_module; + +static boxes_state * +state_of(PyObject *self) +{ + PyObject *module = PyType_GetModuleByDef(Py_TYPE(self), &boxes_module); + return module == NULL ? NULL : (boxes_state *)PyModule_GetState(module); +} + +static PyObject * +box_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + int tracked = PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC); + BoxObject *self = tracked ? PyObject_GC_New(BoxObject, type) + : PyObject_New(BoxObject, type); + if (self == NULL) { + return NULL; + } + self->item = NULL; + if (tracked) { + PyObject_GC_Track(self); + } + return (PyObject *)self; +} + +static void +loose_dealloc(BoxObject *self) +{ + PyTypeObject *type = Py_TYPE(self); + boxes_state *state = state_of((PyObject *)self); + if (state != NULL) { + state->freed++; + } + Py_CLEAR(self->item); + type->tp_free((PyObject *)self); + Py_DECREF(type); +} + +static int +tracked_traverse(BoxObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->item); + return 0; +} + +static int +half_traverse(BoxObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static int +tracked_clear(BoxObject *self) +{ + Py_CLEAR(self->item); + return 0; +} + +static void +tracked_finalize(PyObject *self) +{ + boxes_state *state = state_of(self); + if (state != NULL) { + state->finalized++; + } +} + +static void +tracked_dealloc(BoxObject *self) +{ + PyTypeObject *type = Py_TYPE(self); + if (PyObject_CallFinalizerFromDealloc((PyObject *)self) < 0) { + return; + } + PyObject_GC_UnTrack(self); + boxes_state *state = state_of((PyObject *)self); + if (state != NULL) { + state->freed++; + } + Py_CLEAR(self->item); + type->tp_free((PyObject *)self); + Py_DECREF(type); +} + +static PyMemberDef box_members[] = { + {"item", Py_T_OBJECT_EX, offsetof(BoxObject, item), 0, "the one thing in the box"}, + {NULL, 0, 0, 0, NULL}, +}; + +static PyType_Slot loose_slots[] = { + {Py_tp_new, box_new}, + {Py_tp_dealloc, loose_dealloc}, + {Py_tp_members, box_members}, + {0, NULL}, +}; + +static PyType_Spec loose_spec = { + .name = "boxes.Loose", + .basicsize = sizeof(BoxObject), + .flags = Py_TPFLAGS_DEFAULT, + .slots = loose_slots, +}; + +static PyType_Slot half_slots[] = { + {Py_tp_new, box_new}, + {Py_tp_dealloc, tracked_dealloc}, + {Py_tp_traverse, half_traverse}, + {Py_tp_clear, tracked_clear}, + {Py_tp_members, box_members}, + {0, NULL}, +}; + +static PyType_Spec half_spec = { + .name = "boxes.Half", + .basicsize = sizeof(BoxObject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .slots = half_slots, +}; + +static PyType_Slot tracked_slots[] = { + {Py_tp_new, box_new}, + {Py_tp_dealloc, tracked_dealloc}, + {Py_tp_traverse, tracked_traverse}, + {Py_tp_clear, tracked_clear}, + {Py_tp_finalize, tracked_finalize}, + {Py_tp_members, box_members}, + {0, NULL}, +}; + +static PyType_Spec tracked_spec = { + .name = "boxes.Tracked", + .basicsize = sizeof(BoxObject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .slots = tracked_slots, +}; +\"\"\" + +print(f" {len(TYPES.splitlines())} lines of C, three types, one field each") +""") + + +lesson.md(f""" +Three specs, and the differences are small enough to list. `Loose` has no GC flag, no traverse and no clear, so the collector never hears about it. `Half` has the flag and a `tp_traverse` that reports the type and forgets the field. `Tracked` reports both, and adds a `tp_finalize`. + +Reporting the type is not optional decoration. Since 3.9, a {term("heap type")} is expected to visit `Py_TYPE(self)`, because an instance holds a reference to its own type and a type can be part of a cycle like anything else {cite("Objects/typeobject.c:2607-2645@v3.15.0rc1#subtype_traverse")}. + +The rest of the file is the module itself. It is worth reading for one reason: the counters live in {term("module state")} rather than in C statics. A `m_size` above zero gets a struct allocated with every module object {cite("Objects/moduleobject.c:1027-1035@v3.15.0rc1#PyModule_GetState")}, and a type made with `PyType_FromModuleAndSpec` can find its way back to it {cite("Objects/typeobject.c:5973-6005@v3.15.0rc1#PyType_GetModuleByDef")}. Statics would be shared across every {term("subinterpreter")} in the process, which is a bug waiting for somebody else to find. +""") + + +lesson.code(""" +MODULE = \"\"\" +static PyObject * +counts(PyObject *module, PyObject *unused) +{ + boxes_state *state = (boxes_state *)PyModule_GetState(module); + return Py_BuildValue("(ll)", state->freed, state->finalized); +} + +static PyObject * +reset(PyObject *module, PyObject *unused) +{ + boxes_state *state = (boxes_state *)PyModule_GetState(module); + state->freed = 0; + state->finalized = 0; + Py_RETURN_NONE; +} + +static PyMethodDef boxes_methods[] = { + {"counts", counts, METH_NOARGS, "How many boxes were freed and finalized so far."}, + {"reset", reset, METH_NOARGS, "Put both counters back to zero."}, + {NULL, NULL, 0, NULL}, +}; + +static int +add(PyObject *module, PyObject **slot, PyType_Spec *spec, const char *name) +{ + *slot = PyType_FromModuleAndSpec(module, spec, NULL); + if (*slot == NULL) { + return -1; + } + return PyModule_AddObjectRef(module, name, *slot); +} + +static int +boxes_exec(PyObject *module) +{ + boxes_state *state = (boxes_state *)PyModule_GetState(module); + if (add(module, &state->loose, &loose_spec, "Loose") < 0) { + return -1; + } + if (add(module, &state->half, &half_spec, "Half") < 0) { + return -1; + } + return add(module, &state->tracked, &tracked_spec, "Tracked"); +} + +static int +boxes_traverse(PyObject *module, visitproc visit, void *arg) +{ + boxes_state *state = (boxes_state *)PyModule_GetState(module); + Py_VISIT(state->loose); + Py_VISIT(state->half); + Py_VISIT(state->tracked); + return 0; +} + +static int +boxes_clear(PyObject *module) +{ + boxes_state *state = (boxes_state *)PyModule_GetState(module); + Py_CLEAR(state->loose); + Py_CLEAR(state->half); + Py_CLEAR(state->tracked); + return 0; +} + +static PyModuleDef_Slot boxes_slots[] = { + {Py_mod_exec, boxes_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + /* Honest, because two threads bumping state->freed would lose counts. */ + {Py_mod_gil, Py_MOD_GIL_USED}, + {0, NULL}, +}; + +static struct PyModuleDef boxes_module = { + PyModuleDef_HEAD_INIT, + .m_name = "boxes", + .m_size = sizeof(boxes_state), + .m_methods = boxes_methods, + .m_slots = boxes_slots, + .m_traverse = boxes_traverse, + .m_clear = boxes_clear, +}; + +PyMODINIT_FUNC +PyInit_boxes(void) +{ + return PyModuleDef_Init(&boxes_module); +} +\"\"\" + +if not HAVE_C: + print(NOTHING) +else: + TROUBLE = build("boxes", TYPES + MODULE) + HAVE_C = not TROUBLE + if HAVE_C: + import boxes + + made = [name for name in ("Loose", "Half", "Tracked") if hasattr(boxes, name)] + print(f" built and imported, with these types: {made}") + else: + print(f" the compiler said: {TROUBLE}") +""") + + +lesson.md(f""" +Now put one of each in a cycle. A box whose `item` is the box itself has a reference count of one that nothing outside can reach, which is exactly the case reference counting alone cannot handle. Drop the name, run a collection, and ask the module how many boxes it has freed. + +{lesson.claim("The GC flag on its own is not enough. A tp_traverse that forgets a field leaks just as completely as no tp_traverse at all")} +""") + + +lesson.code(""" +import gc + +KINDS = ("Loose", "Half", "Tracked") + +if not HAVE_C: + print(NOTHING) +else: + print(" type collector sees it traverse reports freed after a collection") + for name in KINDS: + kind = getattr(boxes, name) + boxes.reset() + box = kind() + seen = gc.is_tracked(box) + box.item = box + reports = len(gc.get_referents(box)) + del box + gc.collect() + print(f" {name:9} {seen!s:19} {reports:16} {boxes.counts()[0]}") +""") + + +lesson.md(f""" +`Loose` is invisible. `Half` is visible and reports one thing, its type, so the collector subtracts one reference from a box whose count is two and decides somebody outside still wants it. `Tracked` reports both and gets collected. The three rows differ by four lines of C. + +`gc.get_referents` in that table is `tp_traverse` called directly, which is the closest thing to a test you can write for it. If the numbers there do not match the fields your object owns, you have the `Half` bug. + +{figure("three-boxes-one-cycle", "a table of three container types with what the collector sees of each and whether it was freed")} + +There is a short list of what a `tp_traverse` handler may call, and it is short for a reason: traversal runs in the middle of a collection, where allocating, raising or running Python would be a disaster. Anything that could set an exception is out, which is why 3.15 added a whole family of functions ending in `_DuringGC` that do the same job as their ordinary versions and promise to have no side effects and never raise. + +{figure("what-traverse-may-call", "a table of the six things a tp_traverse handler is allowed to call")} + +## Where cleanup goes + +`Tracked` has a third slot the other two do not, and it is the answer to a question every container eventually raises: where does cleanup go when the object is part of a cycle? + +The {term("clear function")}, `tp_clear`, is what the collector calls to break the cycle {cite("Python/gc.c:1083-1120@v3.15.0rc1#delete_garbage")}. It drops exactly what `tp_traverse` reported. It is not the place for closing a file or releasing a lock, because it can be called while the rest of the cycle is being emptied around it. + +`tp_finalize` is that place. It is the modern `__del__`: it runs once per object, before anything is torn down, on both routes to death. When a count reaches zero, `tp_dealloc` calls it on the way in {cite("Objects/object.c:595-630@v3.15.0rc1#PyObject_CallFinalizerFromDealloc")}. When the collector finds a cycle, it runs `tp_finalize` on the whole unreachable set first, then starts clearing. The runtime keeps a {term("finalized bit")} so it cannot happen twice. + +{figure("where-the-two-hooks-run", "the two routes an object takes to being freed and where each hook runs on it")} + +{lesson.claim("A tp_finalize runs exactly once per object whether the object died on a reference count or inside a cycle")} +""") + + +lesson.code(""" +if not HAVE_C: + print(NOTHING) +else: + boxes.reset() + plain = boxes.Tracked() + plain.item = [1, 2, 3] + del plain + freed, final = boxes.counts() + print(f" died on its reference count: finalized {final}, freed {freed}") + + boxes.reset() + looped = boxes.Tracked() + looped.item = looped + del looped + gc.collect() + freed, final = boxes.counts() + print(f" died inside a cycle: finalized {final}, freed {freed}") +""") + + +lesson.md(f""" +One hook, both routes, once each. That is the whole reason `tp_finalize` replaced the old `tp_del` slot, which M09 covered from the collector's side: an object with the old slot in a cycle used to be uncollectable and got put on `gc.garbage` for a human to deal with. + +## The promise at the bottom of the file + +The fourth rule is one line, and on most builds it does nothing at all. + +An {term("extension module")} on a {term("free threaded build")} has to say whether it is safe without the lock. `Py_mod_gil` with `Py_MOD_GIL_NOT_USED` is a promise. Saying nothing, or saying `Py_MOD_GIL_USED`, makes the runtime turn the lock back on for the whole process the moment your module is imported, and print a warning naming you. + +The `boxes` module above says `Py_MOD_GIL_USED`, and that is not modesty. Two threads calling `state->freed++` would lose counts, because that is a read, an add and a write with nothing in between. Keeping the promise instead would mean a {term("critical section")} around every field the two types share, which C02 and C06 went through from the reading side. `errpath` says `Py_MOD_GIL_NOT_USED` and means it, because it has no state at all. + +{lesson.claim("One imported module that has not declared itself safe turns the lock back on for the whole process", unobservable="the flip only happens on an interpreter built with --disable-gil, and this notebook is almost certainly not one")} + +CPython's own test suite ships a shared object that exports two init functions differing only in that line, which makes the difference easy to weigh without compiling anything. This recording loads each of them on a free threaded build, in a child of its own so the answer is not contaminated by the previous one. + +{recording(ON_FREETHREADED)} + +The third case is the escape hatch. `PYTHON_GIL=0` tells the runtime you have read the module's source yourself and disagree with its declaration, and the warning goes away with the lock. It is exactly as safe as your reading was. + +## Proving it + +Four rules, and every way of breaking them looks like working code. So the last thing to write is the test that would have caught any of it. + +The idea fits in a handful of lines. Give the thing you are watching a type of its own so it is easy to count. Collect. Count. Do the work. Collect and count again. Repeat, and throw the first few rounds away. If the count goes up by the same amount every single time, that is a leak; if it goes up once and settles, that is a cache filling. + +{lesson.claim("An ordinary test run cannot see a reference leak at all, which is why CPython runs its own suite a second way")} +""") + + +lesson.code(""" +class Held: + \"\"\"Something for a box to hold, so that the thing being leaked has a name.\"\"\" + + +def alive(): + \"\"\"How many Held objects the collector can still find anywhere in this process.\"\"\" + return sum(1 for one in gc.get_objects() if isinstance(one, Held)) + + +def hunt(work, repeats=6, warmups=3): + \"\"\"Run something over and over and report what each run left behind.\"\"\" + deltas = [] + for _ in range(repeats): + gc.collect() + before = alive() + work() + gc.collect() + deltas.append(alive() - before) + return deltas[warmups:] + + +def a_cycle(kind): + \"\"\"One box and one Held pointing at each other, with no name left for either.\"\"\" + + def work(): + box = kind() + held = Held() + held.box = box + box.item = held + + return work + + +if not HAVE_C: + print(NOTHING) +else: + for name in KINDS: + left = hunt(a_cycle(getattr(boxes, name))) + verdict = "a leak" if all(one >= 1 for one in left) else "fine" + print(f" {name:9} {left!s:12} {verdict}") +""") + + +lesson.md(f""" +That is CPython's own `-R` flag with the interesting parts taken out. The real one counts references and allocated blocks and open file descriptors rather than instances of one class you picked, which means it needs a {term("debug build")}, since an ordinary build keeps no running total of references. It throws away the first few runs, because caches fill on first use and that is not a leak. And it only calls something a failure if every measured run leaked at least one {cite("Lib/test/libregrtest/refleak.py:196-209@v3.15.0rc1#check_rc_deltas")}, then prints the deltas it saw {cite("Lib/test/libregrtest/refleak.py:214-236@v3.15.0rc1#leaked")}. + +{figure("what-the-hunter-counts", "four tests with the verdict an ordinary run gives and the verdict the leak hunter gives")} + +This recording runs four small tests on a debug build, once the way anybody runs a test suite and once with `-R 3:3`, which means three warm up runs and three measured ones. + +{recording(ON_DEBUG)} + +All four pass the first time. Two fail the second, and one of the two that passes still gets a note, because it left something behind on the first measured run and then stopped. That last case is the reason the hunter is careful rather than strict, and also the reason a first attempt at your own leak test will produce noise. + +## Try it yourself + +Change `half_traverse` to report `self->item` and nothing else, so it visits the field but forgets the type. Rebuild and rerun the table. The box is still collected, because nothing in these examples puts a type in a cycle, which is exactly why the rule about visiting `Py_TYPE(self)` is easy to break and hard to notice. + +Take the `Py_DECREF` out of `tracked_dealloc`'s `Py_CLEAR(self->item)` and run the hunter over `Tracked` again. You have written the `Half` bug a second way, with a complete `tp_traverse` and a `tp_dealloc` that does not finish the job. + +Add a `tp_finalize` to `Loose` and put one in a cycle. Nothing runs, because a type the collector cannot see never gets a second chance. + +Give `Tracked` a `__del__` written in Python by subclassing it, then put an instance in a cycle. Compare what runs against the C `tp_finalize` on the same object, and in which order. + +## What you now know + +An error branch that forgets one `Py_DECREF` leaks silently and forever, and the shape that prevents it is a single cleanup block reached by `goto`. + +A container type needs three things before the collector can help it: the `Py_TPFLAGS_HAVE_GC` flag, allocation through `PyObject_GC_New`, and a `tp_traverse` that reports every field, including the type of a heap type. Miss the flag and you are invisible. Miss a field and you are visible and lying, which leaks just as thoroughly. + +`tp_clear` breaks cycles and belongs to the collector. `tp_finalize` is where your cleanup goes, and it runs once per object on either route to death. + +Module state exists so that C extensions do not keep things in statics, which subinterpreters share. + +`Py_mod_gil` is a promise about thread safety that the runtime cannot check, so it believes you, and turns the lock back on for everybody if you decline to make it. + +None of these failures produce an error. The tool that finds them runs your tests six times on a debug build and counts what is left, and it is the difference between code that passes review and code that runs for a year. + +## What is next + +That is the end of the runtime part, and the end of watching CPython work. R01 asked what happens before your first line, R08 asked what happens after your last one, and this one asked what the runtime expects from code you write in its own language. + +What is left in this milestone is not lessons. It is the blueprints: the written down descriptions of state layout, the object model under free threading, the import system and the C API surface, precise enough that somebody could build them again in another language. The reimplementation track after that is the whole point of writing any of this down. +""") + + +raise SystemExit(lesson.save()) diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams.py b/lessons/r09-writing-a-c-extension-properly/diagrams.py new file mode 100644 index 0000000..3cdd7ac --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +"""The diagrams for R09, the four things a C extension owes the runtime. + +Each scene is written out twice, as an editable `.excalidraw` and as the `.svg` the lesson +embeds. Run this file to regenerate them, or `just build-diagrams` for every lesson. + +The order follows the lesson. The one missing line on an error path first, then the three +boxes and what the collector can see of each, then the step that needs tp_traverse and the +short list of what tp_traverse may call, then where the two teardown hooks run, and last +what the leak hunter reports about four tests that all pass an ordinary run. +""" + +from nbdiagram import Gallery, figures + +gallery = Gallery("r09-writing-a-c-extension-properly") + +gallery.add( + figures.compare( + "the-branch-nobody-tests", + ( + "the version with one line missing", + [ + "pair = PyTuple_Pack(2, arg, arg)", + "the tuple now holds two refs on arg", + "hashing it fails, so return NULL", + "the tuple is never freed", + ], + ), + ( + "the version with the line", + [ + "pair = PyTuple_Pack(2, arg, arg)", + "the tuple now holds two refs on arg", + "hashing it fails, so Py_DECREF first", + "the tuple is freed, and lets go of arg", + ], + ), + title="Two error paths that differ by one line", + verdict="A thousand failed calls, two thousand references, and no error anybody can see.", + ) +) + + +gallery.add( + figures.table( + "three-boxes-one-cycle", + ["the type", "collector sees it", "tp_traverse reports", "freed after a collection"], + [ + ["Loose, no GC flag", "no", "nothing, there is none", "no"], + ["Half, traverse forgets", "yes", "its type only", "no"], + ["Tracked, traverse complete", "yes", "its type and its item", "yes"], + ], + title="One container written three ways, each put in a cycle and collected", + caption="The flag gets you into the collector. A complete tp_traverse gets you back out.", + tones=["warning", "warning", "durable"], + ) +) + + +gallery.add( + figures.flow( + "how-the-collector-uses-traverse", + [ + "take the reference count of every candidate", + "call tp_traverse on each of them", + "subtract one for every visit landing inside the set", + "whatever is still above zero is held from outside", + ], + title="Why an incomplete tp_traverse leaks instead of crashing", + tones=["quiet", "focus", "focus", "durable"], + ) +) + + +gallery.add( + figures.table( + "what-traverse-may-call", + ["what you may call in tp_traverse", "what it is for"], + [ + ["the visit function you were handed", "reporting one reference"], + ["Py_VISIT", "the same thing, with the null check written for you"], + ["Py_TYPE and Py_SIZE", "reading the header, valid for this call only"], + ["PyObject_VisitManagedDict", "reporting an instance dict the runtime owns"], + ["PyType_HasFeature and the Check macros", "asking what kind of object this is"], + ["the _DuringGC functions, new in 3.15", "reaching module state without side effects"], + ], + title="The whole list of what a tp_traverse handler is allowed to do", + caption="Anything that can allocate, raise or run Python is not on it.", + tones=["focus", "focus", "quiet", "quiet", "quiet", "durable"], + ) +) + + +gallery.add( + figures.compare( + "where-the-two-hooks-run", + ( + "the count reached zero on its own", + [ + "tp_dealloc is called", + "it calls tp_finalize first", + "then it frees the memory", + ], + ), + ( + "the collector found a cycle", + [ + "tp_finalize runs on everything first", + "then tp_clear breaks the links", + "counts fall, and tp_dealloc follows", + ], + ), + title="The two ways your object dies, and the one hook that runs either way", + verdict="tp_finalize runs at most once per object, and the runtime keeps that flag for you.", + verdict_tone="durable", + ) +) + + +gallery.add( + figures.table( + "what-the-hunter-counts", + ["the test", "an ordinary run", "six runs with the hunter"], + [ + ["holds nothing", "passed", "nothing left behind"], + ["fills a cache once", "passed", "[1, 0, 0] references, called fine"], + ["appends to a global list", "passed", "[1, 1, 1] references, a failure"], + ["duplicates a file handle", "passed", "[1, 1, 1] file descriptors, a failure"], + ], + title="Four small tests, and the two verdicts each one gets", + caption="Every one of them passes the run you do before you open a pull request.", + tones=["durable", "intermediate", "warning", "warning"], + ) +) + + +raise SystemExit(gallery.save()) diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams/how-the-collector-uses-traverse.excalidraw b/lessons/r09-writing-a-c-extension-properly/diagrams/how-the-collector-uses-traverse.excalidraw new file mode 100644 index 0000000..b7fa66b --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams/how-the-collector-uses-traverse.excalidraw @@ -0,0 +1,506 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://github.com/tamnd/cpython-internals", + "elements": [ + { + "id": "a6f6e3a8ed1c85decbc1", + "type": "text", + "x": 0.0, + "y": 0.0, + "width": 739.08, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Why an incomplete tp_traverse leaks instead of crashing", + "originalText": "Why an incomplete tp_traverse leaks instead of crashing", + "fontSize": 24, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "c2d6f9d881ea4377827d", + "type": "rectangle", + "x": 0.0, + "y": 50.0, + "width": 576.5, + "height": 70.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "afce4cdfe694fc65f229", + "type": "text" + }, + { + "id": "cf87b4411661181f3dae", + "type": "arrow" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "afce4cdfe694fc65f229", + "type": "text", + "x": 16.0, + "y": 72.5, + "width": 544.5, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "take the reference count of every candidate", + "originalText": "take the reference count of every candidate", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "c2d6f9d881ea4377827d", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "a17194af124a4370f956", + "type": "rectangle", + "x": 0.0, + "y": 180.0, + "width": 576.5, + "height": 70.0, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "50b8289ed6603fa12575", + "type": "text" + }, + { + "id": "cf87b4411661181f3dae", + "type": "arrow" + }, + { + "id": "81f19c355e46c008d358", + "type": "arrow" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "50b8289ed6603fa12575", + "type": "text", + "x": 16.0, + "y": 202.5, + "width": 544.5, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "call tp_traverse on each of them", + "originalText": "call tp_traverse on each of them", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "a17194af124a4370f956", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "e38c70fefd0edbceee6d", + "type": "rectangle", + "x": 0.0, + "y": 310.0, + "width": 576.5, + "height": 70.0, + "angle": 0, + "strokeColor": "#e8590c", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "bfcc92e4a0acc4877ecd", + "type": "text" + }, + { + "id": "81f19c355e46c008d358", + "type": "arrow" + }, + { + "id": "3331563afaa4b2815a3c", + "type": "arrow" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "bfcc92e4a0acc4877ecd", + "type": "text", + "x": 16.0, + "y": 332.5, + "width": 544.5, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "subtract one for every visit landing inside the set", + "originalText": "subtract one for every visit landing inside the set", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "e38c70fefd0edbceee6d", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "36c28985ea525ab7ae1c", + "type": "rectangle", + "x": 0.0, + "y": 440.0, + "width": 576.5, + "height": 70.0, + "angle": 0, + "strokeColor": "#099268", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "a0c3c2f70625705d3d73", + "type": "text" + }, + { + "id": "3331563afaa4b2815a3c", + "type": "arrow" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "a0c3c2f70625705d3d73", + "type": "text", + "x": 16.0, + "y": 462.5, + "width": 544.5, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "whatever is still above zero is held from outside", + "originalText": "whatever is still above zero is held from outside", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "36c28985ea525ab7ae1c", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "cf87b4411661181f3dae", + "type": "arrow", + "x": 288.25, + "y": 120.0, + "width": 0.0, + "height": 60.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 60.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "c2d6f9d881ea4377827d", + "focus": 0, + "gap": 4 + }, + "endBinding": { + "elementId": "a17194af124a4370f956", + "focus": 0, + "gap": 4 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "81f19c355e46c008d358", + "type": "arrow", + "x": 288.25, + "y": 250.0, + "width": 0.0, + "height": 60.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 60.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "a17194af124a4370f956", + "focus": 0, + "gap": 4 + }, + "endBinding": { + "elementId": "e38c70fefd0edbceee6d", + "focus": 0, + "gap": 4 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "3331563afaa4b2815a3c", + "type": "arrow", + "x": 288.25, + "y": 380.0, + "width": 0.0, + "height": 60.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.0, + 60.0 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "e38c70fefd0edbceee6d", + "focus": 0, + "gap": 4 + }, + "endBinding": { + "elementId": "36c28985ea525ab7ae1c", + "focus": 0, + "gap": 4 + }, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams/how-the-collector-uses-traverse.svg b/lessons/r09-writing-a-c-extension-properly/diagrams/how-the-collector-uses-traverse.svg new file mode 100644 index 0000000..fd704ff --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams/how-the-collector-uses-traverse.svg @@ -0,0 +1 @@ +Why an incomplete tp_traverse leaks instead of crashingtake the reference count of every candidatecall tp_traverse on each of themsubtract one for every visit landing inside the setwhatever is still above zero is held from outside diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams/the-branch-nobody-tests.excalidraw b/lessons/r09-writing-a-c-extension-properly/diagrams/the-branch-nobody-tests.excalidraw new file mode 100644 index 0000000..44d1749 --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams/the-branch-nobody-tests.excalidraw @@ -0,0 +1,753 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://github.com/tamnd/cpython-internals", + "elements": [ + { + "id": "fc7be1bd470e9481594b", + "type": "text", + "x": 0.0, + "y": 0.0, + "width": 505.44000000000005, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Two error paths that differ by one line", + "originalText": "Two error paths that differ by one line", + "fontSize": 24, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "cbf9b4a1e3405245f138", + "type": "text", + "x": 59.05000000000001, + "y": 50.0, + "width": 369.9, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the version with one line missing", + "originalText": "the version with one line missing", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "b938373957650d8213f1", + "type": "rectangle", + "x": 0.0, + "y": 87.0, + "width": 488.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "ad6b66964328fd300de5", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "ad6b66964328fd300de5", + "type": "text", + "x": 16.0, + "y": 103.0, + "width": 456.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "pair = PyTuple_Pack(2, arg, arg)", + "originalText": "pair = PyTuple_Pack(2, arg, arg)", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "b938373957650d8213f1", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "51580c32cd4aefe42eeb", + "type": "rectangle", + "x": 0.0, + "y": 147.0, + "width": 488.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "b079cf601140510849bb", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "b079cf601140510849bb", + "type": "text", + "x": 16.0, + "y": 163.0, + "width": 456.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "the tuple now holds two refs on arg", + "originalText": "the tuple now holds two refs on arg", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "51580c32cd4aefe42eeb", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "87b9a87ded601f0ec364", + "type": "rectangle", + "x": 0.0, + "y": 207.0, + "width": 488.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "6bb8afaa43425a941e2d", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "6bb8afaa43425a941e2d", + "type": "text", + "x": 16.0, + "y": 223.0, + "width": 456.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "hashing it fails, so return NULL", + "originalText": "hashing it fails, so return NULL", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "87b9a87ded601f0ec364", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "ba8792945aa99607ad8d", + "type": "rectangle", + "x": 0.0, + "y": 267.0, + "width": 488.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "0a5cf5872a67f9fdf442", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "0a5cf5872a67f9fdf442", + "type": "text", + "x": 16.0, + "y": 283.0, + "width": 456.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "the tuple is never freed", + "originalText": "the tuple is never freed", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "ba8792945aa99607ad8d", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "ed4f680f372b71d817c4", + "type": "text", + "x": 656.4, + "y": 50.0, + "width": 271.2, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the version with the line", + "originalText": "the version with the line", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "929864731184e0780173", + "type": "rectangle", + "x": 548.0, + "y": 87.0, + "width": 488.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#099268", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "f7006155256299df415d", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "f7006155256299df415d", + "type": "text", + "x": 564.0, + "y": 103.0, + "width": 456.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "pair = PyTuple_Pack(2, arg, arg)", + "originalText": "pair = PyTuple_Pack(2, arg, arg)", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "929864731184e0780173", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "eb644e98c2dc6a469e55", + "type": "rectangle", + "x": 548.0, + "y": 147.0, + "width": 488.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "07b04fd0be53750a62f2", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "07b04fd0be53750a62f2", + "type": "text", + "x": 564.0, + "y": 163.0, + "width": 456.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "the tuple now holds two refs on arg", + "originalText": "the tuple now holds two refs on arg", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "eb644e98c2dc6a469e55", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "7869d83a3d64190a5a91", + "type": "rectangle", + "x": 548.0, + "y": 207.0, + "width": 488.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "1e7c544e5c60bd1e63ee", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "1e7c544e5c60bd1e63ee", + "type": "text", + "x": 564.0, + "y": 223.0, + "width": 456.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "hashing it fails, so Py_DECREF first", + "originalText": "hashing it fails, so Py_DECREF first", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "7869d83a3d64190a5a91", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "de88b1b9890988e6b4db", + "type": "rectangle", + "x": 548.0, + "y": 267.0, + "width": 488.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "6b3edb9cd04a75bca719", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "6b3edb9cd04a75bca719", + "type": "text", + "x": 564.0, + "y": 283.0, + "width": 456.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "the tuple is freed, and lets go of arg", + "originalText": "the tuple is freed, and lets go of arg", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "de88b1b9890988e6b4db", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "fc620078a1bd9b944c04", + "type": "rectangle", + "x": 0.0, + "y": 344.0, + "width": 1036.0, + "height": 60.0, + "angle": 0, + "strokeColor": "#e03131", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "e80514bc164c04f8bb60", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "e80514bc164c04f8bb60", + "type": "text", + "x": 16.0, + "y": 361.5, + "width": 1004.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "A thousand failed calls, two thousand references, and no error anybody can see.", + "originalText": "A thousand failed calls, two thousand references, and no error anybody can see.", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "fc620078a1bd9b944c04", + "lineHeight": 1.25, + "autoResize": false + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams/the-branch-nobody-tests.svg b/lessons/r09-writing-a-c-extension-properly/diagrams/the-branch-nobody-tests.svg new file mode 100644 index 0000000..c47e01d --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams/the-branch-nobody-tests.svg @@ -0,0 +1 @@ +Two error paths that differ by one linethe version with one line missingpair = PyTuple_Pack(2, arg, arg)the tuple now holds two refs on arghashing it fails, so return NULLthe tuple is never freedthe version with the linepair = PyTuple_Pack(2, arg, arg)the tuple now holds two refs on arghashing it fails, so Py_DECREF firstthe tuple is freed, and lets go of argA thousand failed calls, two thousand references, and no error anybody can see. diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams/three-boxes-one-cycle.excalidraw b/lessons/r09-writing-a-c-extension-properly/diagrams/three-boxes-one-cycle.excalidraw new file mode 100644 index 0000000..47ab3c8 --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams/three-boxes-one-cycle.excalidraw @@ -0,0 +1,983 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://github.com/tamnd/cpython-internals", + "elements": [ + { + "id": "90fa2db5cf036db58c7a", + "type": "text", + "x": 0.0, + "y": 0.0, + "width": 895.6800000000001, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "One container written three ways, each put in a cycle and collected", + "originalText": "One container written three ways, each put in a cycle and collected", + "fontSize": 24, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "347353132820053beabe", + "type": "text", + "x": 14.0, + "y": 50.0, + "width": 69.6, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the type", + "originalText": "the type", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "e15b230ff6834f4f0088", + "type": "text", + "x": 291.6, + "y": 50.0, + "width": 141.6, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "collector sees it", + "originalText": "collector sees it", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "e5f966e3c852be6c203b", + "type": "text", + "x": 482.8, + "y": 50.0, + "width": 157.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "tp_traverse reports", + "originalText": "tp_traverse reports", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "4df1a5bd83039701b230", + "type": "text", + "x": 722.0, + "y": 50.0, + "width": 198.0, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "freed after a collection", + "originalText": "freed after a collection", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "5c0c1decc6629a417f89", + "type": "line", + "x": 0.0, + "y": 76.0, + "width": 966.4, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 966.4, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "d41df4d1744d1edd084a", + "type": "line", + "x": 0.0, + "y": 76.0, + "width": 966.4, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 966.4, + 0.0 + ], + [ + 966.4, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "e1b66f0ae452be90b997", + "type": "text", + "x": 14.0, + "y": 86.0, + "width": 163.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Loose, no GC flag", + "originalText": "Loose, no GC flag", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "5bf019dd67caf9193175", + "type": "text", + "x": 291.6, + "y": 86.0, + "width": 19.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "no", + "originalText": "no", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "e31161a593afe5d34957", + "type": "text", + "x": 482.8, + "y": 86.0, + "width": 211.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "nothing, there is none", + "originalText": "nothing, there is none", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "fd4b2dff33b837ceb5ec", + "type": "text", + "x": 722.0, + "y": 86.0, + "width": 19.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "no", + "originalText": "no", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "3566f719eda3d037ec97", + "type": "line", + "x": 0.0, + "y": 116.0, + "width": 966.4, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 966.4, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "8c00215b602a12527730", + "type": "line", + "x": 0.0, + "y": 116.0, + "width": 966.4, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 966.4, + 0.0 + ], + [ + 966.4, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "5d30beee207ccd0e70d1", + "type": "text", + "x": 14.0, + "y": 126.0, + "width": 211.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Half, traverse forgets", + "originalText": "Half, traverse forgets", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "79f2ae0bd04a5900c34e", + "type": "text", + "x": 291.6, + "y": 126.0, + "width": 28.799999999999997, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "yes", + "originalText": "yes", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "084f091b2916afe42318", + "type": "text", + "x": 482.8, + "y": 126.0, + "width": 124.8, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "its type only", + "originalText": "its type only", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "84bba808999104e38b9e", + "type": "text", + "x": 722.0, + "y": 126.0, + "width": 19.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "no", + "originalText": "no", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "da3d1aef8719a2d9e6b3", + "type": "line", + "x": 0.0, + "y": 156.0, + "width": 966.4, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 966.4, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "b85e955e8e0d67a39170", + "type": "line", + "x": 0.0, + "y": 156.0, + "width": 966.4, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 966.4, + 0.0 + ], + [ + 966.4, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "3746e30fa2dea46123db", + "type": "text", + "x": 14.0, + "y": 166.0, + "width": 249.6, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Tracked, traverse complete", + "originalText": "Tracked, traverse complete", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "1deae4514f2068f544d4", + "type": "text", + "x": 291.6, + "y": 166.0, + "width": 28.799999999999997, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "yes", + "originalText": "yes", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "4766538066f2d6ad1285", + "type": "text", + "x": 482.8, + "y": 166.0, + "width": 201.6, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "its type and its item", + "originalText": "its type and its item", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "435ab3c48dcd75449a8b", + "type": "text", + "x": 722.0, + "y": 166.0, + "width": 28.799999999999997, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "yes", + "originalText": "yes", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "568f4548c30eba32f6da", + "type": "line", + "x": 0.0, + "y": 196.0, + "width": 966.4, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 966.4, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "754048916ade31f43e62", + "type": "text", + "x": 0.0, + "y": 216.0, + "width": 692.16, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "The flag gets you into the collector. A complete tp_traverse gets you back out.", + "originalText": "The flag gets you into the collector. A complete tp_traverse gets you back out.", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams/three-boxes-one-cycle.svg b/lessons/r09-writing-a-c-extension-properly/diagrams/three-boxes-one-cycle.svg new file mode 100644 index 0000000..18eb65f --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams/three-boxes-one-cycle.svg @@ -0,0 +1 @@ +One container written three ways, each put in a cycle and collectedthe typecollector sees ittp_traverse reportsfreed after a collectionLoose, no GC flagnonothing, there is nonenoHalf, traverse forgetsyesits type onlynoTracked, traverse completeyesits type and its itemyesThe flag gets you into the collector. A complete tp_traverse gets you back out. diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams/what-the-hunter-counts.excalidraw b/lessons/r09-writing-a-c-extension-properly/diagrams/what-the-hunter-counts.excalidraw new file mode 100644 index 0000000..556ebf7 --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams/what-the-hunter-counts.excalidraw @@ -0,0 +1,1035 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://github.com/tamnd/cpython-internals", + "elements": [ + { + "id": "5d9e7d9eb4b6966dc0b5", + "type": "text", + "x": 0.0, + "y": 0.0, + "width": 701.28, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Four small tests, and the two verdicts each one gets", + "originalText": "Four small tests, and the two verdicts each one gets", + "fontSize": 24, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "fe223d5507c6b318d2d1", + "type": "text", + "x": 14.0, + "y": 50.0, + "width": 66.0, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the test", + "originalText": "the test", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "f7490fe75eeebe611afb", + "type": "text", + "x": 272.4, + "y": 50.0, + "width": 129.6, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "an ordinary run", + "originalText": "an ordinary run", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "be616d388ae102d88d7c", + "type": "text", + "x": 444.4, + "y": 50.0, + "width": 210.95999999999998, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "six runs with the hunter", + "originalText": "six runs with the hunter", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "f882b429b37086745bc8", + "type": "line", + "x": 0.0, + "y": 76.0, + "width": 813.5999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 813.5999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "3507c0f5cb50d06f7fab", + "type": "line", + "x": 0.0, + "y": 76.0, + "width": 813.5999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 813.5999999999999, + 0.0 + ], + [ + 813.5999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "e0f48a299eaaa1cdf8b8", + "type": "text", + "x": 14.0, + "y": 86.0, + "width": 124.8, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "holds nothing", + "originalText": "holds nothing", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "a2fe97b0037280c4d3b1", + "type": "text", + "x": 272.4, + "y": 86.0, + "width": 57.599999999999994, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "passed", + "originalText": "passed", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "d7a0e9562ad7a665568a", + "type": "text", + "x": 444.4, + "y": 86.0, + "width": 182.4, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "nothing left behind", + "originalText": "nothing left behind", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "f98ce9a1eea69f883ee0", + "type": "line", + "x": 0.0, + "y": 116.0, + "width": 813.5999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 813.5999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "a4b4f24cd77a98a3f00f", + "type": "line", + "x": 0.0, + "y": 116.0, + "width": 813.5999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#d0bfff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 813.5999999999999, + 0.0 + ], + [ + 813.5999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "709f539ff00027168796", + "type": "text", + "x": 14.0, + "y": 126.0, + "width": 172.79999999999998, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "fills a cache once", + "originalText": "fills a cache once", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "5ad2802493967079f971", + "type": "text", + "x": 272.4, + "y": 126.0, + "width": 57.599999999999994, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "passed", + "originalText": "passed", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "b061b70ced76c531c1fa", + "type": "text", + "x": 444.4, + "y": 126.0, + "width": 316.8, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "[1, 0, 0] references, called fine", + "originalText": "[1, 0, 0] references, called fine", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "3b74f23b969fcbfd649d", + "type": "line", + "x": 0.0, + "y": 156.0, + "width": 813.5999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 813.5999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "114f88c7ca4043c665c0", + "type": "line", + "x": 0.0, + "y": 156.0, + "width": 813.5999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 813.5999999999999, + 0.0 + ], + [ + 813.5999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "92250185da9ac42846fc", + "type": "text", + "x": 14.0, + "y": 166.0, + "width": 230.39999999999998, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "appends to a global list", + "originalText": "appends to a global list", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "a65550435b894a31d13b", + "type": "text", + "x": 272.4, + "y": 166.0, + "width": 57.599999999999994, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "passed", + "originalText": "passed", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "b87cc9828dd6324e700d", + "type": "text", + "x": 444.4, + "y": 166.0, + "width": 297.59999999999997, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "[1, 1, 1] references, a failure", + "originalText": "[1, 1, 1] references, a failure", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "384a3cb073d998722ca1", + "type": "line", + "x": 0.0, + "y": 196.0, + "width": 813.5999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 813.5999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "53cf960bc4bd74b7436e", + "type": "line", + "x": 0.0, + "y": 196.0, + "width": 813.5999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#ffc9c9", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 813.5999999999999, + 0.0 + ], + [ + 813.5999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "6a8b22ee577431ce08d8", + "type": "text", + "x": 14.0, + "y": 206.0, + "width": 230.39999999999998, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "duplicates a file handle", + "originalText": "duplicates a file handle", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "74acc568b477b6f2664a", + "type": "text", + "x": 272.4, + "y": 206.0, + "width": 57.599999999999994, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "passed", + "originalText": "passed", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "eca6e2f07747b527b85c", + "type": "text", + "x": 444.4, + "y": 206.0, + "width": 355.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "[1, 1, 1] file descriptors, a failure", + "originalText": "[1, 1, 1] file descriptors, a failure", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "f07b0f241dbac0c01617", + "type": "line", + "x": 0.0, + "y": 236.0, + "width": 813.5999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 813.5999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "bb6db9e663a2fec10147", + "type": "text", + "x": 0.0, + "y": 256.0, + "width": 644.16, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Every one of them passes the run you do before you open a pull request.", + "originalText": "Every one of them passes the run you do before you open a pull request.", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams/what-the-hunter-counts.svg b/lessons/r09-writing-a-c-extension-properly/diagrams/what-the-hunter-counts.svg new file mode 100644 index 0000000..669152e --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams/what-the-hunter-counts.svg @@ -0,0 +1 @@ +Four small tests, and the two verdicts each one getsthe testan ordinary runsix runs with the hunterholds nothingpassednothing left behindfills a cache oncepassed[1, 0, 0] references, called fineappends to a global listpassed[1, 1, 1] references, a failureduplicates a file handlepassed[1, 1, 1] file descriptors, a failureEvery one of them passes the run you do before you open a pull request. diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams/what-traverse-may-call.excalidraw b/lessons/r09-writing-a-c-extension-properly/diagrams/what-traverse-may-call.excalidraw new file mode 100644 index 0000000..a22b691 --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams/what-traverse-may-call.excalidraw @@ -0,0 +1,1176 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://github.com/tamnd/cpython-internals", + "elements": [ + { + "id": "ba4313c923ec556030fd", + "type": "text", + "x": 0.0, + "y": 0.0, + "width": 817.92, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "The whole list of what a tp_traverse handler is allowed to do", + "originalText": "The whole list of what a tp_traverse handler is allowed to do", + "fontSize": 24, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "8e3376aef5fe582baf80", + "type": "text", + "x": 14.0, + "y": 50.0, + "width": 289.91999999999996, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "what you may call in tp_traverse", + "originalText": "what you may call in tp_traverse", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "71e49b6150454399c302", + "type": "text", + "x": 406.8, + "y": 50.0, + "width": 118.56, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "what it is for", + "originalText": "what it is for", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "696c99f53a35406d5866", + "type": "line", + "x": 0.0, + "y": 76.0, + "width": 910.3999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "3f21714f001230d41043", + "type": "line", + "x": 0.0, + "y": 76.0, + "width": 910.3999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ], + [ + 910.3999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "63fab96563627f4d953a", + "type": "text", + "x": 14.0, + "y": 86.0, + "width": 326.4, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the visit function you were handed", + "originalText": "the visit function you were handed", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "d41329eefa07852a753a", + "type": "text", + "x": 406.8, + "y": 86.0, + "width": 220.79999999999998, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "reporting one reference", + "originalText": "reporting one reference", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "48f3cc7c346978a354af", + "type": "line", + "x": 0.0, + "y": 116.0, + "width": 910.3999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "29e7e08d9064d95e8ddd", + "type": "line", + "x": 0.0, + "y": 116.0, + "width": 910.3999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#ffd8a8", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ], + [ + 910.3999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "96c8da93be58a5179243", + "type": "text", + "x": 14.0, + "y": 126.0, + "width": 76.8, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Py_VISIT", + "originalText": "Py_VISIT", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "f499561f6277e29931eb", + "type": "text", + "x": 406.8, + "y": 126.0, + "width": 489.59999999999997, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the same thing, with the null check written for you", + "originalText": "the same thing, with the null check written for you", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "895ff09561edb97ce37d", + "type": "line", + "x": 0.0, + "y": 156.0, + "width": 910.3999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "aed4bf797ccc9be2f6bc", + "type": "line", + "x": 0.0, + "y": 156.0, + "width": 910.3999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ], + [ + 910.3999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "6bcbf5d74b1c8671fc4b", + "type": "text", + "x": 14.0, + "y": 166.0, + "width": 182.4, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Py_TYPE and Py_SIZE", + "originalText": "Py_TYPE and Py_SIZE", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "4602ca08834704d5bd3f", + "type": "text", + "x": 406.8, + "y": 166.0, + "width": 422.4, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "reading the header, valid for this call only", + "originalText": "reading the header, valid for this call only", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "461d41345818146718b8", + "type": "line", + "x": 0.0, + "y": 196.0, + "width": 910.3999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "75722e1989a5349f6538", + "type": "line", + "x": 0.0, + "y": 196.0, + "width": 910.3999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ], + [ + 910.3999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "969e10d261f93d31ca12", + "type": "text", + "x": 14.0, + "y": 206.0, + "width": 240.0, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "PyObject_VisitManagedDict", + "originalText": "PyObject_VisitManagedDict", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "0fa419f43ed91ec71331", + "type": "text", + "x": 406.8, + "y": 206.0, + "width": 412.8, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "reporting an instance dict the runtime owns", + "originalText": "reporting an instance dict the runtime owns", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "fd02bb029a288f24e434", + "type": "line", + "x": 0.0, + "y": 236.0, + "width": 910.3999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "fc26729b15dfb6967265", + "type": "line", + "x": 0.0, + "y": 236.0, + "width": 910.3999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ], + [ + 910.3999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "8ceb263cbfd800d15caf", + "type": "text", + "x": 14.0, + "y": 246.0, + "width": 364.8, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "PyType_HasFeature and the Check macros", + "originalText": "PyType_HasFeature and the Check macros", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "c249223d396ad62d1417", + "type": "text", + "x": 406.8, + "y": 246.0, + "width": 326.4, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "asking what kind of object this is", + "originalText": "asking what kind of object this is", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "a739d64dc8ea8a3624e5", + "type": "line", + "x": 0.0, + "y": 276.0, + "width": 910.3999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "6431c78bd1f68a321165", + "type": "line", + "x": 0.0, + "y": 276.0, + "width": 910.3999999999999, + "height": 40.0, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ], + [ + 910.3999999999999, + 40.0 + ], + [ + 0, + 40.0 + ], + [ + 0, + 0.0 + ] + ], + "lastCommittedPoint": null, + "polygon": true + }, + { + "id": "50792d1d3d070c6a73e9", + "type": "text", + "x": 14.0, + "y": 286.0, + "width": 345.59999999999997, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the _DuringGC functions, new in 3.15", + "originalText": "the _DuringGC functions, new in 3.15", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "ae32037bda366648de5b", + "type": "text", + "x": 406.8, + "y": 286.0, + "width": 403.2, + "height": 20.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "reaching module state without side effects", + "originalText": "reaching module state without side effects", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "fea09c146581d81a7818", + "type": "line", + "x": 0.0, + "y": 316.0, + "width": 910.3999999999999, + "height": 0.0, + "angle": 0, + "strokeColor": "#ced4da", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0.0 + ], + [ + 910.3999999999999, + 0.0 + ] + ], + "lastCommittedPoint": null + }, + { + "id": "2af0443749b68ac6ef92", + "type": "text", + "x": 0.0, + "y": 336.0, + "width": 520.8, + "height": 20.0, + "angle": 0, + "strokeColor": "#5c5f66", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "Anything that can allocate, raise or run Python is not on it.", + "originalText": "Anything that can allocate, raise or run Python is not on it.", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams/what-traverse-may-call.svg b/lessons/r09-writing-a-c-extension-properly/diagrams/what-traverse-may-call.svg new file mode 100644 index 0000000..7d3b17d --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams/what-traverse-may-call.svg @@ -0,0 +1 @@ +The whole list of what a tp_traverse handler is allowed to dowhat you may call in tp_traversewhat it is forthe visit function you were handedreporting one referencePy_VISITthe same thing, with the null check written for youPy_TYPE and Py_SIZEreading the header, valid for this call onlyPyObject_VisitManagedDictreporting an instance dict the runtime ownsPyType_HasFeature and the Check macrosasking what kind of object this isthe _DuringGC functions, new in 3.15reaching module state without side effectsAnything that can allocate, raise or run Python is not on it. diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams/where-the-two-hooks-run.excalidraw b/lessons/r09-writing-a-c-extension-properly/diagrams/where-the-two-hooks-run.excalidraw new file mode 100644 index 0000000..14e0121 --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams/where-the-two-hooks-run.excalidraw @@ -0,0 +1,613 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://github.com/tamnd/cpython-internals", + "elements": [ + { + "id": "4c2df94be9c6973fb530", + "type": "text", + "x": 0.0, + "y": 0.0, + "width": 934.9200000000001, + "height": 30.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "The two ways your object dies, and the one hook that runs either way", + "originalText": "The two ways your object dies, and the one hook that runs either way", + "fontSize": 24, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "410c53c6d73d8b80694b", + "type": "text", + "x": 43.89999999999998, + "y": 50.0, + "width": 376.20000000000005, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the count reached zero on its own", + "originalText": "the count reached zero on its own", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "71904f4fe3df8a2b30fa", + "type": "rectangle", + "x": 0.0, + "y": 87.0, + "width": 464.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#1971c2", + "backgroundColor": "#a5d8ff", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "665148335e0a3e59939c", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "665148335e0a3e59939c", + "type": "text", + "x": 16.0, + "y": 103.0, + "width": 432.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "tp_dealloc is called", + "originalText": "tp_dealloc is called", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "71904f4fe3df8a2b30fa", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "069ee1773cd58d63c8cf", + "type": "rectangle", + "x": 0.0, + "y": 147.0, + "width": 464.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "932836e6306b2bd322aa", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "932836e6306b2bd322aa", + "type": "text", + "x": 16.0, + "y": 163.0, + "width": 432.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "it calls tp_finalize first", + "originalText": "it calls tp_finalize first", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "069ee1773cd58d63c8cf", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "d8da18376f01864dac2b", + "type": "rectangle", + "x": 0.0, + "y": 207.0, + "width": 464.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "a6f4120465fbdf04fbfb", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "a6f4120465fbdf04fbfb", + "type": "text", + "x": 16.0, + "y": 223.0, + "width": 432.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "then it frees the memory", + "originalText": "then it frees the memory", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "d8da18376f01864dac2b", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "d2716d19a6a86f19e148", + "type": "text", + "x": 609.75, + "y": 50.0, + "width": 292.5, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "customData": null, + "text": "the collector found a cycle", + "originalText": "the collector found a cycle", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": true + }, + { + "id": "e239f16e5cbcc0c49259", + "type": "rectangle", + "x": 524.0, + "y": 87.0, + "width": 464.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#099268", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "891062fbaa1fe06d2ffa", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "891062fbaa1fe06d2ffa", + "type": "text", + "x": 540.0, + "y": 103.0, + "width": 432.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "tp_finalize runs on everything first", + "originalText": "tp_finalize runs on everything first", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "e239f16e5cbcc0c49259", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "c9ee8bc0c537defe9387", + "type": "rectangle", + "x": 524.0, + "y": 147.0, + "width": 464.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "873dee0ba72c40a4b385", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "873dee0ba72c40a4b385", + "type": "text", + "x": 540.0, + "y": 163.0, + "width": 432.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "then tp_clear breaks the links", + "originalText": "then tp_clear breaks the links", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "c9ee8bc0c537defe9387", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "3b506944ba249eec67a8", + "type": "rectangle", + "x": 524.0, + "y": 207.0, + "width": 464.0, + "height": 57.0, + "angle": 0, + "strokeColor": "#495057", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "6074da21281ee8a8a740", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "6074da21281ee8a8a740", + "type": "text", + "x": 540.0, + "y": 223.0, + "width": 432.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "counts fall, and tp_dealloc follows", + "originalText": "counts fall, and tp_dealloc follows", + "fontSize": 20, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "middle", + "containerId": "3b506944ba249eec67a8", + "lineHeight": 1.25, + "autoResize": false + }, + { + "id": "6df0e717dc3319792d75", + "type": "rectangle", + "x": 0.0, + "y": 284.0, + "width": 988.0, + "height": 60.0, + "angle": 0, + "strokeColor": "#099268", + "backgroundColor": "#96f2d7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": { + "type": 3 + }, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [ + { + "id": "339995e1faee232871db", + "type": "text" + } + ], + "updated": 1, + "link": null, + "locked": false + }, + { + "id": "339995e1faee232871db", + "type": "text", + "x": 16.0, + "y": 301.5, + "width": 956.0, + "height": 25.0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 1, + "version": 1, + "versionNonce": 1, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "text": "tp_finalize runs at most once per object, and the runtime keeps that flag for you.", + "originalText": "tp_finalize runs at most once per object, and the runtime keeps that flag for you.", + "fontSize": 20, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "6df0e717dc3319792d75", + "lineHeight": 1.25, + "autoResize": false + } + ], + "appState": { + "gridSize": 20, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} diff --git a/lessons/r09-writing-a-c-extension-properly/diagrams/where-the-two-hooks-run.svg b/lessons/r09-writing-a-c-extension-properly/diagrams/where-the-two-hooks-run.svg new file mode 100644 index 0000000..d630e49 --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/diagrams/where-the-two-hooks-run.svg @@ -0,0 +1 @@ +The two ways your object dies, and the one hook that runs either waythe count reached zero on its owntp_dealloc is calledit calls tp_finalize firstthen it frees the memorythe collector found a cycletp_finalize runs on everything firstthen tp_clear breaks the linkscounts fall, and tp_dealloc followstp_finalize runs at most once per object, and the runtime keeps that flag for you. diff --git a/lessons/r09-writing-a-c-extension-properly/r09.ipynb b/lessons/r09-writing-a-c-extension-properly/r09.ipynb new file mode 100644 index 0000000..97d44f4 --- /dev/null +++ b/lessons/r09-writing-a-c-extension-properly/r09.ipynb @@ -0,0 +1,987 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "r09-01", + "metadata": {}, + "source": [ + "# R09. Writing a C extension properly\n", + "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/lessons/r09-writing-a-c-extension-properly/r09.ipynb)\n", + "\n", + "Every lesson so far has watched CPython from the Python side. This one crosses over and writes the C, which is where all the rules you have been reading about stop being descriptions and start being your job.\n", + "\n", + "There are only four of them, and none is hard. Hand back what you own and let go of what you borrowed. If your object can hold another object, tell the collector. Do your cleanup in the slot meant for it. Say honestly whether your code is safe without the lock. What makes them worth a lesson is that breaking any of the four produces no error, no warning and no crash. It produces a program that works, and slowly stops working, in a way that only a tool built for the purpose can see.\n", + "\n", + "![two error paths side by side, one missing a single Py_DECREF](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/r09-writing-a-c-extension-properly/diagrams/the-branch-nobody-tests.svg)" + ] + }, + { + "cell_type": "markdown", + "id": "r09-02", + "metadata": {}, + "source": [ + "## About the source references\n", + "\n", + "Now and then this lesson points at CPython's own source, like this: `Python/gc.c:490-515@v3.15.0rc1`.\n", + "\n", + "Read it as three parts: the file, the lines, and the release those line numbers belong to. Sometimes there is a fourth part after a `#`, which is the name of the thing those lines are inside.\n", + "\n", + "Every reference is a link, and every one is checked against the pinned source on each change, so a stale reference fails the build instead of sending you somewhere wrong. You never have to read any of it. The references are there so you can go deeper when you want to, and so you can check that this lesson is not making things up.\n", + "\n", + "## Setup\n", + "\n", + "Colab does not come with the small package these lessons use, so the next cell installs it. If you are running this from a checkout of the repository it is already installed and the cell does nothing." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r09-03", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "if sys.version_info < (3, 14):\n", + " print(\"This lesson needs CPython 3.14 or newer.\")\n", + " print(f\"This runtime is {sys.version.split()[0]}, and the cells below will not run on it.\")\n", + "else:\n", + " try:\n", + " import pyxray\n", + " except ImportError:\n", + " %pip install -q \"pyxray @ git+https://github.com/tamnd/cpython-internals@main#subdirectory=pyxray\"\n", + " import pyxray" + ] + }, + { + "cell_type": "markdown", + "id": "r09-04", + "metadata": {}, + "source": [ + "## Which Python is this\n", + "\n", + "This lesson compiles C. Colab has a compiler and so does any machine with a working development setup, and a browser tab has neither a compiler nor the headers, so the cells check first and say so rather than failing. If you are reading this in a browser, everything below still reads, and the two recordings at the end are real output from real builds.\n", + "\n", + "## Which interpreter is this" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r09-05", + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, + "outputs": [], + "source": [ + "import pyxray\n", + "\n", + "pyxray.show()" + ] + }, + { + "cell_type": "markdown", + "id": "r09-06", + "metadata": {}, + "source": [ + "## The branch nobody tests\n", + "\n", + "Start with the smallest of the four rules, because everything else is a version of it.\n", + "\n", + "Every C API function that hands you an object hands you one of two things. A [new reference](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#new-reference) is yours, and you owe the runtime a `Py_DECREF` before you lose track of it. A [borrowed reference](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#borrowed-reference) is not yours, and decrefing it is a bug of the other kind. The documentation says which for every function, and there is no way to tell by looking at the call.\n", + "\n", + "Here is a function that gets it right on the path you test and wrong on the path you do not. It packs its argument into a tuple twice over and then hashes the tuple, which works for a number and fails for a list. `PyTuple_Pack` returns a new reference, and that tuple is holding two references to your argument. On the way out through the error the tuple is simply dropped.\n", + "\n", + "Missing one Py_DECREF on an error branch costs two references per failed call, and nothing in the process complains" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r09-07", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import subprocess\n", + "import sysconfig\n", + "import tempfile\n", + "\n", + "ERRPATH = \"\"\"\n", + "#include \n", + "\n", + "/* pair(x) hands back (x, x), but only if that tuple can be hashed. */\n", + "\n", + "static PyObject *\n", + "leaky(PyObject *module, PyObject *arg)\n", + "{\n", + " PyObject *pair = PyTuple_Pack(2, arg, arg);\n", + " if (pair == NULL) {\n", + " return NULL;\n", + " }\n", + " if (PyObject_Hash(pair) == -1) {\n", + " return NULL;\n", + " }\n", + " return pair;\n", + "}\n", + "\n", + "static PyObject *\n", + "careful(PyObject *module, PyObject *arg)\n", + "{\n", + " PyObject *pair = PyTuple_Pack(2, arg, arg);\n", + " if (pair == NULL) {\n", + " return NULL;\n", + " }\n", + " if (PyObject_Hash(pair) == -1) {\n", + " Py_DECREF(pair);\n", + " return NULL;\n", + " }\n", + " return pair;\n", + "}\n", + "\n", + "static PyMethodDef methods[] = {\n", + " {\"leaky\", leaky, METH_O, \"Build a pair, and get the error path wrong.\"},\n", + " {\"careful\", careful, METH_O, \"Build a pair, and get the error path right.\"},\n", + " {NULL, NULL, 0, NULL},\n", + "};\n", + "\n", + "static PyModuleDef_Slot slots[] = {\n", + " {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},\n", + " {Py_mod_gil, Py_MOD_GIL_NOT_USED},\n", + " {0, NULL},\n", + "};\n", + "\n", + "static struct PyModuleDef errpath = {\n", + " PyModuleDef_HEAD_INIT, \"errpath\", NULL, 0, methods, slots, NULL, NULL, NULL,\n", + "};\n", + "\n", + "PyMODINIT_FUNC\n", + "PyInit_errpath(void)\n", + "{\n", + " return PyModuleDef_Init(&errpath);\n", + "}\n", + "\"\"\"\n", + "\n", + "SUFFIX = sysconfig.get_config_var(\"EXT_SUFFIX\") or \".so\"\n", + "WORK = tempfile.mkdtemp()\n", + "sys.path.insert(0, WORK)\n", + "NOTHING = \" no compiler and headers here, so there is nothing to build\"\n", + "\n", + "\n", + "def build(name, source):\n", + " \"\"\"Compile one C file into something this interpreter can import. Empty means it worked.\"\"\"\n", + " written = os.path.join(WORK, name + \".c\")\n", + " with open(written, \"w\") as handle:\n", + " handle.write(source)\n", + " line = [*(sysconfig.get_config_var(\"CC\") or \"cc\").split(), \"-shared\", \"-fPIC\"]\n", + " if sys.platform == \"darwin\":\n", + " line += [\"-undefined\", \"dynamic_lookup\"]\n", + " line += [\"-I\", sysconfig.get_path(\"include\"), written]\n", + " line += [\"-o\", os.path.join(WORK, name + SUFFIX)]\n", + " try:\n", + " done = subprocess.run(line, capture_output=True, text=True, timeout=300)\n", + " except OSError:\n", + " return \"there is no C compiler on this runtime\"\n", + " return \"\" if done.returncode == 0 else done.stderr.strip().splitlines()[-1]\n", + "\n", + "\n", + "TROUBLE = build(\"errpath\", ERRPATH)\n", + "HAVE_C = not TROUBLE\n", + "if not HAVE_C:\n", + " print(NOTHING)\n", + "else:\n", + " import errpath\n", + "\n", + " for name in (\"leaky\", \"careful\"):\n", + " work = getattr(errpath, name)\n", + " victim = []\n", + " before = sys.getrefcount(victim)\n", + " failed = 0\n", + " for _ in range(1000):\n", + " try:\n", + " work(victim)\n", + " except TypeError:\n", + " failed += 1\n", + " left = sys.getrefcount(victim) - before\n", + " print(f\" {name:8} {failed} calls raised, and left {left} references behind\")" + ] + }, + { + "cell_type": "markdown", + "id": "r09-08", + "metadata": {}, + "source": [ + "Two thousand references to a list nobody can reach any more, and the process is perfectly happy. It will stay happy until the machine runs out of memory, which on a web server handling bad input all day is a Tuesday.\n", + "\n", + "The fix is one line, and the habit that produces it is worth more than the line. C has no `finally`, so CPython's own source uses a single exit: every function that owns something has one cleanup block at the bottom, and every failure jumps to it with `goto error`. Once a function has more than one owned object that is the only shape that stays correct, because the alternative is repeating the right sequence of decrefs at every `return NULL` and getting one of them wrong. `Py_CLEAR` [Include/refcount.h:483-500@v3.15.0rc1#Py_CLEAR](https://github.com/python/cpython/blob/v3.15.0rc1/Include/refcount.h#L483-L500) exists for the same reason: it sets the field to `NULL` before dropping the reference, so nothing can see a half freed pointer if the deallocation runs code that comes back around.\n", + "\n", + "## A box the collector cannot see\n", + "\n", + "Now the second rule, which needs a container to show.\n", + "\n", + "The [cycle collector](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#cycle-collector) finds garbage that reference counting cannot: a group of objects pointing at each other with nothing pointing in from outside. It does that by asking every candidate what it is holding, subtracting those references from the counts, and seeing what is left [Python/gc.c:490-515@v3.15.0rc1#subtract_refs](https://github.com/python/cpython/blob/v3.15.0rc1/Python/gc.c#L490-L515) [Python/gc.c:438-465@v3.15.0rc1#visit_decref](https://github.com/python/cpython/blob/v3.15.0rc1/Python/gc.c#L438-L465).\n", + "\n", + "![four steps from counting references to deciding what is held from outside](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/r09-writing-a-c-extension-properly/diagrams/how-the-collector-uses-traverse.svg)\n", + "\n", + "Asking is the [traverse function](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#traverse-function) slot, `tp_traverse`, and there is no way for the runtime to check your answer. Report too little and the collector concludes your object is held from outside and leaves the whole cycle alone. That is the failure mode: not a crash, a leak.\n", + "\n", + "Two lines of C decide whether any of that machinery applies to your type at all. `Py_TPFLAGS_HAVE_GC` [Include/object.h:520-530@v3.15.0rc1#Py_TPFLAGS_HAVE_GC](https://github.com/python/cpython/blob/v3.15.0rc1/Include/object.h#L520-L530) in the flags, and allocation through `PyObject_GC_New` [Python/gc.c:2026-2044@v3.15.0rc1#_PyObject_GC_New](https://github.com/python/cpython/blob/v3.15.0rc1/Python/gc.c#L2026-L2044), which puts a small header in front of your object so the collector has somewhere to keep its bookkeeping.\n", + "\n", + "Here is one container, a box holding a single object, written three ways. The next cell is only the types." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r09-09", + "metadata": {}, + "outputs": [], + "source": [ + "TYPES = \"\"\"\n", + "#include \n", + "#include \n", + "\n", + "typedef struct {\n", + " PyObject_HEAD\n", + " PyObject *item;\n", + "} BoxObject;\n", + "\n", + "typedef struct {\n", + " PyObject *loose;\n", + " PyObject *half;\n", + " PyObject *tracked;\n", + " long freed;\n", + " long finalized;\n", + "} boxes_state;\n", + "\n", + "static struct PyModuleDef boxes_module;\n", + "\n", + "static boxes_state *\n", + "state_of(PyObject *self)\n", + "{\n", + " PyObject *module = PyType_GetModuleByDef(Py_TYPE(self), &boxes_module);\n", + " return module == NULL ? NULL : (boxes_state *)PyModule_GetState(module);\n", + "}\n", + "\n", + "static PyObject *\n", + "box_new(PyTypeObject *type, PyObject *args, PyObject *kwds)\n", + "{\n", + " int tracked = PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC);\n", + " BoxObject *self = tracked ? PyObject_GC_New(BoxObject, type)\n", + " : PyObject_New(BoxObject, type);\n", + " if (self == NULL) {\n", + " return NULL;\n", + " }\n", + " self->item = NULL;\n", + " if (tracked) {\n", + " PyObject_GC_Track(self);\n", + " }\n", + " return (PyObject *)self;\n", + "}\n", + "\n", + "static void\n", + "loose_dealloc(BoxObject *self)\n", + "{\n", + " PyTypeObject *type = Py_TYPE(self);\n", + " boxes_state *state = state_of((PyObject *)self);\n", + " if (state != NULL) {\n", + " state->freed++;\n", + " }\n", + " Py_CLEAR(self->item);\n", + " type->tp_free((PyObject *)self);\n", + " Py_DECREF(type);\n", + "}\n", + "\n", + "static int\n", + "tracked_traverse(BoxObject *self, visitproc visit, void *arg)\n", + "{\n", + " Py_VISIT(Py_TYPE(self));\n", + " Py_VISIT(self->item);\n", + " return 0;\n", + "}\n", + "\n", + "static int\n", + "half_traverse(BoxObject *self, visitproc visit, void *arg)\n", + "{\n", + " Py_VISIT(Py_TYPE(self));\n", + " return 0;\n", + "}\n", + "\n", + "static int\n", + "tracked_clear(BoxObject *self)\n", + "{\n", + " Py_CLEAR(self->item);\n", + " return 0;\n", + "}\n", + "\n", + "static void\n", + "tracked_finalize(PyObject *self)\n", + "{\n", + " boxes_state *state = state_of(self);\n", + " if (state != NULL) {\n", + " state->finalized++;\n", + " }\n", + "}\n", + "\n", + "static void\n", + "tracked_dealloc(BoxObject *self)\n", + "{\n", + " PyTypeObject *type = Py_TYPE(self);\n", + " if (PyObject_CallFinalizerFromDealloc((PyObject *)self) < 0) {\n", + " return;\n", + " }\n", + " PyObject_GC_UnTrack(self);\n", + " boxes_state *state = state_of((PyObject *)self);\n", + " if (state != NULL) {\n", + " state->freed++;\n", + " }\n", + " Py_CLEAR(self->item);\n", + " type->tp_free((PyObject *)self);\n", + " Py_DECREF(type);\n", + "}\n", + "\n", + "static PyMemberDef box_members[] = {\n", + " {\"item\", Py_T_OBJECT_EX, offsetof(BoxObject, item), 0, \"the one thing in the box\"},\n", + " {NULL, 0, 0, 0, NULL},\n", + "};\n", + "\n", + "static PyType_Slot loose_slots[] = {\n", + " {Py_tp_new, box_new},\n", + " {Py_tp_dealloc, loose_dealloc},\n", + " {Py_tp_members, box_members},\n", + " {0, NULL},\n", + "};\n", + "\n", + "static PyType_Spec loose_spec = {\n", + " .name = \"boxes.Loose\",\n", + " .basicsize = sizeof(BoxObject),\n", + " .flags = Py_TPFLAGS_DEFAULT,\n", + " .slots = loose_slots,\n", + "};\n", + "\n", + "static PyType_Slot half_slots[] = {\n", + " {Py_tp_new, box_new},\n", + " {Py_tp_dealloc, tracked_dealloc},\n", + " {Py_tp_traverse, half_traverse},\n", + " {Py_tp_clear, tracked_clear},\n", + " {Py_tp_members, box_members},\n", + " {0, NULL},\n", + "};\n", + "\n", + "static PyType_Spec half_spec = {\n", + " .name = \"boxes.Half\",\n", + " .basicsize = sizeof(BoxObject),\n", + " .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,\n", + " .slots = half_slots,\n", + "};\n", + "\n", + "static PyType_Slot tracked_slots[] = {\n", + " {Py_tp_new, box_new},\n", + " {Py_tp_dealloc, tracked_dealloc},\n", + " {Py_tp_traverse, tracked_traverse},\n", + " {Py_tp_clear, tracked_clear},\n", + " {Py_tp_finalize, tracked_finalize},\n", + " {Py_tp_members, box_members},\n", + " {0, NULL},\n", + "};\n", + "\n", + "static PyType_Spec tracked_spec = {\n", + " .name = \"boxes.Tracked\",\n", + " .basicsize = sizeof(BoxObject),\n", + " .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,\n", + " .slots = tracked_slots,\n", + "};\n", + "\"\"\"\n", + "\n", + "print(f\" {len(TYPES.splitlines())} lines of C, three types, one field each\")" + ] + }, + { + "cell_type": "markdown", + "id": "r09-10", + "metadata": {}, + "source": [ + "Three specs, and the differences are small enough to list. `Loose` has no GC flag, no traverse and no clear, so the collector never hears about it. `Half` has the flag and a `tp_traverse` that reports the type and forgets the field. `Tracked` reports both, and adds a `tp_finalize`.\n", + "\n", + "Reporting the type is not optional decoration. Since 3.9, a [heap type](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#heap-type) is expected to visit `Py_TYPE(self)`, because an instance holds a reference to its own type and a type can be part of a cycle like anything else [Objects/typeobject.c:2607-2645@v3.15.0rc1#subtype_traverse](https://github.com/python/cpython/blob/v3.15.0rc1/Objects/typeobject.c#L2607-L2645).\n", + "\n", + "The rest of the file is the module itself. It is worth reading for one reason: the counters live in [module state](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#module-state) rather than in C statics. A `m_size` above zero gets a struct allocated with every module object [Objects/moduleobject.c:1027-1035@v3.15.0rc1#PyModule_GetState](https://github.com/python/cpython/blob/v3.15.0rc1/Objects/moduleobject.c#L1027-L1035), and a type made with `PyType_FromModuleAndSpec` can find its way back to it [Objects/typeobject.c:5973-6005@v3.15.0rc1#PyType_GetModuleByDef](https://github.com/python/cpython/blob/v3.15.0rc1/Objects/typeobject.c#L5973-L6005). Statics would be shared across every [subinterpreter](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#subinterpreter) in the process, which is a bug waiting for somebody else to find." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r09-11", + "metadata": {}, + "outputs": [], + "source": [ + "MODULE = \"\"\"\n", + "static PyObject *\n", + "counts(PyObject *module, PyObject *unused)\n", + "{\n", + " boxes_state *state = (boxes_state *)PyModule_GetState(module);\n", + " return Py_BuildValue(\"(ll)\", state->freed, state->finalized);\n", + "}\n", + "\n", + "static PyObject *\n", + "reset(PyObject *module, PyObject *unused)\n", + "{\n", + " boxes_state *state = (boxes_state *)PyModule_GetState(module);\n", + " state->freed = 0;\n", + " state->finalized = 0;\n", + " Py_RETURN_NONE;\n", + "}\n", + "\n", + "static PyMethodDef boxes_methods[] = {\n", + " {\"counts\", counts, METH_NOARGS, \"How many boxes were freed and finalized so far.\"},\n", + " {\"reset\", reset, METH_NOARGS, \"Put both counters back to zero.\"},\n", + " {NULL, NULL, 0, NULL},\n", + "};\n", + "\n", + "static int\n", + "add(PyObject *module, PyObject **slot, PyType_Spec *spec, const char *name)\n", + "{\n", + " *slot = PyType_FromModuleAndSpec(module, spec, NULL);\n", + " if (*slot == NULL) {\n", + " return -1;\n", + " }\n", + " return PyModule_AddObjectRef(module, name, *slot);\n", + "}\n", + "\n", + "static int\n", + "boxes_exec(PyObject *module)\n", + "{\n", + " boxes_state *state = (boxes_state *)PyModule_GetState(module);\n", + " if (add(module, &state->loose, &loose_spec, \"Loose\") < 0) {\n", + " return -1;\n", + " }\n", + " if (add(module, &state->half, &half_spec, \"Half\") < 0) {\n", + " return -1;\n", + " }\n", + " return add(module, &state->tracked, &tracked_spec, \"Tracked\");\n", + "}\n", + "\n", + "static int\n", + "boxes_traverse(PyObject *module, visitproc visit, void *arg)\n", + "{\n", + " boxes_state *state = (boxes_state *)PyModule_GetState(module);\n", + " Py_VISIT(state->loose);\n", + " Py_VISIT(state->half);\n", + " Py_VISIT(state->tracked);\n", + " return 0;\n", + "}\n", + "\n", + "static int\n", + "boxes_clear(PyObject *module)\n", + "{\n", + " boxes_state *state = (boxes_state *)PyModule_GetState(module);\n", + " Py_CLEAR(state->loose);\n", + " Py_CLEAR(state->half);\n", + " Py_CLEAR(state->tracked);\n", + " return 0;\n", + "}\n", + "\n", + "static PyModuleDef_Slot boxes_slots[] = {\n", + " {Py_mod_exec, boxes_exec},\n", + " {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},\n", + " /* Honest, because two threads bumping state->freed would lose counts. */\n", + " {Py_mod_gil, Py_MOD_GIL_USED},\n", + " {0, NULL},\n", + "};\n", + "\n", + "static struct PyModuleDef boxes_module = {\n", + " PyModuleDef_HEAD_INIT,\n", + " .m_name = \"boxes\",\n", + " .m_size = sizeof(boxes_state),\n", + " .m_methods = boxes_methods,\n", + " .m_slots = boxes_slots,\n", + " .m_traverse = boxes_traverse,\n", + " .m_clear = boxes_clear,\n", + "};\n", + "\n", + "PyMODINIT_FUNC\n", + "PyInit_boxes(void)\n", + "{\n", + " return PyModuleDef_Init(&boxes_module);\n", + "}\n", + "\"\"\"\n", + "\n", + "if not HAVE_C:\n", + " print(NOTHING)\n", + "else:\n", + " TROUBLE = build(\"boxes\", TYPES + MODULE)\n", + " HAVE_C = not TROUBLE\n", + " if HAVE_C:\n", + " import boxes\n", + "\n", + " made = [name for name in (\"Loose\", \"Half\", \"Tracked\") if hasattr(boxes, name)]\n", + " print(f\" built and imported, with these types: {made}\")\n", + " else:\n", + " print(f\" the compiler said: {TROUBLE}\")" + ] + }, + { + "cell_type": "markdown", + "id": "r09-12", + "metadata": {}, + "source": [ + "Now put one of each in a cycle. A box whose `item` is the box itself has a reference count of one that nothing outside can reach, which is exactly the case reference counting alone cannot handle. Drop the name, run a collection, and ask the module how many boxes it has freed.\n", + "\n", + "The GC flag on its own is not enough. A tp_traverse that forgets a field leaks just as completely as no tp_traverse at all" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r09-13", + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "\n", + "KINDS = (\"Loose\", \"Half\", \"Tracked\")\n", + "\n", + "if not HAVE_C:\n", + " print(NOTHING)\n", + "else:\n", + " print(\" type collector sees it traverse reports freed after a collection\")\n", + " for name in KINDS:\n", + " kind = getattr(boxes, name)\n", + " boxes.reset()\n", + " box = kind()\n", + " seen = gc.is_tracked(box)\n", + " box.item = box\n", + " reports = len(gc.get_referents(box))\n", + " del box\n", + " gc.collect()\n", + " print(f\" {name:9} {seen!s:19} {reports:16} {boxes.counts()[0]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "r09-14", + "metadata": {}, + "source": [ + "`Loose` is invisible. `Half` is visible and reports one thing, its type, so the collector subtracts one reference from a box whose count is two and decides somebody outside still wants it. `Tracked` reports both and gets collected. The three rows differ by four lines of C.\n", + "\n", + "`gc.get_referents` in that table is `tp_traverse` called directly, which is the closest thing to a test you can write for it. If the numbers there do not match the fields your object owns, you have the `Half` bug.\n", + "\n", + "![a table of three container types with what the collector sees of each and whether it was freed](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/r09-writing-a-c-extension-properly/diagrams/three-boxes-one-cycle.svg)\n", + "\n", + "There is a short list of what a `tp_traverse` handler may call, and it is short for a reason: traversal runs in the middle of a collection, where allocating, raising or running Python would be a disaster. Anything that could set an exception is out, which is why 3.15 added a whole family of functions ending in `_DuringGC` that do the same job as their ordinary versions and promise to have no side effects and never raise.\n", + "\n", + "![a table of the six things a tp_traverse handler is allowed to call](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/r09-writing-a-c-extension-properly/diagrams/what-traverse-may-call.svg)\n", + "\n", + "## Where cleanup goes\n", + "\n", + "`Tracked` has a third slot the other two do not, and it is the answer to a question every container eventually raises: where does cleanup go when the object is part of a cycle?\n", + "\n", + "The [clear function](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#clear-function), `tp_clear`, is what the collector calls to break the cycle [Python/gc.c:1083-1120@v3.15.0rc1#delete_garbage](https://github.com/python/cpython/blob/v3.15.0rc1/Python/gc.c#L1083-L1120). It drops exactly what `tp_traverse` reported. It is not the place for closing a file or releasing a lock, because it can be called while the rest of the cycle is being emptied around it.\n", + "\n", + "`tp_finalize` is that place. It is the modern `__del__`: it runs once per object, before anything is torn down, on both routes to death. When a count reaches zero, `tp_dealloc` calls it on the way in [Objects/object.c:595-630@v3.15.0rc1#PyObject_CallFinalizerFromDealloc](https://github.com/python/cpython/blob/v3.15.0rc1/Objects/object.c#L595-L630). When the collector finds a cycle, it runs `tp_finalize` on the whole unreachable set first, then starts clearing. The runtime keeps a [finalized bit](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#finalized-bit) so it cannot happen twice.\n", + "\n", + "![the two routes an object takes to being freed and where each hook runs on it](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/r09-writing-a-c-extension-properly/diagrams/where-the-two-hooks-run.svg)\n", + "\n", + "A tp_finalize runs exactly once per object whether the object died on a reference count or inside a cycle" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r09-15", + "metadata": {}, + "outputs": [], + "source": [ + "if not HAVE_C:\n", + " print(NOTHING)\n", + "else:\n", + " boxes.reset()\n", + " plain = boxes.Tracked()\n", + " plain.item = [1, 2, 3]\n", + " del plain\n", + " freed, final = boxes.counts()\n", + " print(f\" died on its reference count: finalized {final}, freed {freed}\")\n", + "\n", + " boxes.reset()\n", + " looped = boxes.Tracked()\n", + " looped.item = looped\n", + " del looped\n", + " gc.collect()\n", + " freed, final = boxes.counts()\n", + " print(f\" died inside a cycle: finalized {final}, freed {freed}\")" + ] + }, + { + "cell_type": "markdown", + "id": "r09-16", + "metadata": {}, + "source": [ + "One hook, both routes, once each. That is the whole reason `tp_finalize` replaced the old `tp_del` slot, which M09 covered from the collector's side: an object with the old slot in a cycle used to be uncollectable and got put on `gc.garbage` for a human to deal with.\n", + "\n", + "## The promise at the bottom of the file\n", + "\n", + "The fourth rule is one line, and on most builds it does nothing at all.\n", + "\n", + "An [extension module](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#extension-module) on a [free threaded build](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#free-threaded-build) has to say whether it is safe without the lock. `Py_mod_gil` with `Py_MOD_GIL_NOT_USED` is a promise. Saying nothing, or saying `Py_MOD_GIL_USED`, makes the runtime turn the lock back on for the whole process the moment your module is imported, and print a warning naming you.\n", + "\n", + "The `boxes` module above says `Py_MOD_GIL_USED`, and that is not modesty. Two threads calling `state->freed++` would lose counts, because that is a read, an add and a write with nothing in between. Keeping the promise instead would mean a [critical section](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#critical-section) around every field the two types share, which C02 and C06 went through from the reading side. `errpath` says `Py_MOD_GIL_NOT_USED` and means it, because it has no state at all.\n", + "\n", + "One imported module that has not declared itself safe turns the lock back on for the whole process\n", + "\n", + "CPython's own test suite ships a shared object that exports two init functions differing only in that line, which makes the difference easy to weigh without compiling anything. This recording loads each of them on a free threaded build, in a child of its own so the answer is not contaminated by the previous one.\n", + "\n", + "What is the line at the bottom of an extension module that declares itself safe worth?\n", + "\n", + "```python\n", + "\"\"\"What a module has to declare before a free threaded build will trust it.\n", + "\n", + "CPython's own test suite ships a shared object that exports several init functions, two\n", + "of which differ only in one line: whether they say the module can run without the lock.\n", + "Loading each one in a child of its own shows what that line is worth.\n", + "\"\"\"\n", + "\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "import textwrap\n", + "\n", + "CHILD = \"\"\"\n", + "import importlib.machinery, importlib.util, sys, _testmultiphase\n", + "\n", + "name = sys.argv[1]\n", + "print(f\" the lock before the import: {sys._is_gil_enabled()}\")\n", + "loader = importlib.machinery.ExtensionFileLoader(name, _testmultiphase.__file__)\n", + "spec = importlib.util.spec_from_loader(name, loader)\n", + "loader.exec_module(importlib.util.module_from_spec(spec))\n", + "print(f\" the lock after the import: {sys._is_gil_enabled()}\")\n", + "\"\"\"\n", + "\n", + "CASES = (\n", + " (\"_test_from_modexport\", \"which declares Py_MOD_GIL_NOT_USED\", {}),\n", + " (\"_test_from_modexport_gil_used\", \"which declares Py_MOD_GIL_USED\", {}),\n", + " (\"_test_from_modexport_gil_used\", \"the same one, in a child started with PYTHON_GIL=0\", {}),\n", + ")\n", + "\n", + "OVERRIDE = {\"PYTHON_GIL\": \"0\"}\n", + "\n", + "\n", + "def load(name, extra):\n", + " \"\"\"Import one init function out of that shared object, in a child of its own.\"\"\"\n", + " return subprocess.run(\n", + " [sys.executable, \"-c\", CHILD, name],\n", + " capture_output=True,\n", + " text=True,\n", + " timeout=180,\n", + " env=os.environ | {\"PYTHON_COLORS\": \"0\"} | extra,\n", + " )\n", + "\n", + "\n", + "print(\"what a compiled module has to declare before this build will trust it\")\n", + "print()\n", + "print(f\" the abi flags on this build: {sys.abiflags!r}\")\n", + "print()\n", + "\n", + "turned_on = 0\n", + "for position, (name, what, _) in enumerate(CASES):\n", + " done = load(name, OVERRIDE if position == 2 else {})\n", + " print(f\" {name}, {what}\")\n", + " print(done.stdout, end=\"\")\n", + " turned_on += \"the lock after the import: True\" in done.stdout\n", + " said = [one for one in done.stderr.splitlines() if \"RuntimeWarning\" in one]\n", + " if not said:\n", + " print(\" it went through without a word\")\n", + " for one in said:\n", + " print(\" on the way it warned, at some length:\")\n", + " text = one.split(\"RuntimeWarning: \")[1]\n", + " print(textwrap.indent(textwrap.fill(text, 74), \" \"))\n", + " print()\n", + "\n", + "print(f\"~ ways of loading the same shared object: {len(CASES)}\")\n", + "print(f\"~ of those that turned the lock back on: {turned_on}\")\n", + "```\n", + "\n", + "```text\n", + "what a compiled module has to declare before this build will trust it\n", + "\n", + " the abi flags on this build: 't'\n", + "\n", + " _test_from_modexport, which declares Py_MOD_GIL_NOT_USED\n", + " the lock before the import: False\n", + " the lock after the import: False\n", + " it went through without a word\n", + "\n", + " _test_from_modexport_gil_used, which declares Py_MOD_GIL_USED\n", + " the lock before the import: False\n", + " the lock after the import: True\n", + " on the way it warned, at some length:\n", + " The global interpreter lock (GIL) has been enabled to load module\n", + " '_test_from_modexport_gil_used', which has not declared that it can run\n", + " safely without the GIL. To override this behavior and keep the GIL\n", + " disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0.\n", + "\n", + " _test_from_modexport_gil_used, the same one, in a child started with PYTHON_GIL=0\n", + " the lock before the import: False\n", + " the lock after the import: False\n", + " it went through without a word\n", + "\n", + "~ ways of loading the same shared object: 3\n", + "~ of those that turned the lock back on: 1\n", + "```\n", + "\n", + "That ran on Python 3.15.0rc1 in the freethreaded build this project publishes, which is `ghcr.io/tamnd/cpython-internals/cpython:freethreaded@sha256:db72284e3a49f43c38b96bec2baed1380b8348e27ea6f54f6e8d0810b59c3144`. You do not need that build to read the numbers, and you do need it to produce them, which is why this is recorded rather than left as a cell you run. If you want to watch it happen yourself, `docker run --rm -i ghcr.io/tamnd/cpython-internals/cpython:freethreaded@sha256:db72284e3a49f43c38b96bec2baed1380b8348e27ea6f54f6e8d0810b59c3144 python3 -` takes the program on standard input.\n", + "\n", + "The third case is the escape hatch. `PYTHON_GIL=0` tells the runtime you have read the module's source yourself and disagree with its declaration, and the warning goes away with the lock. It is exactly as safe as your reading was.\n", + "\n", + "## Proving it\n", + "\n", + "Four rules, and every way of breaking them looks like working code. So the last thing to write is the test that would have caught any of it.\n", + "\n", + "The idea fits in a handful of lines. Give the thing you are watching a type of its own so it is easy to count. Collect. Count. Do the work. Collect and count again. Repeat, and throw the first few rounds away. If the count goes up by the same amount every single time, that is a leak; if it goes up once and settles, that is a cache filling.\n", + "\n", + "An ordinary test run cannot see a reference leak at all, which is why CPython runs its own suite a second way" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "r09-17", + "metadata": {}, + "outputs": [], + "source": [ + "class Held:\n", + " \"\"\"Something for a box to hold, so that the thing being leaked has a name.\"\"\"\n", + "\n", + "\n", + "def alive():\n", + " \"\"\"How many Held objects the collector can still find anywhere in this process.\"\"\"\n", + " return sum(1 for one in gc.get_objects() if isinstance(one, Held))\n", + "\n", + "\n", + "def hunt(work, repeats=6, warmups=3):\n", + " \"\"\"Run something over and over and report what each run left behind.\"\"\"\n", + " deltas = []\n", + " for _ in range(repeats):\n", + " gc.collect()\n", + " before = alive()\n", + " work()\n", + " gc.collect()\n", + " deltas.append(alive() - before)\n", + " return deltas[warmups:]\n", + "\n", + "\n", + "def a_cycle(kind):\n", + " \"\"\"One box and one Held pointing at each other, with no name left for either.\"\"\"\n", + "\n", + " def work():\n", + " box = kind()\n", + " held = Held()\n", + " held.box = box\n", + " box.item = held\n", + "\n", + " return work\n", + "\n", + "\n", + "if not HAVE_C:\n", + " print(NOTHING)\n", + "else:\n", + " for name in KINDS:\n", + " left = hunt(a_cycle(getattr(boxes, name)))\n", + " verdict = \"a leak\" if all(one >= 1 for one in left) else \"fine\"\n", + " print(f\" {name:9} {left!s:12} {verdict}\")" + ] + }, + { + "cell_type": "markdown", + "id": "r09-18", + "metadata": {}, + "source": [ + "That is CPython's own `-R` flag with the interesting parts taken out. The real one counts references and allocated blocks and open file descriptors rather than instances of one class you picked, which means it needs a [debug build](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#debug-build), since an ordinary build keeps no running total of references. It throws away the first few runs, because caches fill on first use and that is not a leak. And it only calls something a failure if every measured run leaked at least one [Lib/test/libregrtest/refleak.py:196-209@v3.15.0rc1#check_rc_deltas](https://github.com/python/cpython/blob/v3.15.0rc1/Lib/test/libregrtest/refleak.py#L196-L209), then prints the deltas it saw [Lib/test/libregrtest/refleak.py:214-236@v3.15.0rc1#leaked](https://github.com/python/cpython/blob/v3.15.0rc1/Lib/test/libregrtest/refleak.py#L214-L236).\n", + "\n", + "![four tests with the verdict an ordinary run gives and the verdict the leak hunter gives](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/r09-writing-a-c-extension-properly/diagrams/what-the-hunter-counts.svg)\n", + "\n", + "This recording runs four small tests on a debug build, once the way anybody runs a test suite and once with `-R 3:3`, which means three warm up runs and three measured ones.\n", + "\n", + "What does CPython's own leak hunter see that an ordinary test run walks straight past?\n", + "\n", + "```python\n", + "\"\"\"What the reference leak hunter catches that an ordinary run does not.\n", + "\n", + "Four small tests, written into a directory of their own, run twice each. Once the way\n", + "anybody runs a test suite, and once under the flag CPython's buildbots use, which runs\n", + "every test six times and counts what the interpreter is still holding afterwards.\n", + "\"\"\"\n", + "\n", + "import os\n", + "import pathlib\n", + "import re\n", + "import subprocess\n", + "import sys\n", + "import tempfile\n", + "\n", + "CASES = {\n", + " \"test_clean\": \"\"\"\n", + "import unittest\n", + "\n", + "\n", + "class Clean(unittest.TestCase):\n", + " def test_it(self):\n", + " held = [object()]\n", + " self.assertTrue(held)\n", + "\"\"\",\n", + " \"test_cached\": \"\"\"\n", + "import unittest\n", + "\n", + "CACHE = []\n", + "\n", + "\n", + "class Cached(unittest.TestCase):\n", + " def test_it(self):\n", + " if len(CACHE) < 4:\n", + " CACHE.append(object())\n", + " self.assertTrue(CACHE)\n", + "\"\"\",\n", + " \"test_leaky\": \"\"\"\n", + "import unittest\n", + "\n", + "HOARD = []\n", + "\n", + "\n", + "class Leaky(unittest.TestCase):\n", + " def test_it(self):\n", + " HOARD.append(object())\n", + " self.assertTrue(HOARD)\n", + "\"\"\",\n", + " \"test_handles\": \"\"\"\n", + "import os\n", + "import unittest\n", + "\n", + "\n", + "class Handles(unittest.TestCase):\n", + " def test_it(self):\n", + " copy = os.dup(0)\n", + " self.assertGreater(copy, 0)\n", + "\"\"\",\n", + "}\n", + "\n", + "REPORT = re.compile(r\"^\\w+ leaked (\\[[^]]*\\]) ([a-z ]+), sum=(-?\\d+)(.*)$\")\n", + "\n", + "ROOT = pathlib.Path(tempfile.mkdtemp())\n", + "for name, body in CASES.items():\n", + " (ROOT / f\"{name}.py\").write_text(body)\n", + "\n", + "\n", + "def run(name, flags=()):\n", + " \"\"\"Run one of the four tests, with or without the leak hunting flag.\"\"\"\n", + " return subprocess.run(\n", + " [sys.executable, \"-m\", \"test\", \"--testdir\", str(ROOT), *flags, name],\n", + " capture_output=True,\n", + " text=True,\n", + " timeout=600,\n", + " env=os.environ | {\"PYTHON_COLORS\": \"0\"},\n", + " )\n", + "\n", + "\n", + "print(\"four small tests, run the way anybody runs a test suite\")\n", + "print()\n", + "ordinary = 0\n", + "for name in CASES:\n", + " done = run(name)\n", + " ordinary += done.returncode == 0\n", + " print(f\" {name:14} {'failed' if done.returncode else 'passed'}\")\n", + "\n", + "print()\n", + "print(\"the same four, run six times each with the leak hunter watching\")\n", + "print()\n", + "hunted = 0\n", + "for name in CASES:\n", + " done = run(name, (\"-R\", \"3:3\"))\n", + " hunted += done.returncode == 0\n", + " said = [m for m in map(REPORT.match, done.stderr.splitlines()) if m]\n", + " counts = \", \".join(f\"{m.group(1)} {m.group(2)}\" for m in said)\n", + " excused = \" (which the hunter calls fine)\" if said and said[0].group(4) else \"\"\n", + " verdict = \"failed\" if done.returncode else \"passed\"\n", + " print(f\" {name:14} {verdict:7} {counts or 'nothing was left behind'}{excused}\")\n", + "\n", + "print()\n", + "print(f\"~ tests that pass an ordinary run: {ordinary} of {len(CASES)}\")\n", + "print(f\"~ tests that pass the leak hunter: {hunted} of {len(CASES)}\")\n", + "print(\"~ repetitions the hunter ran each test for: 6\")\n", + "print(\"~ of those repetitions that were warm ups: 3\")\n", + "```\n", + "\n", + "```text\n", + "four small tests, run the way anybody runs a test suite\n", + "\n", + " test_clean passed\n", + " test_cached passed\n", + " test_leaky passed\n", + " test_handles passed\n", + "\n", + "the same four, run six times each with the leak hunter watching\n", + "\n", + " test_clean passed nothing was left behind\n", + " test_cached passed [1, 0, 0] references, [1, 0, 0] memory blocks (which the hunter calls fine)\n", + " test_leaky failed [1, 1, 1] references, [1, 1, 1] memory blocks\n", + " test_handles failed [1, 1, 1] file descriptors\n", + "\n", + "~ tests that pass an ordinary run: 4 of 4\n", + "~ tests that pass the leak hunter: 2 of 4\n", + "~ repetitions the hunter ran each test for: 6\n", + "~ of those repetitions that were warm ups: 3\n", + "```\n", + "\n", + "That ran on Python 3.15.0rc1 in the debug build this project publishes, which is `ghcr.io/tamnd/cpython-internals/cpython:debug@sha256:7baea8f3dd4de2e4c3b020543729b147e636494ae9758dabffb4675793e37170`. You do not need that build to read the numbers, and you do need it to produce them, which is why this is recorded rather than left as a cell you run. If you want to watch it happen yourself, `docker run --rm -i ghcr.io/tamnd/cpython-internals/cpython:debug@sha256:7baea8f3dd4de2e4c3b020543729b147e636494ae9758dabffb4675793e37170 python3 -` takes the program on standard input.\n", + "\n", + "All four pass the first time. Two fail the second, and one of the two that passes still gets a note, because it left something behind on the first measured run and then stopped. That last case is the reason the hunter is careful rather than strict, and also the reason a first attempt at your own leak test will produce noise.\n", + "\n", + "## Try it yourself\n", + "\n", + "Change `half_traverse` to report `self->item` and nothing else, so it visits the field but forgets the type. Rebuild and rerun the table. The box is still collected, because nothing in these examples puts a type in a cycle, which is exactly why the rule about visiting `Py_TYPE(self)` is easy to break and hard to notice.\n", + "\n", + "Take the `Py_DECREF` out of `tracked_dealloc`'s `Py_CLEAR(self->item)` and run the hunter over `Tracked` again. You have written the `Half` bug a second way, with a complete `tp_traverse` and a `tp_dealloc` that does not finish the job.\n", + "\n", + "Add a `tp_finalize` to `Loose` and put one in a cycle. Nothing runs, because a type the collector cannot see never gets a second chance.\n", + "\n", + "Give `Tracked` a `__del__` written in Python by subclassing it, then put an instance in a cycle. Compare what runs against the C `tp_finalize` on the same object, and in which order.\n", + "\n", + "## What you now know\n", + "\n", + "An error branch that forgets one `Py_DECREF` leaks silently and forever, and the shape that prevents it is a single cleanup block reached by `goto`.\n", + "\n", + "A container type needs three things before the collector can help it: the `Py_TPFLAGS_HAVE_GC` flag, allocation through `PyObject_GC_New`, and a `tp_traverse` that reports every field, including the type of a heap type. Miss the flag and you are invisible. Miss a field and you are visible and lying, which leaks just as thoroughly.\n", + "\n", + "`tp_clear` breaks cycles and belongs to the collector. `tp_finalize` is where your cleanup goes, and it runs once per object on either route to death.\n", + "\n", + "Module state exists so that C extensions do not keep things in statics, which subinterpreters share.\n", + "\n", + "`Py_mod_gil` is a promise about thread safety that the runtime cannot check, so it believes you, and turns the lock back on for everybody if you decline to make it.\n", + "\n", + "None of these failures produce an error. The tool that finds them runs your tests six times on a debug build and counts what is left, and it is the difference between code that passes review and code that runs for a year.\n", + "\n", + "## What is next\n", + "\n", + "That is the end of the runtime part, and the end of watching CPython work. R01 asked what happens before your first line, R08 asked what happens after your last one, and this one asked what the runtime expects from code you write in its own language.\n", + "\n", + "What is left in this milestone is not lessons. It is the blueprints: the written down descriptions of state layout, the object model under free threading, the import system and the C API surface, precise enough that somebody could build them again in another language. The reimplementation track after that is the whole point of writing any of this down." + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/probes/pyodide/lessons.json b/probes/pyodide/lessons.json index 69152a1..ecd344f 100644 --- a/probes/pyodide/lessons.json +++ b/probes/pyodide/lessons.json @@ -70,7 +70,7 @@ { "name": "b03-07", "status": "ok", - "printed": "test_adding (__main__.Arithmetic.test_adding) ... ok\ntest_dividing (__main__.Arithmetic.test_dividing) ... ok\ntest_powers (__main__.Arithmetic.test_powers) ... FAIL\n\n======================================================================\nFAIL: test_powers (__main__.Arithmetic.test_powers)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"b03-07\", line 13, in test_powers\nAssertionError: 1024 != 1000\n\n----------------------------------------------------------------------\nRan 3 tests in 0.002s\n\nFAILED (failures=1)\n\n3 tests run, 1 failed, 0 errored\n" + "printed": "test_adding (__main__.Arithmetic.test_adding) ... ok\ntest_dividing (__main__.Arithmetic.test_dividing) ... ok\ntest_powers (__main__.Arithmetic.test_powers) ... FAIL\n\n======================================================================\nFAIL: test_powers (__main__.Arithmetic.test_powers)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"b03-07\", line 13, in test_powers\nAssertionError: 1024 != 1000\n\n----------------------------------------------------------------------\nRan 3 tests in 0.003s\n\nFAILED (failures=1)\n\n3 tests run, 1 failed, 0 errored\n" }, { "name": "b03-10", @@ -329,7 +329,7 @@ { "name": "c05-18", "status": "ok", - "printed": " the collector is scheduled when generation zero passes 2000\n json.loads built them in 167 ms, collector ran 1 times\n a list display did it in 172 ms, collector ran 89 times\n the two of them built the same number of dicts: True\n" + "printed": " the collector is scheduled when generation zero passes 2000\n json.loads built them in 86 ms, collector ran 1 times\n a list display did it in 247 ms, collector ran 89 times\n the two of them built the same number of dicts: True\n" }, { "name": "c05-22", @@ -622,7 +622,7 @@ { "name": "e04-09", "status": "ok", - "printed": " a plain object at 23132824 mod 4 0 mod 8 0\n a list at 26373664 mod 4 0 mod 8 0\n a dict at 23276920 mod 4 0 mod 8 0\n a string at 19195008 mod 4 0 mod 8 0\n a big integer at 21709904 mod 4 0 mod 8 0\n a float at 23423504 mod 4 0 mod 8 0\n a module at 14795336 mod 4 0 mod 8 0\n None at 3200984 mod 4 0 mod 8 0\n the number 7 at 3253660 mod 4 0 mod 8 4\n" + "printed": " a plain object at 25164368 mod 4 0 mod 8 0\n a list at 24631392 mod 4 0 mod 8 0\n a dict at 22116024 mod 4 0 mod 8 0\n a string at 20337712 mod 4 0 mod 8 0\n a big integer at 22602128 mod 4 0 mod 8 0\n a float at 21677000 mod 4 0 mod 8 0\n a module at 14795336 mod 4 0 mod 8 0\n None at 3200984 mod 4 0 mod 8 0\n the number 7 at 3253660 mod 4 0 mod 8 4\n" }, { "name": "e04-12", @@ -691,12 +691,12 @@ { "name": "e05-18", "status": "ok", - "printed": " no_try 158.6 ns per iteration\n with_try 145.6 ns per iteration\n checking 159.8 ns per iteration\n" + "printed": " no_try 55.9 ns per iteration\n with_try 58.0 ns per iteration\n checking 128.3 ns per iteration\n" }, { "name": "e05-21", "status": "ok", - "printed": " through 1 frames 792.8 ns\n through 5 frames 3393.8 ns\n through 20 frames 9703.6 ns\n through 50 frames 20547.8 ns\n" + "printed": " through 1 frames 648.8 ns\n through 5 frames 1241.1 ns\n through 20 frames 3571.3 ns\n through 50 frames 8376.8 ns\n" }, { "name": "e05-24", @@ -711,7 +711,7 @@ { "name": "e05-29", "status": "ok", - "printed": " for loop table 0 bytes 100.2 ns per item\n while and next table 12 bytes 105.2 ns per item\n" + "printed": " for loop table 0 bytes 34.2 ns per item\n while and next table 12 bytes 50.6 ns per item\n" }, { "name": "e05-32", @@ -780,7 +780,7 @@ { "name": "e06-31", "status": "ok", - "printed": " every value an int 26.9 ns per addition\n every value a float 31.3 ns per addition\n alternating between the two 52.6 ns per addition\n" + "printed": " every value an int 21.6 ns per addition\n every value a float 21.3 ns per addition\n alternating between the two 36.7 ns per addition\n" }, { "name": "e06-34", @@ -834,7 +834,7 @@ { "name": "e07-25", "status": "ok", - "printed": " with the jit on\n this build has no JIT in it, so there is nothing to switch on\n\n with the jit off, for comparison\n arithmetic loop 140.5 ns per iteration\n loop with a call 173.6 ns per iteration\n" + "printed": " with the jit on\n this build has no JIT in it, so there is nothing to switch on\n\n with the jit off, for comparison\n arithmetic loop 57.8 ns per iteration\n loop with a call 60.3 ns per iteration\n" } ] }, @@ -863,7 +863,7 @@ { "name": "e08-11", "status": "ok", - "printed": " nobody watching 108.0 ns per turn\n callback counts the line 902.1 ns per turn, 2000015 calls\n callback returns DISABLE 60.9 ns per turn\n" + "printed": " nobody watching 43.5 ns per turn\n callback counts the line 258.0 ns per turn, 2000015 calls\n callback returns DISABLE 62.5 ns per turn\n" }, { "name": "e08-14", @@ -1020,17 +1020,17 @@ { "name": "e11-15", "status": "ok", - "printed": " extra lines instructions run nanoseconds cost of one more\n 0 7010 93766 \n 5 17010 119252 2.55\n 10 27010 400316 28.11\n 20 47010 772923 18.63\n 40 87010 1023695 6.27\n" + "printed": " extra lines instructions run nanoseconds cost of one more\n 0 7010 33821 \n 5 17010 51467 1.76\n 10 27010 160126 10.87\n 20 47010 246341 4.31\n 40 87010 419386 4.33\n" }, { "name": "e11-18", "status": "ok", - "printed": " the same job instructions nanoseconds each\n a while loop 20012 193166 9.65\n a for loop 10008 67736 6.77\n a comprehension 6011 62086 10.33\n" + "printed": " the same job instructions nanoseconds each\n a while loop 20012 155662 7.78\n a for loop 10008 67495 6.74\n a comprehension 6011 53233 8.86\n" }, { "name": "e11-21", "status": "ok", - "printed": " one bytecode instruction 14.49 ns\n one call to an empty def 86.62 ns beyond the code that calls it\n so a call costs about 6 instructions\n" + "printed": " one bytecode instruction 3.89 ns\n one call to an empty def 19.34 ns beyond the code that calls it\n so a call costs about 5 instructions\n" } ] }, @@ -1489,7 +1489,7 @@ { "name": "f10-11", "status": "ok", - "printed": " -- MAKE_CELL 0 (a)\n MAKE_CELL 6 (n)\n\n 1 RESUME 0\n\n 2 LOAD_SMALL_INT 0\n STORE_DEREF 6 (n)\n\n 4 LOAD_FAST_BORROW 0 (a)\n LOAD_FAST_BORROW 6 (n)\n BUILD_TUPLE 2\n LOAD_CONST 1 ()\n MAKE_FUNCTION\n SET_FUNCTION_ATTRIBUTE 8 (closure)\n STORE_FAST 5 (inner)\n\n 7 LOAD_FAST_BORROW 5 (inner)\n RETURN_VALUE\n\nDisassembly of :\n -- COPY_FREE_VARS 2\n\n 4 RESUME 0\n\n 5 LOAD_DEREF 0 (a)\n LOAD_DEREF 1 (n)\n BINARY_OP 0 (+)\n RETURN_VALUE\n" + "printed": " -- MAKE_CELL 0 (a)\n MAKE_CELL 6 (n)\n\n 1 RESUME 0\n\n 2 LOAD_SMALL_INT 0\n STORE_DEREF 6 (n)\n\n 4 LOAD_FAST_BORROW 0 (a)\n LOAD_FAST_BORROW 6 (n)\n BUILD_TUPLE 2\n LOAD_CONST 1 ()\n MAKE_FUNCTION\n SET_FUNCTION_ATTRIBUTE 8 (closure)\n STORE_FAST 5 (inner)\n\n 7 LOAD_FAST_BORROW 5 (inner)\n RETURN_VALUE\n\nDisassembly of :\n -- COPY_FREE_VARS 2\n\n 4 RESUME 0\n\n 5 LOAD_DEREF 0 (a)\n LOAD_DEREF 1 (n)\n BINARY_OP 0 (+)\n RETURN_VALUE\n" }, { "name": "f10-13", @@ -1581,12 +1581,12 @@ { "name": "f12-07", "status": "ok", - "printed": " greet.py is 54 bytes of text\n greet.cpython-314.pyc is 251 bytes\n\n header 2b 0e 0d 0a 00 00 00 00 60 53 9d 6a 36 00 00 00\n the rest 235 bytes, and marshal.loads gives back a\n code object\n" + "printed": " greet.py is 54 bytes of text\n greet.cpython-314.pyc is 251 bytes\n\n header 2b 0e 0d 0a 00 00 00 00 80 6c 9d 6a 36 00 00 00\n the rest 235 bytes, and marshal.loads gives back a\n code object\n" }, { "name": "f12-10", "status": "ok", - "printed": " magic 168627755\n flags 0\n mtime 1788695392 source 1788695392\n size 54 source 54\n" + "printed": " magic 168627755\n flags 0\n mtime 1788701824 source 1788701824\n size 54 source 54\n" }, { "name": "f12-13", @@ -1620,7 +1620,7 @@ { "name": "f12-29", "status": "ok", - "printed": " one bit flipped in the magic: bad magic number in 'broken': b'*\\x0e\\r\\n'\n\n put a wrong mtime in greet.cpython-314.pyc: 1788695455\n imported it with no complaint: 'hi world'\n and the header now says 1788695455\n the file was quietly rewritten: False\n" + "printed": " one bit flipped in the magic: bad magic number in 'broken': b'*\\x0e\\r\\n'\n\n put a wrong mtime in greet.cpython-314.pyc: 1788701823\n imported it with no complaint: 'hi world'\n and the header now says 1788701823\n the file was quietly rewritten: False\n" } ] }, @@ -1963,7 +1963,7 @@ { "name": "m07-29", "status": "ok", - "printed": " frozen objects before 0\n tracked in generation 2 24962\n frozen after gc.freeze() 24962\n left in generation 2 0\n frozen after gc.unfreeze() 0\n" + "printed": " frozen objects before 0\n tracked in generation 2 24965\n frozen after gc.freeze() 24965\n left in generation 2 0\n frozen after gc.unfreeze() 0\n" } ] }, @@ -1987,7 +1987,7 @@ { "name": "m08-09", "status": "ok", - "printed": " objects in each generation [3, 0, 24948]\n as soon as it exists generations [0]\n after a pass over gen 0 generations [2]\n after a pass over gen 1 generations [2]\n" + "printed": " objects in each generation [3, 0, 24951]\n as soon as it exists generations [0]\n after a pass over gen 0 generations [2]\n after a pass over gen 1 generations [2]\n" }, { "name": "m08-12", @@ -2002,12 +2002,12 @@ { "name": "m08-17", "status": "ok", - "printed": " cycles on the heap 200000\n one full pass over it 41 ms\n and over an empty one 7 ms\n" + "printed": " cycles on the heap 200000\n one full pass over it 27 ms\n and over an empty one 6 ms\n" }, { "name": "m08-20", "status": "ok", - "printed": " this version does not report candidates, so there is nothing to compare\n objects the collector is tracking 24964\n" + "printed": " this version does not report candidates, so there is nothing to compare\n objects the collector is tracking 24967\n" } ] }, @@ -2085,7 +2085,7 @@ { "name": "o01-07", "status": "ok", - "printed": " a pointer here is 4 bytes, so the header is 8\n it lives at 19423616\n\n first word, read as a count 1\n sys.getrefcount, less its own argument 1\n the same number: True\n\n second word 22677104\n id(Thing) is 22677104\n so the second word is the type pointer: True\n\n after a second name, the count reads 2\n after deleting that name, it reads 1\n" + "printed": " a pointer here is 4 bytes, so the header is 8\n it lives at 22217584\n\n first word, read as a count 1\n sys.getrefcount, less its own argument 1\n the same number: True\n\n second word 24746680\n id(Thing) is 24746680\n so the second word is the type pointer: True\n\n after a second name, the count reads 2\n after deleting that name, it reads 1\n" }, { "name": "o01-10", @@ -2958,7 +2958,7 @@ { "name": "r03-22", "status": "ok", - "printed": " first time, source file, no .pyc yet: 494.6 microseconds\n from a file, with a .pyc already there: 421.1 microseconds\n asking again for one already loaded: 582.0 nanoseconds\n the second of those is 724 times the third\n" + "printed": " first time, source file, no .pyc yet: 72.7 microseconds\n from a file, with a .pyc already there: 67.7 microseconds\n asking again for one already loaded: 164.1 nanoseconds\n the second of those is 413 times the third\n" } ] }, @@ -3007,7 +3007,7 @@ { "name": "r04-25", "status": "ok", - "printed": " asking FrozenImporter for os: 2.65 microseconds\n asking PathFinder for the same name: 17184.84 microseconds\n this runtime keeps no pyc for os, so the last two rows have nothing to measure\n" + "printed": " asking FrozenImporter for os: 1.50 microseconds\n asking PathFinder for the same name: 4691.80 microseconds\n this runtime keeps no pyc for os, so the last two rows have nothing to measure\n" }, { "name": "r04-29", @@ -3031,7 +3031,7 @@ { "name": "r05-07", "status": "ok", - "printed": " what that one import line cost here: 434.3 ms\n modules it needs, all told: 8\n which are: _elementtree, pyexpat, pyexpat.errors, pyexpat.model, xml, xml.etree, xml.etree.ElementPath, xml.etree.ElementTree\n how many of those this cell had to load: 8\n\n modules this process is now carrying: 242\n" + "printed": " what that one import line cost here: 23.8 ms\n modules it needs, all told: 8\n which are: _elementtree, pyexpat, pyexpat.errors, pyexpat.model, xml, xml.etree, xml.etree.ElementPath, xml.etree.ElementTree\n how many of those this cell had to load: 8\n\n modules this process is now carrying: 242\n" }, { "name": "r05-10", @@ -3242,6 +3242,50 @@ } ] }, + { + "slug": "r09-writing-a-c-extension-properly", + "cells": [ + { + "name": "r09-03", + "status": "ok" + }, + { + "name": "r09-05", + "status": "ok", + "printed": "cpython 3.14.2 on Emscripten wasm32, WebAssembly\n note: this is 3.14, and everything here is written against 3.15, so bytecode and some object layouts will differ from the prose\n" + }, + { + "name": "r09-07", + "status": "ok", + "printed": " no compiler and headers here, so there is nothing to build\n" + }, + { + "name": "r09-09", + "status": "ok", + "printed": " 154 lines of C, three types, one field each\n" + }, + { + "name": "r09-11", + "status": "ok", + "printed": " no compiler and headers here, so there is nothing to build\n" + }, + { + "name": "r09-13", + "status": "ok", + "printed": " no compiler and headers here, so there is nothing to build\n" + }, + { + "name": "r09-15", + "status": "ok", + "printed": " no compiler and headers here, so there is nothing to build\n" + }, + { + "name": "r09-17", + "status": "ok", + "printed": " no compiler and headers here, so there is nothing to build\n" + } + ] + }, { "slug": "t01-one-line-seven-stages", "cells": [ @@ -3624,7 +3668,7 @@ { "name": "t04-30", "status": "ok", - "printed": "free variables: ('total',)\nthe cells: (,)\nfirst call: 1\nsecond call: 2\nthe cell now: 2\n" + "printed": "free variables: ('total',)\nthe cells: (,)\nfirst call: 1\nsecond call: 2\nthe cell now: 2\n" }, { "name": "t04-33", @@ -3901,7 +3945,7 @@ { "name": "t07-38", "status": "ok", - "printed": "asked twice, got the same object: True\nand it is still here after the call returned: \nits name: make_one\n" + "printed": "asked twice, got the same object: True\nand it is still here after the call returned: \nits name: make_one\n" }, { "name": "t07-40", @@ -3935,12 +3979,12 @@ { "name": "t08-11", "status": "ok", - "printed": "NoneType at 0x30d7d8, refcount is parked, this object is never freed, 8 bytes, not tracked\nint at 0x31a7cc, refcount is parked, this object is never freed, 16 bytes, not tracked\nstr at 0x1784c00, 3 reference(s), 26 bytes, not tracked\nlist at 0x1727228, 2 reference(s), 44 bytes, tracked by the cycle collector\ndict at 0x185baa8, 2 reference(s), 108 bytes, tracked by the cycle collector\nfunction at 0x1537420, 4 reference(s), 84 bytes, tracked by the cycle collector\n" + "printed": "NoneType at 0x30d7d8, refcount is parked, this object is never freed, 8 bytes, not tracked\nint at 0x31a7cc, refcount is parked, this object is never freed, 16 bytes, not tracked\nstr at 0x11458d8, 3 reference(s), 26 bytes, not tracked\nlist at 0x1899510, 2 reference(s), 44 bytes, tracked by the cycle collector\ndict at 0x16f8940, 2 reference(s), 108 bytes, tracked by the cycle collector\nfunction at 0x1709a20, 4 reference(s), 84 bytes, tracked by the cycle collector\n" }, { "name": "t08-14", "status": "ok", - "printed": "a == b True same contents\na is b False different objects\na is c True same object\n\nid(a) 0x18bb570\nid(b) 0x17bcff8\nid(c) 0x18bb570\n" + "printed": "a == b True same contents\na is b False different objects\na is c True same object\n\nid(a) 0x17d58c0\nid(b) 0x17f92b8\nid(c) 0x17d58c0\n" }, { "name": "t08-17", @@ -3980,7 +4024,7 @@ { "name": "t08-35", "status": "ok", - "printed": "dict at 0x1001878\ndict at 0x1535168\nlist at 0x1255d88\n" + "printed": "dict at 0x113f650\ndict at 0x130a1b0\nlist at 0x1899760\n" }, { "name": "t08-37", @@ -4039,7 +4083,7 @@ { "name": "t09-21", "status": "ok", - "printed": "Node at 0x1709600 -> Node at 0x16f3b20 -> Node at 0x17d1cf8 -> Node at 0x1709600\n" + "printed": "Node at 0x16c50f8 -> Node at 0x169f5a8 -> Node at 0x18bf408 -> Node at 0x16c50f8\n" }, { "name": "t09-23", @@ -4069,7 +4113,7 @@ { "name": "t09-37", "status": "ok", - "printed": "first object was at 0x16def48\nsecond object is at 0x16def48\nsame address reused -> True\n" + "printed": "first object was at 0x14162a8\nsecond object is at 0x14162a8\nsame address reused -> True\n" } ] }, diff --git a/probes/pyodide/lessons.md b/probes/pyodide/lessons.md index c67fd45..bdb8083 100644 --- a/probes/pyodide/lessons.md +++ b/probes/pyodide/lessons.md @@ -2,7 +2,7 @@ Generated by `just build-probe`. Do not edit by hand, the change will be overwritten. -79 lesson(s) on Pyodide 3.14.2: 79 ran end to end, 776 cell(s) in total. +80 lesson(s) on Pyodide 3.14.2: 80 ran end to end, 784 cell(s) in total. The checks in `report.md` next to this ask whether a surface exists. This runs the lessons themselves: every code cell of every notebook, in order, in one Pyodide runtime, with `pyxray` mounted off the disk rather than installed. The install cell is the one thing changed, and only its `%pip` line, which a reader in a browser does not need either. @@ -75,6 +75,7 @@ The checks in `report.md` next to this ask whether a surface exists. This runs t | r06-the-c-api-tiers | 11 | runs end to end | | r07-the-stable-abi | 10 | runs end to end | | r08-when-the-interpreter-stops | 9 | runs end to end | +| r09-writing-a-c-extension-properly | 8 | runs end to end | | t01-one-line-seven-stages | 18 | runs end to end | | t02-text-becomes-tokens | 32 | runs end to end | | t03-tokens-become-a-tree | 13 | runs end to end | diff --git a/pyxray/src/pyxray/glossary.py b/pyxray/src/pyxray/glossary.py index 0467e4e..6f53e47 100644 --- a/pyxray/src/pyxray/glossary.py +++ b/pyxray/src/pyxray/glossary.py @@ -1521,6 +1521,24 @@ def anchor(name: str) -> str: see=("finalizer", "resurrection"), met="O14", ), + Term( + name="traverse function", + short="The slot where a container tells the collector what it is holding.", + long="`tp_traverse` is handed a callback and calls it once for every object this one owns a reference to. The collector uses that to subtract the references living inside the set it is examining, so a field you forget to report looks exactly like a reference from somewhere outside and the whole cycle survives. A heap type has to report `Py_TYPE(self)` as well, since an instance holds a reference to its own type.", + cite="Objects/typeobject.c:2607-2645@v3.15.0rc1#subtype_traverse", + also=("`tp_traverse`", "`Py_VISIT`", "`visitproc`"), + see=("cycle collector", "reference cycle", "heap type"), + met="R09", + ), + Term( + name="clear function", + short="The slot the collector calls to break a cycle it has decided is garbage.", + long="`tp_clear` drops the references `tp_traverse` reported, usually one `Py_CLEAR` per field. It is the only part of a collection that changes anything, and it is called across the whole unreachable set before any of it is freed, so an object still has to behave while the rest of its cycle is being emptied around it. Anything the object owns that is not a Python object belongs in `tp_dealloc` instead.", + cite="Python/gc.c:1083-1120@v3.15.0rc1#delete_garbage", + also=("`tp_clear`", "`Py_CLEAR`"), + see=("traverse function", "cycle collector", "finalizer"), + met="R09", + ), ), ) @@ -2042,6 +2060,15 @@ def anchor(name: str) -> str: see=("stable ABI", "SOABI", "PyABIInfo"), met="R07", ), + Term( + name="module state", + short="A struct hanging off a module object, one copy per module rather than per process.", + long="A module definition with a positive `m_size` gets that many bytes allocated alongside every module object, reachable with `PyModule_GetState`. It is where an extension keeps what a C file would otherwise keep in statics, which matters because statics are shared by every interpreter in the process and module state is not. A type built from a spec can find its own module's state again through `PyType_GetModuleByDef`.", + cite="Objects/moduleobject.c:1027-1035@v3.15.0rc1#PyModule_GetState", + also=("`m_size`", "`PyModule_GetState`", "`PyType_GetModuleByDef`"), + see=("extension module", "two phase initialisation", "subinterpreter"), + met="R09", + ), Term( name="SOABI", short="The tag a build stamps into the file names of the extensions it compiles.", diff --git a/tools/tier1/src/tier1/experiments.py b/tools/tier1/src/tier1/experiments.py index 8c15831..71bf3a9 100644 --- a/tools/tier1/src/tier1/experiments.py +++ b/tools/tier1/src/tier1/experiments.py @@ -3538,6 +3538,208 @@ def leftover(program): ) +PROGRAM_TWENTYNINE = r'''"""What the reference leak hunter catches that an ordinary run does not. + +Four small tests, written into a directory of their own, run twice each. Once the way +anybody runs a test suite, and once under the flag CPython's buildbots use, which runs +every test six times and counts what the interpreter is still holding afterwards. +""" + +import os +import pathlib +import re +import subprocess +import sys +import tempfile + +CASES = { + "test_clean": """ +import unittest + + +class Clean(unittest.TestCase): + def test_it(self): + held = [object()] + self.assertTrue(held) +""", + "test_cached": """ +import unittest + +CACHE = [] + + +class Cached(unittest.TestCase): + def test_it(self): + if len(CACHE) < 4: + CACHE.append(object()) + self.assertTrue(CACHE) +""", + "test_leaky": """ +import unittest + +HOARD = [] + + +class Leaky(unittest.TestCase): + def test_it(self): + HOARD.append(object()) + self.assertTrue(HOARD) +""", + "test_handles": """ +import os +import unittest + + +class Handles(unittest.TestCase): + def test_it(self): + copy = os.dup(0) + self.assertGreater(copy, 0) +""", +} + +REPORT = re.compile(r"^\w+ leaked (\[[^]]*\]) ([a-z ]+), sum=(-?\d+)(.*)$") + +ROOT = pathlib.Path(tempfile.mkdtemp()) +for name, body in CASES.items(): + (ROOT / f"{name}.py").write_text(body) + + +def run(name, flags=()): + """Run one of the four tests, with or without the leak hunting flag.""" + return subprocess.run( + [sys.executable, "-m", "test", "--testdir", str(ROOT), *flags, name], + capture_output=True, + text=True, + timeout=600, + env=os.environ | {"PYTHON_COLORS": "0"}, + ) + + +print("four small tests, run the way anybody runs a test suite") +print() +ordinary = 0 +for name in CASES: + done = run(name) + ordinary += done.returncode == 0 + print(f" {name:14} {'failed' if done.returncode else 'passed'}") + +print() +print("the same four, run six times each with the leak hunter watching") +print() +hunted = 0 +for name in CASES: + done = run(name, ("-R", "3:3")) + hunted += done.returncode == 0 + said = [m for m in map(REPORT.match, done.stderr.splitlines()) if m] + counts = ", ".join(f"{m.group(1)} {m.group(2)}" for m in said) + excused = " (which the hunter calls fine)" if said and said[0].group(4) else "" + verdict = "failed" if done.returncode else "passed" + print(f" {name:14} {verdict:7} {counts or 'nothing was left behind'}{excused}") + +print() +print(f"~ tests that pass an ordinary run: {ordinary} of {len(CASES)}") +print(f"~ tests that pass the leak hunter: {hunted} of {len(CASES)}") +print("~ repetitions the hunter ran each test for: 6") +print("~ of those repetitions that were warm ups: 3") +''' + + +WHAT_THE_LEAK_HUNTER_CATCHES = Experiment( + slug="r09-what-the-leak-hunter-catches", + lesson="R09", + title="Four small tests, run once the ordinary way and once with the leak hunter watching", + asks="What does CPython's own leak hunter see that an ordinary test run walks straight past?", + needs=( + "it needs a debug build, because counting what the interpreter is still holding is " + "something only that build keeps a total of" + ), + build="debug", + program=PROGRAM_TWENTYNINE, +) + + +PROGRAM_THIRTY = r'''"""What a module has to declare before a free threaded build will trust it. + +CPython's own test suite ships a shared object that exports several init functions, two +of which differ only in one line: whether they say the module can run without the lock. +Loading each one in a child of its own shows what that line is worth. +""" + +import os +import subprocess +import sys +import textwrap + +CHILD = """ +import importlib.machinery, importlib.util, sys, _testmultiphase + +name = sys.argv[1] +print(f" the lock before the import: {sys._is_gil_enabled()}") +loader = importlib.machinery.ExtensionFileLoader(name, _testmultiphase.__file__) +spec = importlib.util.spec_from_loader(name, loader) +loader.exec_module(importlib.util.module_from_spec(spec)) +print(f" the lock after the import: {sys._is_gil_enabled()}") +""" + +CASES = ( + ("_test_from_modexport", "which declares Py_MOD_GIL_NOT_USED", {}), + ("_test_from_modexport_gil_used", "which declares Py_MOD_GIL_USED", {}), + ("_test_from_modexport_gil_used", "the same one, in a child started with PYTHON_GIL=0", {}), +) + +OVERRIDE = {"PYTHON_GIL": "0"} + + +def load(name, extra): + """Import one init function out of that shared object, in a child of its own.""" + return subprocess.run( + [sys.executable, "-c", CHILD, name], + capture_output=True, + text=True, + timeout=180, + env=os.environ | {"PYTHON_COLORS": "0"} | extra, + ) + + +print("what a compiled module has to declare before this build will trust it") +print() +print(f" the abi flags on this build: {sys.abiflags!r}") +print() + +turned_on = 0 +for position, (name, what, _) in enumerate(CASES): + done = load(name, OVERRIDE if position == 2 else {}) + print(f" {name}, {what}") + print(done.stdout, end="") + turned_on += "the lock after the import: True" in done.stdout + said = [one for one in done.stderr.splitlines() if "RuntimeWarning" in one] + if not said: + print(" it went through without a word") + for one in said: + print(" on the way it warned, at some length:") + text = one.split("RuntimeWarning: ")[1] + print(textwrap.indent(textwrap.fill(text, 74), " ")) + print() + +print(f"~ ways of loading the same shared object: {len(CASES)}") +print(f"~ of those that turned the lock back on: {turned_on}") +''' + + +WHAT_A_MODULE_MUST_DECLARE = Experiment( + slug="r09-what-a-module-must-declare", + lesson="R09", + title="One shared object, three ways to load it, and what each one does to the lock", + asks="What is the line at the bottom of an extension module that declares itself safe worth?", + needs=( + "it needs a free threaded build, because a build that always has the lock has nothing " + "to turn back on" + ), + build="freethreaded", + program=PROGRAM_THIRTY, +) + + EXPERIMENTS: tuple[Experiment, ...] = ( COMPILING_COSTS_NOTHING_THAT_LASTS, A_LEAK_YOU_CAN_SEE, @@ -3582,6 +3784,8 @@ def leftover(program): WHAT_A_BUILD_WILL_LOAD_WITHOUT_THE_LOCK, WHAT_THE_END_STILL_RUNS, WHAT_THE_END_LEAVES_BEHIND, + WHAT_THE_LEAK_HUNTER_CATCHES, + WHAT_A_MODULE_MUST_DECLARE, )