From 88e2f3d1174310d0ba47b2deaa87c1c5fc35567e Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 14 Oct 2026 14:51:42 +0200 Subject: [PATCH 1/4] feat: Instrument garbage collection --- sentry_sdk/consts.py | 19 +++ sentry_sdk/integrations/gc.py | 124 ++++++++++++++++ tests/integrations/gc/__init__.py | 0 tests/integrations/gc/test_gc.py | 232 ++++++++++++++++++++++++++++++ 4 files changed, 375 insertions(+) create mode 100644 sentry_sdk/integrations/gc.py create mode 100644 tests/integrations/gc/__init__.py create mode 100644 tests/integrations/gc/test_gc.py diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 6a53c783ef..6e9fbff722 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -486,6 +486,24 @@ class SPANDATA: Example: "http.handler" """ + CPYTHON_GC_COLLECTED_OBJECTS = "cpython.gc.collected_objects" + """ + The number of objects collected during a garbage collection run. + Example: 42 + """ + + CPYTHON_GC_GENERATION = "cpython.gc.generation" + """ + The generation that a garbage collection run collected. + Example: 2 + """ + + CPYTHON_GC_UNCOLLECTABLE_OBJECTS = "cpython.gc.uncollectable_objects" + """ + The number of objects found to be uncollectable during a garbage collection run. + Example: 0 + """ + DB_MONGODB_COLLECTION = "db.mongodb.collection" """ The MongoDB collection being accessed within the database. @@ -1310,6 +1328,7 @@ class OP: FUNCTION = "function" FUNCTION_AWS = "function.aws" FUNCTION_GCP = "function.gcp" + GC = "gc" GEN_AI_CHAT = "gen_ai.chat" GEN_AI_CREATE_AGENT = "gen_ai.create_agent" GEN_AI_EMBEDDINGS = "gen_ai.embeddings" diff --git a/sentry_sdk/integrations/gc.py b/sentry_sdk/integrations/gc.py new file mode 100644 index 0000000000..d9c1e84060 --- /dev/null +++ b/sentry_sdk/integrations/gc.py @@ -0,0 +1,124 @@ +import gc +import threading +from typing import TYPE_CHECKING + +import sentry_sdk +from sentry_sdk.consts import OP, SPANDATA +from sentry_sdk.integrations import Integration +from sentry_sdk.traces import StreamedSpan +from sentry_sdk.tracing_utils import has_span_streaming_enabled +from sentry_sdk.utils import capture_internal_exceptions + +if TYPE_CHECKING: + from typing import Any, Dict, Optional, Union + + from sentry_sdk.tracing import Span + + +ORIGIN = "auto.gc" + +# The span of the garbage collection run that is currently in progress on this +# thread. A collection runs synchronously from start to stop in the thread that +# triggered it, and collections can't nest, so there's at most one at a time. +_pending = threading.local() + + +class GcIntegration(Integration): + """ + Creates a span for each garbage collection run that happens while a span is + active. + + :param min_generation: Only collections of this generation or higher are + recorded. Collections of the younger generations are very frequent and + usually take well under a millisecond, so by default only full + collections (generation 2) are recorded. + """ + + identifier = "gc" + origin = ORIGIN + + def __init__(self, min_generation: int = 2) -> None: + self.min_generation = min_generation + + @staticmethod + def setup_once() -> None: + # gc.callbacks is the interpreter's official hook into garbage + # collection. The callback is invoked right before ("start") and right + # after ("stop") every collection. + gc.callbacks.append(_gc_callback) + + +def _gc_callback(phase: str, info: "Dict[str, Any]") -> None: + # This runs in the middle of arbitrary user (or SDK) code, so it must never + # raise. An exception here would be reported as unraisable on stderr. + with capture_internal_exceptions(): + if phase == "start": + _start_gc_span(info) + elif phase == "stop": + _end_gc_span(info) + + +def _start_gc_span(info: "Dict[str, Any]") -> None: + # This runs on every collection, including the very frequent young + # generation ones, so bail out as cheaply as possible. + client = sentry_sdk.get_client() + integration = client.get_integration(GcIntegration) + if integration is None: + return + + generation = info["generation"] + if generation < integration.min_generation: + return + + # The span is started without making it the active span: finalizers + # (__del__, weakref callbacks) run during the collection and might create + # spans of their own, which shouldn't end up as children of the gc span. + # We also only ever attach the gc span to an existing span, never start a + # new trace for it. + span: "Union[Span, StreamedSpan]" + if has_span_streaming_enabled(client.options): + parent = sentry_sdk.traces.get_current_span() + if parent is None: + return + + span = sentry_sdk.traces.start_span( + name="garbage collection", + attributes={ + "sentry.op": OP.GC, + "sentry.origin": GcIntegration.origin, + SPANDATA.CPYTHON_GC_GENERATION: generation, + }, + active=False, + ) + else: + legacy_parent = sentry_sdk.get_current_span() + if legacy_parent is None: + return + + span = legacy_parent.start_child( + op=OP.GC, + name="garbage collection", + origin=GcIntegration.origin, + ) + span.set_data(SPANDATA.CPYTHON_GC_GENERATION, generation) + + _pending.span = span + + +def _end_gc_span(info: "Dict[str, Any]") -> None: + span: "Optional[Union[Span, StreamedSpan]]" = getattr(_pending, "span", None) + if span is None: + return + + _pending.span = None + + if isinstance(span, StreamedSpan): + span.set_attribute(SPANDATA.CPYTHON_GC_COLLECTED_OBJECTS, info["collected"]) + span.set_attribute( + SPANDATA.CPYTHON_GC_UNCOLLECTABLE_OBJECTS, info["uncollectable"] + ) + span.end() + else: + span.set_data(SPANDATA.CPYTHON_GC_COLLECTED_OBJECTS, info["collected"]) + span.set_data(SPANDATA.CPYTHON_GC_UNCOLLECTABLE_OBJECTS, info["uncollectable"]) + span.finish() diff --git a/tests/integrations/gc/__init__.py b/tests/integrations/gc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integrations/gc/test_gc.py b/tests/integrations/gc/test_gc.py new file mode 100644 index 0000000000..91994bddd3 --- /dev/null +++ b/tests/integrations/gc/test_gc.py @@ -0,0 +1,232 @@ +import gc + +import pytest + +import sentry_sdk +from sentry_sdk import start_transaction +from sentry_sdk.integrations.gc import GcIntegration + + +class Node: + pass + + +def create_garbage_cycles(count): + for _ in range(count): + a, b = Node(), Node() + a.other = b + b.other = a + + +@pytest.fixture(autouse=True) +def no_automatic_gc(): + # Automatic collections can happen at any point during a test and would + # create additional spans. Explicit gc.collect() calls still work. + gc.collect() + gc.disable() + yield + gc.enable() + + +def collect_in_parent_span(span_streaming, capture_events, capture_items, collect): + """ + Runs `collect` inside a parent span and returns (parent_span_id, gc_spans). + The spans are normalized to (name, op, origin, parent_span_id, data). + """ + if span_streaming: + items = capture_items("span") + with sentry_sdk.traces.start_span(name="parent") as parent: + collect() + sentry_sdk.flush() + + gc_spans = [ + ( + item.payload["name"], + item.payload["attributes"]["sentry.op"], + item.payload["attributes"]["sentry.origin"], + item.payload.get("parent_span_id"), + item.payload["attributes"], + ) + for item in items + if item.payload["attributes"].get("sentry.op") == "gc" + ] + return parent.span_id, gc_spans + + events = capture_events() + with start_transaction(name="parent") as transaction: + collect() + + (event,) = events + gc_spans = [ + ( + span["description"], + span["op"], + span["origin"], + span["parent_span_id"], + span["data"], + ) + for span in event["spans"] + if span["op"] == "gc" + ] + return transaction.span_id, gc_spans + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_gc_span(sentry_init, capture_events, capture_items, span_streaming): + sentry_init( + integrations=[GcIntegration()], + traces_sample_rate=1.0, + trace_lifecycle="stream" if span_streaming else "static", + ) + + def collect(): + create_garbage_cycles(100) + gc.collect() + + parent_span_id, gc_spans = collect_in_parent_span( + span_streaming, capture_events, capture_items, collect + ) + + (span,) = gc_spans + name, op, origin, span_parent_id, data = span + assert name == "garbage collection" + assert op == "gc" + assert origin == "auto.gc" + assert span_parent_id == parent_span_id + assert data["cpython.gc.generation"] == 2 + assert data["cpython.gc.collected_objects"] >= 200 + assert data["cpython.gc.uncollectable_objects"] == 0 + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_young_generations_skipped_by_default( + sentry_init, capture_events, capture_items, span_streaming +): + sentry_init( + integrations=[GcIntegration()], + traces_sample_rate=1.0, + trace_lifecycle="stream" if span_streaming else "static", + ) + + def collect(): + gc.collect(0) + gc.collect(1) + + _, gc_spans = collect_in_parent_span( + span_streaming, capture_events, capture_items, collect + ) + + assert gc_spans == [] + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_min_generation(sentry_init, capture_events, capture_items, span_streaming): + sentry_init( + integrations=[GcIntegration(min_generation=1)], + traces_sample_rate=1.0, + trace_lifecycle="stream" if span_streaming else "static", + ) + + def collect(): + gc.collect(0) + gc.collect(1) + gc.collect(2) + + _, gc_spans = collect_in_parent_span( + span_streaming, capture_events, capture_items, collect + ) + + assert [span[4]["cpython.gc.generation"] for span in gc_spans] == [1, 2] + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_no_span_without_parent( + sentry_init, capture_events, capture_items, span_streaming +): + sentry_init( + integrations=[GcIntegration()], + traces_sample_rate=1.0, + trace_lifecycle="stream" if span_streaming else "static", + ) + events = capture_events() + items = capture_items() + + gc.collect() + sentry_sdk.flush() + + assert events == [] + assert items == [] + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_no_span_when_integration_disabled( + sentry_init, capture_events, capture_items, span_streaming +): + # Make sure the callback is registered, then switch to a client that + # doesn't have the integration enabled. + sentry_init(integrations=[GcIntegration()]) + sentry_init( + default_integrations=False, + traces_sample_rate=1.0, + trace_lifecycle="stream" if span_streaming else "static", + ) + + _, gc_spans = collect_in_parent_span( + span_streaming, capture_events, capture_items, gc.collect + ) + + assert gc_spans == [] + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_finalizer_spans_are_not_children_of_gc_span( + sentry_init, capture_events, capture_items, span_streaming +): + sentry_init( + integrations=[GcIntegration()], + traces_sample_rate=1.0, + trace_lifecycle="stream" if span_streaming else "static", + ) + + class WithFinalizer: + def __del__(self): + if span_streaming: + with sentry_sdk.traces.start_span( + name="finalizer", attributes={"sentry.op": "finalizer"} + ): + pass + else: + with sentry_sdk.start_span(op="finalizer", name="finalizer"): + pass + + def collect(): + a, b = WithFinalizer(), WithFinalizer() + a.other = b + b.other = a + del a, b + gc.collect() + + if span_streaming: + items = capture_items("span") + with sentry_sdk.traces.start_span(name="parent") as parent: + collect() + sentry_sdk.flush() + + spans = [item.payload for item in items] + finalizer_spans = [s for s in spans if s["name"] == "finalizer"] + (gc_span,) = [s for s in spans if s["attributes"].get("sentry.op") == "gc"] + parent_span_id = parent.span_id + else: + events = capture_events() + with start_transaction(name="parent") as transaction: + collect() + + (event,) = events + finalizer_spans = [s for s in event["spans"] if s["op"] == "finalizer"] + (gc_span,) = [s for s in event["spans"] if s["op"] == "gc"] + parent_span_id = transaction.span_id + + assert gc_span["parent_span_id"] == parent_span_id + assert len(finalizer_spans) == 2 + for span in finalizer_spans: + assert span["parent_span_id"] == parent_span_id From d3282d588cbbd77242f71f9e91e79294205a4425 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 14 Oct 2026 15:44:44 +0200 Subject: [PATCH 2/4] . --- sentry_sdk/integrations/gc.py | 12 ++++- tests/integrations/gc/test_gc.py | 77 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/gc.py b/sentry_sdk/integrations/gc.py index d9c1e84060..304c1bd510 100644 --- a/sentry_sdk/integrations/gc.py +++ b/sentry_sdk/integrations/gc.py @@ -1,4 +1,5 @@ import gc +import sysconfig import threading from typing import TYPE_CHECKING @@ -17,6 +18,12 @@ ORIGIN = "auto.gc" +# The garbage collector of free-threaded builds isn't generational: every +# collection scans all objects, regardless of the generation it reports. This +# is a property of the build, so it also applies if the GIL was re-enabled at +# runtime. +_GC_IS_GENERATIONAL = not sysconfig.get_config_var("Py_GIL_DISABLED") + # The span of the garbage collection run that is currently in progress on this # thread. A collection runs synchronously from start to stop in the thread that # triggered it, and collections can't nest, so there's at most one at a time. @@ -31,7 +38,8 @@ class GcIntegration(Integration): :param min_generation: Only collections of this generation or higher are recorded. Collections of the younger generations are very frequent and usually take well under a millisecond, so by default only full - collections (generation 2) are recorded. + collections (generation 2) are recorded. Ignored on free-threaded + builds, where every collection is a full collection. """ identifier = "gc" @@ -67,7 +75,7 @@ def _start_gc_span(info: "Dict[str, Any]") -> None: return generation = info["generation"] - if generation < integration.min_generation: + if _GC_IS_GENERATIONAL and generation < integration.min_generation: return # The span is started without making it the active span: finalizers diff --git a/tests/integrations/gc/test_gc.py b/tests/integrations/gc/test_gc.py index 91994bddd3..193f68bc55 100644 --- a/tests/integrations/gc/test_gc.py +++ b/tests/integrations/gc/test_gc.py @@ -1,4 +1,5 @@ import gc +import sysconfig import pytest @@ -6,6 +7,15 @@ from sentry_sdk import start_transaction from sentry_sdk.integrations.gc import GcIntegration +FREE_THREADED_BUILD = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + +requires_generational_gc = pytest.mark.skipif( + FREE_THREADED_BUILD, reason="The free-threaded GC isn't generational" +) +requires_free_threaded_build = pytest.mark.skipif( + not FREE_THREADED_BUILD, reason="Requires a free-threaded build" +) + class Node: pass @@ -98,6 +108,7 @@ def collect(): assert data["cpython.gc.uncollectable_objects"] == 0 +@requires_generational_gc @pytest.mark.parametrize("span_streaming", [True, False]) def test_young_generations_skipped_by_default( sentry_init, capture_events, capture_items, span_streaming @@ -119,6 +130,7 @@ def collect(): assert gc_spans == [] +@requires_generational_gc @pytest.mark.parametrize("span_streaming", [True, False]) def test_min_generation(sentry_init, capture_events, capture_items, span_streaming): sentry_init( @@ -139,6 +151,71 @@ def collect(): assert [span[4]["cpython.gc.generation"] for span in gc_spans] == [1, 2] +@requires_free_threaded_build +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_min_generation_ignored_on_free_threaded_build( + sentry_init, capture_events, capture_items, span_streaming +): + sentry_init( + integrations=[GcIntegration(min_generation=2)], + traces_sample_rate=1.0, + trace_lifecycle="stream" if span_streaming else "static", + ) + + def collect(): + gc.collect(0) + gc.collect(1) + gc.collect(2) + + _, gc_spans = collect_in_parent_span( + span_streaming, capture_events, capture_items, collect + ) + + assert [span[4]["cpython.gc.generation"] for span in gc_spans] == [0, 1, 2] + + +@requires_free_threaded_build +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_automatic_collection_on_free_threaded_build( + sentry_init, capture_events, capture_items, span_streaming +): + # Automatic collections on free-threaded builds always report generation 0, + # but they're full collections and must not be filtered out. + sentry_init( + integrations=[GcIntegration()], + traces_sample_rate=1.0, + trace_lifecycle="stream" if span_streaming else "static", + ) + + collections = [] + + def on_collection(phase, info): + if phase == "stop": + collections.append(info) + + def collect(): + # When exactly an automatic collection happens depends on the + # interpreter's heuristics, so allocate until one does. + gc.callbacks.append(on_collection) + gc.enable() + try: + for _ in range(1000): + create_garbage_cycles(1000) + if collections: + break + finally: + gc.disable() + gc.callbacks.remove(on_collection) + + _, gc_spans = collect_in_parent_span( + span_streaming, capture_events, capture_items, collect + ) + + assert collections + assert len(gc_spans) == len(collections) + assert all(span[4]["cpython.gc.generation"] == 0 for span in gc_spans) + + @pytest.mark.parametrize("span_streaming", [True, False]) def test_no_span_without_parent( sentry_init, capture_events, capture_items, span_streaming From cdef2ad0307b99f02d363feabe4fb4f33f659989 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 14 Oct 2026 15:54:35 +0200 Subject: [PATCH 3/4] simplify --- sentry_sdk/integrations/gc.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/sentry_sdk/integrations/gc.py b/sentry_sdk/integrations/gc.py index 304c1bd510..1019a5e8ff 100644 --- a/sentry_sdk/integrations/gc.py +++ b/sentry_sdk/integrations/gc.py @@ -50,15 +50,12 @@ def __init__(self, min_generation: int = 2) -> None: @staticmethod def setup_once() -> None: - # gc.callbacks is the interpreter's official hook into garbage - # collection. The callback is invoked right before ("start") and right - # after ("stop") every collection. gc.callbacks.append(_gc_callback) def _gc_callback(phase: str, info: "Dict[str, Any]") -> None: # This runs in the middle of arbitrary user (or SDK) code, so it must never - # raise. An exception here would be reported as unraisable on stderr. + # raise. with capture_internal_exceptions(): if phase == "start": _start_gc_span(info) @@ -67,8 +64,6 @@ def _gc_callback(phase: str, info: "Dict[str, Any]") -> None: def _start_gc_span(info: "Dict[str, Any]") -> None: - # This runs on every collection, including the very frequent young - # generation ones, so bail out as cheaply as possible. client = sentry_sdk.get_client() integration = client.get_integration(GcIntegration) if integration is None: @@ -78,11 +73,6 @@ def _start_gc_span(info: "Dict[str, Any]") -> None: if _GC_IS_GENERATIONAL and generation < integration.min_generation: return - # The span is started without making it the active span: finalizers - # (__del__, weakref callbacks) run during the collection and might create - # spans of their own, which shouldn't end up as children of the gc span. - # We also only ever attach the gc span to an existing span, never start a - # new trace for it. span: "Union[Span, StreamedSpan]" if has_span_streaming_enabled(client.options): parent = sentry_sdk.traces.get_current_span() @@ -99,11 +89,7 @@ def _start_gc_span(info: "Dict[str, Any]") -> None: active=False, ) else: - legacy_parent = sentry_sdk.get_current_span() - if legacy_parent is None: - return - - span = legacy_parent.start_child( + span = sentry_sdk.start_span( op=OP.GC, name="garbage collection", origin=GcIntegration.origin, From e7ebe9105f472909d50e0aed16ea028530a4fdd7 Mon Sep 17 00:00:00 2001 From: Ivana Kellyer Date: Wed, 14 Oct 2026 15:57:36 +0200 Subject: [PATCH 4/4] . --- sentry_sdk/integrations/gc.py | 4 ++-- tests/integrations/gc/test_gc.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/gc.py b/sentry_sdk/integrations/gc.py index 1019a5e8ff..1d84d17ec7 100644 --- a/sentry_sdk/integrations/gc.py +++ b/sentry_sdk/integrations/gc.py @@ -80,7 +80,7 @@ def _start_gc_span(info: "Dict[str, Any]") -> None: return span = sentry_sdk.traces.start_span( - name="garbage collection", + name="Garbage collection", attributes={ "sentry.op": OP.GC, "sentry.origin": GcIntegration.origin, @@ -91,7 +91,7 @@ def _start_gc_span(info: "Dict[str, Any]") -> None: else: span = sentry_sdk.start_span( op=OP.GC, - name="garbage collection", + name="Garbage collection", origin=GcIntegration.origin, ) span.set_data(SPANDATA.CPYTHON_GC_GENERATION, generation) diff --git a/tests/integrations/gc/test_gc.py b/tests/integrations/gc/test_gc.py index 193f68bc55..5a42e46752 100644 --- a/tests/integrations/gc/test_gc.py +++ b/tests/integrations/gc/test_gc.py @@ -99,7 +99,7 @@ def collect(): (span,) = gc_spans name, op, origin, span_parent_id, data = span - assert name == "garbage collection" + assert name == "Garbage collection" assert op == "gc" assert origin == "auto.gc" assert span_parent_id == parent_span_id