From 76f22853410d3ded872cbfe1430852cf8c048962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Wed, 13 May 2026 01:46:21 +0200 Subject: [PATCH 1/3] gh-149718: Aggregate same stack frames in Tachyon in some collectors (#149719) --- Lib/profiling/sampling/collector.py | 2 + Lib/profiling/sampling/gecko_collector.py | 2 + Lib/profiling/sampling/heatmap_collector.py | 7 +- Lib/profiling/sampling/pstats_collector.py | 2 + Lib/profiling/sampling/sample.py | 33 +++++++- Lib/profiling/sampling/stack_collector.py | 2 + Lib/test/test_profiling/test_heatmap.py | 15 ++++ .../test_sampling_profiler/test_profiler.py | 79 ++++++++++++++++++- ...-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst | 4 + 9 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index 81ec6344ebdea4..8e0f0c44c4f8f3 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -143,6 +143,8 @@ def iter_async_frames(awaited_info_list): class Collector(ABC): + aggregating = False + @abstractmethod def collect(self, stack_frames, timestamps_us=None): """Collect profiling data from stack frames. diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 8986194268b3ce..54392af9500008 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -63,6 +63,8 @@ class GeckoCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False, opcodes=False): self.sample_interval_usec = sample_interval_usec self.skip_idle = skip_idle diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index 5c36d78f5535e7..6e650ec08f410b 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -452,7 +452,8 @@ def process_frames(self, frames, thread_id, weight=1): next_lineno = extract_lineno(next_frame[1]) self._record_call_relationship( (filename, lineno, funcname), - (next_frame[0], next_lineno, next_frame[2]) + (next_frame[0], next_lineno, next_frame[2]), + weight=weight, ) def _is_valid_frame(self, filename, lineno): @@ -561,7 +562,7 @@ def _get_bytecode_data_for_line(self, filename, lineno): result.sort(key=lambda x: (-x['samples'], x['opcode'])) return result - def _record_call_relationship(self, callee_frame, caller_frame): + def _record_call_relationship(self, callee_frame, caller_frame, weight=1): """Record caller/callee relationship between adjacent frames.""" callee_filename, callee_lineno, callee_funcname = callee_frame caller_filename, caller_lineno, caller_funcname = caller_frame @@ -587,7 +588,7 @@ def _record_call_relationship(self, callee_frame, caller_frame): # Count this call edge for path analysis edge_key = (caller_key, callee_key) - self.edge_samples[edge_key] += 1 + self.edge_samples[edge_key] += weight def export(self, output_path): """Export heatmap data as HTML files in a directory. diff --git a/Lib/profiling/sampling/pstats_collector.py b/Lib/profiling/sampling/pstats_collector.py index 50500296c15acc..43b1daf2a119d4 100644 --- a/Lib/profiling/sampling/pstats_collector.py +++ b/Lib/profiling/sampling/pstats_collector.py @@ -8,6 +8,8 @@ class PstatsCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False): self.result = collections.defaultdict( lambda: dict(total_rec_calls=0, direct_calls=0, cumulative_calls=0) diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 5bbe2483581333..b9e7e2625d09e4 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -47,6 +47,9 @@ def _pause_threads(unwinder, blocking): # If fewer samples are collected, we skip the TUI and just print a message MIN_SAMPLES_FOR_TUI = 200 +# Maximum number of consecutive identical samples to keep before flushing. +MAX_PENDING_SAMPLES = 8192 + class SampleProfiler: def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, opcodes=False, skip_non_matching_threads=True, collect_stats=False, blocking=False): self.pid = pid @@ -109,6 +112,20 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): last_sample_time = start_time realtime_update_interval = 1.0 # Update every second last_realtime_update = start_time + aggregating = getattr(collector, 'aggregating', False) is True + prev_stack = None + pending_count = 0 + pending_timestamps = [] if aggregating else None + + def flush_pending(): + nonlocal pending_count, pending_timestamps + if pending_count == 0: + return + pending_count = 0 + ts = pending_timestamps + pending_timestamps = [] + collector.collect(prev_stack, timestamps_us=ts) + try: while duration_sec is None or running_time_sec < duration_sec: # Check if live collector wants to stop @@ -116,6 +133,7 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): break current_time = time.perf_counter() + current_time_us = int(current_time * 1_000_000) if next_time > current_time: sleep_time = (next_time - current_time) * 0.9 if sleep_time > 0.0001: @@ -125,13 +143,24 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): stack_frames = self._get_stack_trace( async_aware=async_aware ) - collector.collect(stack_frames) + if aggregating: + if stack_frames != prev_stack: + flush_pending() + prev_stack = stack_frames + pending_count += 1 + pending_timestamps.append(current_time_us) + if pending_count >= MAX_PENDING_SAMPLES: + flush_pending() + else: + collector.collect(stack_frames) except ProcessLookupError as e: running_time_sec = current_time - start_time break except (RuntimeError, UnicodeDecodeError, MemoryError, OSError): + flush_pending() collector.collect_failed_sample() errors += 1 + prev_stack = None except Exception as e: if not _is_process_running(self.pid): break @@ -163,6 +192,8 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): interrupted = True running_time_sec = time.perf_counter() - start_time print("Interrupted by user.") + finally: + flush_pending() # Clear real-time stats line if it was being displayed if self.realtime_stats and len(self.sample_intervals) > 0: diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 60df026ed76a6c..42281dc6454c83 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -16,6 +16,8 @@ class StackTraceCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False): self.sample_interval_usec = sample_interval_usec self.skip_idle = skip_idle diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py index b2acb1cf577341..ee27fdd3fa3053 100644 --- a/Lib/test/test_profiling/test_heatmap.py +++ b/Lib/test/test_profiling/test_heatmap.py @@ -345,6 +345,21 @@ def test_process_frames_tracks_edge_samples(self): # Check that edge count is tracked self.assertGreater(len(collector.edge_samples), 0) + def test_process_frames_weight_applies_to_identical_samples(self): + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('callee.py', (5, 5, -1, -1), 'callee', None), + ('caller.py', (10, 10, -1, -1), 'caller', None), + ] + + collector.process_frames(frames, thread_id=1, weight=5) + + edge_key = (('caller.py', 10), ('callee.py', 5)) + self.assertEqual(collector.edge_samples[edge_key], 5) + self.assertEqual(collector.line_samples[('callee.py', 5)], 5) + self.assertEqual(collector.line_samples[('caller.py', 10)], 5) + def test_process_frames_handles_empty_frames(self): """Test that process_frames handles empty frame list.""" collector = HeatmapCollector(sample_interval_usec=100) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py index 68bc59a5414a05..2f5a5e27328659 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py @@ -198,8 +198,83 @@ def test_sample_profiler_sample_method_timing(self): self.assertIn("samples", result) # Verify collector was called multiple times - self.assertGreaterEqual(mock_collector.collect.call_count, 5) - self.assertLessEqual(mock_collector.collect.call_count, 11) + total_weight = sum( + len(c.kwargs.get("timestamps_us") or [None]) + for c in mock_collector.collect.call_args_list + ) + self.assertGreaterEqual(total_weight, 5) + self.assertLessEqual(total_weight, 11) + + def test_sample_profiler_does_not_buffer_non_aggregating_collectors(self): + """Test that non-aggregating collectors get each sample immediately.""" + + stack_frames = [mock.sentinel.stack_frames] + mock_collector = mock.MagicMock() + mock_collector.aggregating = False + + with self._patched_unwinder() as u: + u.instance.get_stack_trace.return_value = stack_frames + + manager = mock.Mock() + manager.attach_mock(u.instance.get_stack_trace, "unwind") + manager.attach_mock(mock_collector.collect, "collect") + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + + times = [0.0, 0.01, 0.011, 0.02, 0.03] + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.025) + + self.assertEqual( + manager.mock_calls, + [ + mock.call.unwind(), + mock.call.collect(stack_frames), + mock.call.unwind(), + mock.call.collect(stack_frames), + ], + ) + + def test_sample_profiler_flushes_aggregated_batches_at_limit(self): + """Test that aggregating collectors flush after MAX_PENDING_SAMPLES samples.""" + + stack_frames = [mock.sentinel.stack_frames] + mock_collector = mock.MagicMock() + mock_collector.aggregating = True + + with self._patched_unwinder() as u: + u.instance.get_stack_trace.return_value = stack_frames + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + + times = [ + 0.0, + 0.01, 0.011, + 0.02, 0.021, + 0.03, 0.031, + 0.04, 0.041, + 0.05, 0.051, + ] + with mock.patch("profiling.sampling.sample.MAX_PENDING_SAMPLES", 2): + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.045) + + batches = [ + (c.args[0], len(c.kwargs["timestamps_us"])) + for c in mock_collector.collect.call_args_list + ] + self.assertEqual( + batches, + [(stack_frames, 2), (stack_frames, 2), (stack_frames, 1)], + ) def test_sample_profiler_error_handling(self): """Test that the sample method handles errors gracefully.""" diff --git a/Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst b/Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst new file mode 100644 index 00000000000000..25344e5a90f022 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst @@ -0,0 +1,4 @@ +Coalesce consecutive identical stack frames in Tachyon, so aggregating +collectors (pstats, collapsed, flamegraph, gecko) receive one collect. +Improves sample rate 3x, error rate and missed rate drop by 70%. Patch by +Maurycy Pawłowski-Wieroński. From 4087ff859958abc897711b501bb66dc308890ba5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 12 May 2026 19:59:09 -0700 Subject: [PATCH 2/3] gh-149642: Fix interaction between exec and lazy_imports=all (#149643) --- Lib/test/test_lazy_import/__init__.py | 53 +++++++++++++++++++ ...-05-10-07-42-36.gh-issue-149642.6ZksML.rst | 2 + Python/ceval.c | 16 ++++-- 3 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index ea534a8ee5b981..5d770eeae07a15 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -301,6 +301,15 @@ def f(): f() self.assertIn("only allowed at module level", str(cm.exception)) + def test_lazy_import_exec_in_class(self): + """lazy import via exec() inside a class should raise SyntaxError.""" + # exec() inside a class body also has non-module-level locals. + with self.assertRaises(SyntaxError) as cm: + class C: + exec("lazy import json") + + self.assertIn("only allowed at module level", str(cm.exception)) + @support.requires_subprocess() def test_lazy_import_exec_at_module_level(self): """lazy import via exec() at module level should work.""" @@ -352,6 +361,50 @@ def test_eager_import_func(self): f = test.test_lazy_import.data.eager_import_func.f self.assertEqual(type(f()), type(sys)) + def test_exec_import_func(self): + """Implicit lazy imports via exec() inside functions should be eager.""" + sys.set_lazy_imports("all") + + def f(): + exec("import test.test_lazy_import.data.basic2") + + f() + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_exec_import_func_with_lazy_modules(self): + """__lazy_modules__ should not make exec() imports lazy inside functions.""" + globals()["__lazy_modules__"] = ["test.test_lazy_import.data.basic2"] + try: + def f(): + exec("import test.test_lazy_import.data.basic2") + + f() + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + finally: + del globals()["__lazy_modules__"] + + def test_exec_import_class(self): + """Implicit lazy imports via exec() inside classes should be eager.""" + sys.set_lazy_imports("all") + + class C: + exec("import test.test_lazy_import.data.basic2") + + self.assertIsNotNone(C) + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_exec_import_class_with_lazy_modules(self): + """__lazy_modules__ should not make exec() imports lazy inside classes.""" + globals()["__lazy_modules__"] = ["test.test_lazy_import.data.basic2"] + try: + class C: + exec("import test.test_lazy_import.data.basic2") + + self.assertIsNotNone(C) + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + finally: + del globals()["__lazy_modules__"] + class WithStatementTests(unittest.TestCase): """Tests for lazy imports in with statement context.""" diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst new file mode 100644 index 00000000000000..815a084db69d8d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst @@ -0,0 +1,2 @@ +Allow imports inside ``exec()`` calls within functions under +``PYTHON_LAZY_IMPORTS=all``. diff --git a/Python/ceval.c b/Python/ceval.c index 060e948e6b01c9..a080ae42b93766 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3059,25 +3059,35 @@ check_lazy_import_compatibility(PyThreadState *tstate, PyObject *globals, return res; } +static int +is_lazy_import_module_level(void) +{ + _PyInterpreterFrame *frame = _PyEval_GetFrame(); + return frame != NULL && frame->f_globals == frame->f_locals; +} + PyObject * _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level, int lazy) { PyObject *res = NULL; + PyImport_LazyImportsMode mode = PyImport_GetLazyImportsMode(); // Check if global policy overrides the local syntax - switch (PyImport_GetLazyImportsMode()) { + switch (mode) { case PyImport_LAZY_NONE: lazy = 0; break; case PyImport_LAZY_ALL: - lazy = 1; + if (!lazy) { + lazy = is_lazy_import_module_level(); + } break; case PyImport_LAZY_NORMAL: break; } - if (!lazy && PyImport_GetLazyImportsMode() != PyImport_LAZY_NONE) { + if (!lazy && mode != PyImport_LAZY_NONE && is_lazy_import_module_level()) { // See if __lazy_modules__ forces this to be lazy. lazy = check_lazy_import_compatibility(tstate, globals, name, level); if (lazy < 0) { From a4e51c8dac9fdd49ae26ff8c6cd3c808fd8ba15e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 12 May 2026 20:00:01 -0700 Subject: [PATCH 3/3] gh-149574: Document that is_typeddict, is_protocol, is_dataclass, isclass return False for generic aliases (#149604) --- Doc/library/dataclasses.rst | 3 ++- Doc/library/inspect.rst | 3 +++ Doc/library/stdtypes.rst | 3 ++- Doc/library/typing.rst | 30 +++++++++++++++++++++++++++--- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 0bce3e5b762b8b..a09c28ad979158 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -498,7 +498,8 @@ Module contents .. function:: is_dataclass(obj) Return ``True`` if its parameter is a dataclass (including subclasses of a - dataclass) or an instance of one, otherwise return ``False``. + dataclass, but not including :ref:`generic aliases `) + or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 8713765b8aebfb..48ae9147587c64 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -416,6 +416,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a class, whether built-in or created in Python code. + This function returns ``False`` for :ref:`generic aliases ` of classes, + such as ``list[int]``. + .. function:: ismethod(object) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 3d943566be34ff..e3bd1a46891adc 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5858,7 +5858,8 @@ type and the :class:`bytes` data type: ``GenericAlias`` objects are instances of the class :class:`types.GenericAlias`, which can also be used to create ``GenericAlias`` -objects directly. +objects directly. Specializations of user-defined :ref:`generic classes ` +may not be instances of :class:`types.GenericAlias`, but they provide similar functionality. .. describe:: T[X, Y, ...] diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index dca51b8014da5a..71b395c80166cc 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3633,14 +3633,27 @@ Introspection helpers Determine if a type is a :class:`Protocol`. - For example:: + For example: + + .. testcode:: class P(Protocol): def a(self) -> str: ... b: int - is_protocol(P) # => True - is_protocol(int) # => False + assert is_protocol(P) + assert not is_protocol(int) + + This function only returns true for ``Protocol`` classes, not for + :ref:`generic aliases ` of them: + + .. testcode:: + + class GenericP[T](Protocol): + def a(self) -> T: ... + b: int + + assert not is_protocol(GenericP[int]) .. versionadded:: 3.13 @@ -3663,6 +3676,17 @@ Introspection helpers # not a typed dict itself assert not is_typeddict(TypedDict) + This function only returns true for ``TypedDict`` classes, not for + :ref:`generic aliases ` of them: + + .. testcode:: + + class GenericFilm[T](TypedDict): + title: str + year: T + + assert not is_typeddict(GenericFilm[int]) + .. versionadded:: 3.10 .. class:: ForwardRef