From 332adfab82cde9265a4bcd014d13eff2d1f1db7f Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Thu, 2 Apr 2026 18:17:44 +0300 Subject: [PATCH 01/16] modified project metadata to include repo links and to make the minimum version python 3.12 --- .github/workflows/ci.yml | 4 ++-- pyproject.toml | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36dac1b..7495e7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: uses: astral-sh/setup-uv@v5 with: enable-cache: true - python-version: "3.13" + python-version: "3.12" - name: Ruff Format Check run: uv run ruff format --check . - name: Ruff Lint Check @@ -34,7 +34,7 @@ jobs: uses: astral-sh/setup-uv@v5 with: enable-cache: true - python-version: "3.13" + python-version: "3.12" - name: Run tests with Pytest and 100% Coverage run: uv run python -m pytest --cov=offline_debug --cov-report=term-missing --cov-fail-under=100 diff --git a/pyproject.toml b/pyproject.toml index ebf2ddb..d54dac2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,14 +2,19 @@ name = "offline-debug" version = "0.1.0-beta1" description = "Debug exceptions offline by saving them to a dump and raising them at a later point." +readme = "README.md" authors = [ { name = "Itai Elidan", email = "itaielidan@gmail.com" } ] -requires-python = ">=3.13" +requires-python = ">=3.12" dependencies = [ "typing-extensions>=4.15.0", ] +[project.urls] +Homepage = "https://github.com/INTODAN/offline-debug" +Repository = "https://github.com/INTODAN/offline-debug" + [build-system] requires = ["uv_build"] build-backend = "uv_build" From 81e634af9624f484e4392fc4bac5df7ba5897102 Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Thu, 2 Apr 2026 18:18:39 +0300 Subject: [PATCH 02/16] bumped version out of beta --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d54dac2..495dc03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "offline-debug" -version = "0.1.0-beta1" +version = "0.1.0" description = "Debug exceptions offline by saving them to a dump and raising them at a later point." readme = "README.md" authors = [ From e1b2e22c321e144e9760ef842b1d196aa3a48374 Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Thu, 2 Apr 2026 19:51:24 +0300 Subject: [PATCH 03/16] cleaned up code and added debug.py file for manual runs. --- offline_debug/serializer.py | 88 +++++++++++++++++++++++-------------- tests/debug.py | 25 +++++++++++ 2 files changed, 81 insertions(+), 32 deletions(-) create mode 100644 tests/debug.py diff --git a/offline_debug/serializer.py b/offline_debug/serializer.py index cba8c81..e51f147 100644 --- a/offline_debug/serializer.py +++ b/offline_debug/serializer.py @@ -1,7 +1,9 @@ +from pathlib import Path +from dataclasses import dataclass +from typing import Optional, List, Any, Dict, Never import pickle import marshal import types -import typing import ctypes import sys @@ -18,6 +20,27 @@ _py_thread_state_get = ctypes.pythonapi.PyThreadState_Get _py_thread_state_get.restype = ctypes.c_void_p +# Constants for serialization +_INTERNAL_ATTRIBUTES = ("__builtins__", "__doc__", "__loader__", "__package__", "__spec__") + + +@dataclass +class _FrameData: + code: bytes + globals: Dict[str, Any] + locals: Dict[str, Any] + lasti: int + lineno: int + stack_depth: int + + +@dataclass +class _ExceptionData: + exc_pickle: bytes + tb_frames: List[_FrameData] + cause: Optional["_ExceptionData"] = None + context: Optional["_ExceptionData"] = None + # Define ctypes structure to access f_back in PyFrameObject class _PyObject(ctypes.Structure): @@ -43,7 +66,7 @@ def _filter_dict(d: dict) -> dict: """Filter dictionary to include only picklable items.""" result = {} for k, v in d.items(): - if k in ("__builtins__", "__doc__", "__loader__", "__package__", "__spec__"): + if k in _INTERNAL_ATTRIBUTES: continue try: pickle.dumps(v) @@ -53,20 +76,20 @@ def _filter_dict(d: dict) -> dict: return result -def _serialize_exc_data(exc: BaseException) -> dict: +def _serialize_exc_data(exc: BaseException) -> _ExceptionData: tb_frames = [] curr_tb = exc.__traceback__ while curr_tb: f = curr_tb.tb_frame tb_frames.append( - { - "code": marshal.dumps(f.f_code), - "globals": _filter_dict(f.f_globals), - "locals": _filter_dict(f.f_locals), - "lasti": curr_tb.tb_lasti, - "lineno": curr_tb.tb_lineno, - "stack_depth": _get_stack_depth(f), - } + _FrameData( + code=marshal.dumps(f.f_code), + globals=_filter_dict(f.f_globals), + locals=_filter_dict(f.f_locals), + lasti=curr_tb.tb_lasti, + lineno=curr_tb.tb_lineno, + stack_depth=_get_stack_depth(f), + ) ) curr_tb = curr_tb.tb_next @@ -77,35 +100,35 @@ def _serialize_exc_data(exc: BaseException) -> dict: RuntimeError(f"Unpicklable exception {type(exc).__name__}: {str(exc)}") ) - return { - "exc_pickle": exc_pickle, - "tb_frames": tb_frames, - "cause": _serialize_exc_data(exc.__cause__) if exc.__cause__ else None, - "context": _serialize_exc_data(exc.__context__) if exc.__context__ else None, - } + return _ExceptionData( + exc_pickle=exc_pickle, + tb_frames=tb_frames, + cause=_serialize_exc_data(exc.__cause__) if exc.__cause__ else None, + context=_serialize_exc_data(exc.__context__) if exc.__context__ else None, + ) -def save_traceback(exc: Exception, file_path: str): +def save_traceback(exc: Exception, file_path: str | Path): """Serialize an exception and its traceback to a file.""" data = _serialize_exc_data(exc) with open(file_path, "wb") as f: pickle.dump(data, f) -def _reconstruct_exc_data(data: dict) -> Exception: - exc = pickle.loads(data["exc_pickle"]) +def _reconstruct_exc_data(data: _ExceptionData) -> Exception: + exc = pickle.loads(data.exc_pickle) tstate = _py_thread_state_get() reconstructed_frames = [] prev_frame = None - for f_data in data["tb_frames"]: - code = marshal.loads(f_data["code"]) + for f_data in data.tb_frames: + code = marshal.loads(f_data.code) - frame = _py_frame_new(tstate, code, f_data["globals"], {}) + frame = _py_frame_new(tstate, code, f_data.globals, {}) - if f_data["locals"]: - frame.f_locals.update(f_data["locals"]) + if f_data.locals: + frame.f_locals.update(f_data.locals) if prev_frame: frame_ptr = _PyFrameObject.from_address(id(frame)) @@ -119,22 +142,22 @@ def _reconstruct_exc_data(data: dict) -> Exception: tb = types.TracebackType( tb_next=tb_next, tb_frame=frame, - tb_lasti=f_data["lasti"], - tb_lineno=f_data["lineno"], + tb_lasti=f_data.lasti, + tb_lineno=f_data.lineno, ) tb_next = tb exc = exc.with_traceback(tb_next) - if data["cause"]: - exc.__cause__ = _reconstruct_exc_data(data["cause"]) - if data["context"]: - exc.__context__ = _reconstruct_exc_data(data["context"]) + if data.cause: + exc.__cause__ = _reconstruct_exc_data(data.cause) + if data.context: + exc.__context__ = _reconstruct_exc_data(data.context) return exc -def load_traceback(file_path: str) -> typing.Never: +def load_traceback(file_path: str | Path) -> Never: """Load an exception and its traceback from a file and raise it.""" with open(file_path, "rb") as f: data = pickle.load(f) @@ -165,3 +188,4 @@ def load_traceback(file_path: str) -> typing.Never: exc = exc.with_traceback(tb_chain) raise exc + diff --git a/tests/debug.py b/tests/debug.py new file mode 100644 index 0000000..e148b8a --- /dev/null +++ b/tests/debug.py @@ -0,0 +1,25 @@ +""" +Intended for manual use, run the test in debug mode and view the exceptions and the function's locals +""" + +import tempfile +from pathlib import Path + +from offline_debug import save_traceback, load_traceback + + +def failure(): + def exception_raising_func() -> None: + local = "local" + raise ValueError(f"exception {local}") + + with tempfile.TemporaryDirectory() as tmpdir: + try: + exception_raising_func() + except ValueError as e: + save_traceback(e, Path(tmpdir) / "traceback.dump") + load_traceback(Path(tmpdir) / "traceback.dump") + + +if __name__ == "__main__": + failure() From b0a7ce0c11a4bea410b9ce13445f2bb66debbb7c Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Thu, 2 Apr 2026 20:02:58 +0300 Subject: [PATCH 04/16] added typing by using dataclasses --- offline_debug/serializer.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/offline_debug/serializer.py b/offline_debug/serializer.py index e51f147..9dddb7c 100644 --- a/offline_debug/serializer.py +++ b/offline_debug/serializer.py @@ -1,6 +1,6 @@ from pathlib import Path from dataclasses import dataclass -from typing import Optional, List, Any, Dict, Never +from typing import Optional, List, Any, Dict, Never, cast import pickle import marshal import types @@ -54,11 +54,12 @@ class _PyFrameObject(ctypes.Structure): ] -def _get_stack_depth(frame): +def _get_stack_depth(frame: types.FrameType) -> int: depth = 0 - while frame: + curr: Optional[types.FrameType] = frame + while curr: depth += 1 - frame = frame.f_back + curr = curr.f_back return depth @@ -77,7 +78,7 @@ def _filter_dict(d: dict) -> dict: def _serialize_exc_data(exc: BaseException) -> _ExceptionData: - tb_frames = [] + tb_frames: List[_FrameData] = [] curr_tb = exc.__traceback__ while curr_tb: f = curr_tb.tb_frame @@ -116,16 +117,17 @@ def save_traceback(exc: Exception, file_path: str | Path): def _reconstruct_exc_data(data: _ExceptionData) -> Exception: - exc = pickle.loads(data.exc_pickle) + exc = cast(Exception, pickle.loads(data.exc_pickle)) tstate = _py_thread_state_get() - reconstructed_frames = [] - prev_frame = None + reconstructed_frames: List[tuple[types.FrameType, _FrameData]] = [] + prev_frame: Optional[types.FrameType] = None for f_data in data.tb_frames: code = marshal.loads(f_data.code) - frame = _py_frame_new(tstate, code, f_data.globals, {}) + # PyFrame_New returns a new reference to a PyFrameObject + frame = cast(types.FrameType, _py_frame_new(tstate, code, f_data.globals, {})) if f_data.locals: frame.f_locals.update(f_data.locals) @@ -137,7 +139,7 @@ def _reconstruct_exc_data(data: _ExceptionData) -> Exception: reconstructed_frames.append((frame, f_data)) prev_frame = frame - tb_next = None + tb_next: Optional[types.TracebackType] = None for frame, f_data in reversed(reconstructed_frames): tb = types.TracebackType( tb_next=tb_next, @@ -160,12 +162,12 @@ def _reconstruct_exc_data(data: _ExceptionData) -> Exception: def load_traceback(file_path: str | Path) -> Never: """Load an exception and its traceback from a file and raise it.""" with open(file_path, "rb") as f: - data = pickle.load(f) + data = cast(_ExceptionData, pickle.load(f)) exc = _reconstruct_exc_data(data) - current_frames = [] - curr = sys._getframe(1) + current_frames: List[types.FrameType] = [] + curr: Optional[types.FrameType] = sys._getframe(1) while curr: current_frames.append(curr) curr = curr.f_back @@ -177,7 +179,7 @@ def load_traceback(file_path: str | Path) -> Never: frame_ptr = _PyFrameObject.from_address(id(reconstructed_outer)) frame_ptr.f_back = id(caller_frame) - tb_chain = exc.__traceback__ + tb_chain: Optional[types.TracebackType] = exc.__traceback__ for frame in current_frames: tb_chain = types.TracebackType( tb_next=tb_chain, @@ -188,4 +190,3 @@ def load_traceback(file_path: str | Path) -> Never: exc = exc.with_traceback(tb_chain) raise exc - From 19eafc06fa7ae78f605c19282fc7950aa81c4d80 Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Thu, 2 Apr 2026 20:12:41 +0300 Subject: [PATCH 05/16] continued to clean up the code. --- offline_debug/serializer.py | 27 ++++++++++++++++++--------- tests/debug.py | 3 +++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/offline_debug/serializer.py b/offline_debug/serializer.py index 9dddb7c..9a6ee9a 100644 --- a/offline_debug/serializer.py +++ b/offline_debug/serializer.py @@ -1,6 +1,7 @@ +from __future__ import annotations from pathlib import Path from dataclasses import dataclass -from typing import Optional, List, Any, Dict, Never, cast +from typing import List, Any, Dict, Never, cast import pickle import marshal import types @@ -20,7 +21,9 @@ _py_thread_state_get = ctypes.pythonapi.PyThreadState_Get _py_thread_state_get.restype = ctypes.c_void_p -# Constants for serialization +# Internal attributes that are either unpicklable or redundant in a new process. +# We exclude these specifically because they are automatically recreated +# when the new frame is initialized or when the module is imported. _INTERNAL_ATTRIBUTES = ("__builtins__", "__doc__", "__loader__", "__package__", "__spec__") @@ -38,8 +41,8 @@ class _FrameData: class _ExceptionData: exc_pickle: bytes tb_frames: List[_FrameData] - cause: Optional["_ExceptionData"] = None - context: Optional["_ExceptionData"] = None + cause: _ExceptionData | None = None + context: _ExceptionData | None = None # Define ctypes structure to access f_back in PyFrameObject @@ -56,7 +59,7 @@ class _PyFrameObject(ctypes.Structure): def _get_stack_depth(frame: types.FrameType) -> int: depth = 0 - curr: Optional[types.FrameType] = frame + curr: types.FrameType | None = frame while curr: depth += 1 curr = curr.f_back @@ -70,6 +73,9 @@ def _filter_dict(d: dict) -> dict: if k in _INTERNAL_ATTRIBUTES: continue try: + # We must verify if the value is picklable because many globals + # (like open file handles, database connections, or modules) + # cannot be saved to disk. pickle.dumps(v) result[k] = v except Exception: @@ -122,7 +128,7 @@ def _reconstruct_exc_data(data: _ExceptionData) -> Exception: tstate = _py_thread_state_get() reconstructed_frames: List[tuple[types.FrameType, _FrameData]] = [] - prev_frame: Optional[types.FrameType] = None + prev_frame: types.FrameType | None = None for f_data in data.tb_frames: code = marshal.loads(f_data.code) @@ -132,6 +138,9 @@ def _reconstruct_exc_data(data: _ExceptionData) -> Exception: if f_data.locals: frame.f_locals.update(f_data.locals) + # Stitch the frames together to reconstruct the full stack trace. + # This allows tools like pdb or IDE debuggers to navigate up and down + # the reconstructed stack. if prev_frame: frame_ptr = _PyFrameObject.from_address(id(frame)) frame_ptr.f_back = id(prev_frame) @@ -139,7 +148,7 @@ def _reconstruct_exc_data(data: _ExceptionData) -> Exception: reconstructed_frames.append((frame, f_data)) prev_frame = frame - tb_next: Optional[types.TracebackType] = None + tb_next: types.TracebackType | None = None for frame, f_data in reversed(reconstructed_frames): tb = types.TracebackType( tb_next=tb_next, @@ -167,7 +176,7 @@ def load_traceback(file_path: str | Path) -> Never: exc = _reconstruct_exc_data(data) current_frames: List[types.FrameType] = [] - curr: Optional[types.FrameType] = sys._getframe(1) + curr: types.FrameType | None = sys._getframe(1) while curr: current_frames.append(curr) curr = curr.f_back @@ -179,7 +188,7 @@ def load_traceback(file_path: str | Path) -> Never: frame_ptr = _PyFrameObject.from_address(id(reconstructed_outer)) frame_ptr.f_back = id(caller_frame) - tb_chain: Optional[types.TracebackType] = exc.__traceback__ + tb_chain: types.TracebackType | None = exc.__traceback__ for frame in current_frames: tb_chain = types.TracebackType( tb_next=tb_chain, diff --git a/tests/debug.py b/tests/debug.py index e148b8a..01ac0ca 100644 --- a/tests/debug.py +++ b/tests/debug.py @@ -7,8 +7,10 @@ from offline_debug import save_traceback, load_traceback +global_variable = 1 def failure(): + global global_variable def exception_raising_func() -> None: local = "local" raise ValueError(f"exception {local}") @@ -18,6 +20,7 @@ def exception_raising_func() -> None: exception_raising_func() except ValueError as e: save_traceback(e, Path(tmpdir) / "traceback.dump") + global_variable += 1 load_traceback(Path(tmpdir) / "traceback.dump") From 6c6e72d58e5010dde069479b210165b7d64ce9af Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Thu, 2 Apr 2026 22:56:31 +0300 Subject: [PATCH 06/16] ruff fixes --- offline_debug/__init__.py | 6 +- offline_debug/serializer.py | 68 +++++++++------- pyproject.toml | 25 ++++++ tests/__init__.py | 1 + tests/debug.py | 31 +++++--- tests/test_serializer.py | 154 +++++++++++++++++++++--------------- 6 files changed, 178 insertions(+), 107 deletions(-) diff --git a/offline_debug/__init__.py b/offline_debug/__init__.py index 53b6712..47ace16 100644 --- a/offline_debug/__init__.py +++ b/offline_debug/__init__.py @@ -1,3 +1,5 @@ -from .serializer import save_traceback, load_traceback +"""Tool for serializing and reconstructing Python exceptions with full stack traces.""" -__all__ = ["save_traceback", "load_traceback"] +from .serializer import load_traceback, save_traceback + +__all__ = ["load_traceback", "save_traceback"] diff --git a/offline_debug/serializer.py b/offline_debug/serializer.py index 9a6ee9a..0726e8a 100644 --- a/offline_debug/serializer.py +++ b/offline_debug/serializer.py @@ -1,12 +1,15 @@ +"""Functions for serializing and reconstructing exceptions with their tracebacks.""" + from __future__ import annotations -from pathlib import Path -from dataclasses import dataclass -from typing import List, Any, Dict, Never, cast -import pickle -import marshal -import types + import ctypes +import marshal +import pickle import sys +import types +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Never, cast # Define C API for frame creation _py_frame_new = ctypes.pythonapi.PyFrame_New @@ -22,16 +25,18 @@ _py_thread_state_get.restype = ctypes.c_void_p # Internal attributes that are either unpicklable or redundant in a new process. -# We exclude these specifically because they are automatically recreated +# We exclude these specifically because they are automatically recreated # when the new frame is initialized or when the module is imported. _INTERNAL_ATTRIBUTES = ("__builtins__", "__doc__", "__loader__", "__package__", "__spec__") @dataclass class _FrameData: + """Serialized data for a single stack frame.""" + code: bytes - globals: Dict[str, Any] - locals: Dict[str, Any] + globals: dict[str, Any] + locals: dict[str, Any] lasti: int lineno: int stack_depth: int @@ -39,8 +44,10 @@ class _FrameData: @dataclass class _ExceptionData: + """Serialized data for an exception and its traceback.""" + exc_pickle: bytes - tb_frames: List[_FrameData] + tb_frames: list[_FrameData] cause: _ExceptionData | None = None context: _ExceptionData | None = None @@ -58,6 +65,7 @@ class _PyFrameObject(ctypes.Structure): def _get_stack_depth(frame: types.FrameType) -> int: + """Calculate the depth of the current stack frame.""" depth = 0 curr: types.FrameType | None = frame while curr: @@ -73,18 +81,19 @@ def _filter_dict(d: dict) -> dict: if k in _INTERNAL_ATTRIBUTES: continue try: - # We must verify if the value is picklable because many globals - # (like open file handles, database connections, or modules) + # We must verify if the value is picklable because many globals + # (like open file handles, database connections, or modules) # cannot be saved to disk. pickle.dumps(v) result[k] = v - except Exception: - result[k] = f"" + except Exception: # noqa: BLE001 + result[k] = f"" return result def _serialize_exc_data(exc: BaseException) -> _ExceptionData: - tb_frames: List[_FrameData] = [] + """Recursively serialize exception data into dataclasses.""" + tb_frames: list[_FrameData] = [] curr_tb = exc.__traceback__ while curr_tb: f = curr_tb.tb_frame @@ -102,9 +111,9 @@ def _serialize_exc_data(exc: BaseException) -> _ExceptionData: try: exc_pickle = pickle.dumps(exc) - except Exception: + except Exception: # noqa: BLE001 exc_pickle = pickle.dumps( - RuntimeError(f"Unpicklable exception {type(exc).__name__}: {str(exc)}") + RuntimeError(f"Unpicklable exception {type(exc).__name__}: {exc!s}") ) return _ExceptionData( @@ -115,31 +124,32 @@ def _serialize_exc_data(exc: BaseException) -> _ExceptionData: ) -def save_traceback(exc: Exception, file_path: str | Path): +def save_traceback(exc: BaseException, file_path: str | Path) -> None: """Serialize an exception and its traceback to a file.""" data = _serialize_exc_data(exc) - with open(file_path, "wb") as f: + with Path(file_path).open("wb") as f: pickle.dump(data, f) -def _reconstruct_exc_data(data: _ExceptionData) -> Exception: - exc = cast(Exception, pickle.loads(data.exc_pickle)) +def _reconstruct_exc_data(data: _ExceptionData) -> BaseException: + """Recursively reconstruct an exception from its serialized data.""" + exc = cast("BaseException", pickle.loads(data.exc_pickle)) # noqa: S301 tstate = _py_thread_state_get() - reconstructed_frames: List[tuple[types.FrameType, _FrameData]] = [] + reconstructed_frames: list[tuple[types.FrameType, _FrameData]] = [] prev_frame: types.FrameType | None = None for f_data in data.tb_frames: - code = marshal.loads(f_data.code) + code = marshal.loads(f_data.code) # noqa: S302 # PyFrame_New returns a new reference to a PyFrameObject - frame = cast(types.FrameType, _py_frame_new(tstate, code, f_data.globals, {})) + frame = cast("types.FrameType", _py_frame_new(tstate, code, f_data.globals, {})) if f_data.locals: frame.f_locals.update(f_data.locals) # Stitch the frames together to reconstruct the full stack trace. - # This allows tools like pdb or IDE debuggers to navigate up and down + # This allows tools like pdb or IDE debuggers to navigate up and down # the reconstructed stack. if prev_frame: frame_ptr = _PyFrameObject.from_address(id(frame)) @@ -170,13 +180,13 @@ def _reconstruct_exc_data(data: _ExceptionData) -> Exception: def load_traceback(file_path: str | Path) -> Never: """Load an exception and its traceback from a file and raise it.""" - with open(file_path, "rb") as f: - data = cast(_ExceptionData, pickle.load(f)) + with Path(file_path).open("rb") as f: + data = cast("_ExceptionData", pickle.load(f)) # noqa: S301 exc = _reconstruct_exc_data(data) - current_frames: List[types.FrameType] = [] - curr: types.FrameType | None = sys._getframe(1) + current_frames: list[types.FrameType] = [] + curr: types.FrameType | None = sys._getframe(1) # noqa: SLF001 while curr: current_frames.append(curr) curr = curr.f_back diff --git a/pyproject.toml b/pyproject.toml index 495dc03..69c1d83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,31 @@ dependencies = [ Homepage = "https://github.com/INTODAN/offline-debug" Repository = "https://github.com/INTODAN/offline-debug" +[tool.ruff] +line-length = 100 +target-version = "py312" + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + "D203", # One blank line before class + "D212", # Multi-line docstring summary should start at the first line + "COM812", # Trailing comma missing + "PLC0415", # import should be at top of file +] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = [ + "S101", # Use of assert detected + "SLF001", # Private member accessed + "D103", # Missing docstring in public function + "ANN201", # Missing return type annotation for public function + "ANN001", # Missing type annotation for function argument + "PLW0603", # Using the global statement + "TRY003", # Avoid specifying long messages outside the exception class + "EM101", # Exception must not use a string literal +] + [build-system] requires = ["uv_build"] build-backend = "uv_build" diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..bf72498 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the offline-debug package.""" diff --git a/tests/debug.py b/tests/debug.py index 01ac0ca..1f066c0 100644 --- a/tests/debug.py +++ b/tests/debug.py @@ -1,28 +1,37 @@ -""" -Intended for manual use, run the test in debug mode and view the exceptions and the function's locals -""" +"""Intended for manual use, run the test in debug mode and view the exceptions.""" + +from __future__ import annotations import tempfile from pathlib import Path -from offline_debug import save_traceback, load_traceback +from offline_debug.serializer import load_traceback, save_traceback global_variable = 1 -def failure(): - global global_variable + +def failure() -> None: + """Raise a nested exception for testing.""" + def exception_raising_func() -> None: local = "local" - raise ValueError(f"exception {local}") + msg = f"exception {local}" + raise ValueError(msg) with tempfile.TemporaryDirectory() as tmpdir: + dump_file = Path(tmpdir) / "exception.dump" try: exception_raising_func() except ValueError as e: - save_traceback(e, Path(tmpdir) / "traceback.dump") - global_variable += 1 - load_traceback(Path(tmpdir) / "traceback.dump") + save_traceback(e, dump_file) + + load_traceback(dump_file) if __name__ == "__main__": - failure() + try: + failure() + except ValueError: + import traceback + + traceback.print_exc() diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 32f3077..1e0a18c 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -1,48 +1,57 @@ -import pytest +"""Tests for the serializer module.""" + +from __future__ import annotations + import sys -from offline_debug import save_traceback, load_traceback +from typing import TYPE_CHECKING, Never + +import pytest + +from offline_debug import load_traceback, save_traceback + +if TYPE_CHECKING: + import types + from pathlib import Path -def get_stack_depth(frame): +def get_stack_depth(frame: types.FrameType | None) -> int: + """Calculate the depth of the given stack frame.""" depth = 0 - while frame: + curr = frame + while curr: depth += 1 - frame = frame.f_back + curr = curr.f_back return depth -def test_stack_depth_preservation(tmp_path): +def test_stack_depth_preservation(tmp_path: Path) -> None: + """Test that the stack depth is preserved during serialization.""" dump_file = tmp_path / "depth.dump" original_depths = [] - def level_3(): + def level_3() -> Never: f = sys._getframe() original_depths.append(get_stack_depth(f)) - raise ValueError("Depth error") + msg = "Depth error" + raise ValueError(msg) - def level_2(): + def level_2() -> None: f = sys._getframe() original_depths.append(get_stack_depth(f)) level_3() - def level_1(): + def level_1() -> None: f = sys._getframe() original_depths.append(get_stack_depth(f)) try: level_2() - except Exception as e: + except Exception as e: # noqa: BLE001 save_traceback(e, str(dump_file)) level_1() - # Now load and verify. - # To match depth, we need to call from a specific depth OR - # just verify that depths are greater than 1 and relatively correct. - # The user specifically asked to confirm stack length matches. - # This is tricky because the caller stack might differ. - - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Depth error") as exc_info: load_traceback(str(dump_file)) tb = exc_info.tb @@ -60,9 +69,6 @@ def level_1(): d2 = get_stack_depth(l2_f) d3 = get_stack_depth(l3_f) - print(f"Original depths: {original_depths}") - print(f"Reconstructed depths: {d1}, {d2}, {d3}") - # At minimum, verify they are linked correctly (depth increases by 1) assert d2 == d1 + 1 assert d3 == d2 + 1 @@ -73,28 +79,30 @@ def level_1(): assert d3 > 1 -def test_simple_exception_full_stack(tmp_path): +def test_simple_exception_full_stack(tmp_path: Path) -> None: + """Test serialization of a simple exception with a full stack.""" dump_file = tmp_path / "simple.dump" - def inner_raise(): + def inner_raise() -> Never: _x = 10 _y = "hello" - raise ValueError("Simple error") + msg = "Simple error" + raise ValueError(msg) - def middle_step(): + def middle_step() -> None: inner_raise() - def capture_it(): + def capture_it() -> None: try: middle_step() - except Exception as e: + except Exception as e: # noqa: BLE001 save_traceback(e, str(dump_file)) capture_it() assert dump_file.exists() # Now we call load_traceback from another stack - def second_stack_caller(): + def second_stack_caller() -> None: load_traceback(str(dump_file)) with pytest.raises(ValueError, match="Simple error") as exc_info: @@ -117,21 +125,24 @@ def second_stack_caller(): assert inner_frame.f_back.f_code.co_name == "middle_step" -def test_chained_exceptions_stack(tmp_path): +def test_chained_exceptions_stack(tmp_path: Path) -> None: + """Test serialization of chained exceptions.""" dump_file = tmp_path / "chained.dump" - def fail_inner(): - raise KeyError("Inner key error") + def fail_inner() -> Never: + msg = "Inner key error" + raise KeyError(msg) - def fail_outer(): + def fail_outer() -> None: try: fail_inner() except KeyError as e: - raise RuntimeError("Outer runtime error") from e + msg = "Outer runtime error" + raise RuntimeError(msg) from e try: fail_outer() - except Exception as e: + except Exception as e: # noqa: BLE001 save_traceback(e, str(dump_file)) with pytest.raises(RuntimeError, match="Outer runtime error") as exc_info: @@ -140,11 +151,12 @@ def fail_outer(): reconstructed_exc = exc_info.value assert isinstance(reconstructed_exc.__cause__, KeyError) - def get_frames(tb): + def get_frames(tb: types.TracebackType | None) -> list[types.FrameType]: f = [] - while tb: - f.append(tb.tb_frame) - tb = tb.tb_next + curr = tb + while curr is not None: + f.append(curr.tb_frame) + curr = curr.tb_next return f outer_frames = get_frames(reconstructed_exc.__traceback__) @@ -155,26 +167,29 @@ def get_frames(tb): assert fo_frame.f_back is not None -def test_unpicklable_locals_verification(tmp_path): +def test_unpicklable_locals_verification(tmp_path: Path) -> None: + """Test that unpicklable local variables are handled gracefully.""" dump_file = tmp_path / "unpicklable.dump" class Unpicklable: - def __reduce__(self): - raise TypeError("Cannot pickle me") + def __reduce__(self) -> Never: + msg = "Cannot pickle me" + raise TypeError(msg) - def __repr__(self): + def __repr__(self) -> str: return "" - def fail_with_unpicklable(): + def fail_with_unpicklable() -> Never: _obj = Unpicklable() - raise ValueError("Error with unpicklable") + msg = "Error with unpicklable" + raise ValueError(msg) try: fail_with_unpicklable() - except Exception as e: + except Exception as e: # noqa: BLE001 save_traceback(e, str(dump_file)) - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Error with unpicklable") as exc_info: load_traceback(str(dump_file)) tb = exc_info.tb @@ -186,23 +201,26 @@ def fail_with_unpicklable(): frame_names = [f.f_code.co_name for f in frames] assert "fail_with_unpicklable" in frame_names f = next(f for f in frames if f.f_code.co_name == "fail_with_unpicklable") + assert "_obj" in f.f_locals assert ">" in f.f_locals["_obj"] -def test_global_variables_in_stack(tmp_path): +def test_global_variables_in_stack(tmp_path: Path) -> None: + """Test that global variables are preserved in the stack.""" dump_file = tmp_path / "globals.dump" - def fail_with_globals(): - global GLOBAL_TEST_VAL + def fail_with_globals() -> None: + global GLOBAL_TEST_VAL # noqa: PLW0602 if GLOBAL_TEST_VAL == "I am global": - raise ValueError("Global test") + msg = "Global test" + raise ValueError(msg) try: fail_with_globals() - except Exception as e: + except Exception as e: # noqa: BLE001 save_traceback(e, str(dump_file)) - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Global test") as exc_info: load_traceback(str(dump_file)) tb = exc_info.tb @@ -220,27 +238,33 @@ def fail_with_globals(): GLOBAL_TEST_VAL = "I am global" -def test_unpicklable_exception_coverage(tmp_path): +def test_unpicklable_exception_coverage(tmp_path: Path) -> None: + """Test that unpicklable exceptions are handled by falling back to RuntimeError.""" dump_file = tmp_path / "unpicklable_exc.dump" - class UnpicklableException(Exception): - def __reduce__(self): - raise TypeError("Cannot pickle me") + class UnpicklableError(Exception): + def __reduce__(self) -> Never: + msg = "Cannot pickle me" + raise TypeError(msg) + + def raise_unpicklable() -> Never: + msg = "Unpicklable" + raise UnpicklableError(msg) try: - raise UnpicklableException("Unpicklable") - except Exception as e: + raise_unpicklable() + except Exception as e: # noqa: BLE001 save_traceback(e, str(dump_file)) - with pytest.raises( - RuntimeError, match="Unpicklable exception UnpicklableException: Unpicklable" - ): + with pytest.raises(RuntimeError, match="Unpicklable exception UnpicklableError: Unpicklable"): load_traceback(str(dump_file)) -def test_typing_never(): - from offline_debug import load_traceback +def test_typing_never() -> None: + """Test that load_traceback is correctly annotated with Never.""" import typing - annotations = typing.get_type_hints(load_traceback) - assert annotations["return"] is typing.Never + from offline_debug import load_traceback + + load_traceback_annotations = typing.get_type_hints(load_traceback) + assert load_traceback_annotations["return"] is typing.Never From ada5de79f179aa97b5577843ee495e3b473172bf Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Thu, 2 Apr 2026 23:47:26 +0300 Subject: [PATCH 07/16] cleaned up the c models and added a bunch of docs --- offline_debug/serializer.py | 58 ++++++++++++++++++++++++++++--------- tests/test_serializer.py | 18 ++++++++++++ 2 files changed, 62 insertions(+), 14 deletions(-) diff --git a/offline_debug/serializer.py b/offline_debug/serializer.py index 0726e8a..963d0a5 100644 --- a/offline_debug/serializer.py +++ b/offline_debug/serializer.py @@ -9,7 +9,7 @@ import types from dataclasses import dataclass from pathlib import Path -from typing import Any, Never, cast +from typing import Any, Never # Define C API for frame creation _py_frame_new = ctypes.pythonapi.PyFrame_New @@ -27,7 +27,7 @@ # Internal attributes that are either unpicklable or redundant in a new process. # We exclude these specifically because they are automatically recreated # when the new frame is initialized or when the module is imported. -_INTERNAL_ATTRIBUTES = ("__builtins__", "__doc__", "__loader__", "__package__", "__spec__") +_INTERNAL_ATTRIBUTES_TO_SKIP = ("__builtins__", "__doc__", "__loader__", "__package__", "__spec__") @dataclass @@ -52,15 +52,22 @@ class _ExceptionData: context: _ExceptionData | None = None -# Define ctypes structure to access f_back in PyFrameObject -class _PyObject(ctypes.Structure): - _fields_ = [("ob_refcnt", ctypes.c_ssize_t), ("ob_type", ctypes.c_void_p)] +class _PyFrameObject(ctypes.Structure): + """ + CPython's internal representation of a stack frame (PyFrameObject). + We use this structure to manually link reconstructed frames by modifying + the f_back pointer, which Python's high-level API does not allow. + """ -class _PyFrameObject(ctypes.Structure): _fields_ = [ - ("ob_base", _PyObject), - ("f_back", ctypes.c_void_p), # Pointer to previous frame + # Memory offset to skip the PyObject header (ob_refcnt and ob_type). + # This is necessary so that ctypes knows the exact position of f_back in memory. + ( + "offset_buffer", + ctypes.c_byte * (ctypes.sizeof(ctypes.c_ssize_t) + ctypes.sizeof(ctypes.c_void_p)), + ), + ("f_back", ctypes.c_void_p), # Pointer to the previous frame in the call stack. ] @@ -78,7 +85,7 @@ def _filter_dict(d: dict) -> dict: """Filter dictionary to include only picklable items.""" result = {} for k, v in d.items(): - if k in _INTERNAL_ATTRIBUTES: + if k in _INTERNAL_ATTRIBUTES_TO_SKIP: continue try: # We must verify if the value is picklable because many globals @@ -132,8 +139,22 @@ def save_traceback(exc: BaseException, file_path: str | Path) -> None: def _reconstruct_exc_data(data: _ExceptionData) -> BaseException: - """Recursively reconstruct an exception from its serialized data.""" - exc = cast("BaseException", pickle.loads(data.exc_pickle)) # noqa: S301 + """ + Recursively reconstruct an exception from its serialized data. + + Note on Python Locals: + Python uses two ways to store local variables: + 1. "Slow" locals: A dictionary used for module-level code and class definitions. + 2. "Fast" locals: A fixed-size array used for functions. This is faster than + dictionary lookups because variables are accessed by index. + + During reconstruction, we must explicitly synchronize these because PyFrame_New + does not automatically populate the "fast" locals array from a dictionary. + """ + exc = pickle.loads(data.exc_pickle) # noqa: S301 + if not isinstance(exc, BaseException): + msg = f"Expected BaseException, but got {type(exc).__name__}" + raise TypeError(msg) tstate = _py_thread_state_get() @@ -142,8 +163,13 @@ def _reconstruct_exc_data(data: _ExceptionData) -> BaseException: for f_data in data.tb_frames: code = marshal.loads(f_data.code) # noqa: S302 - # PyFrame_New returns a new reference to a PyFrameObject - frame = cast("types.FrameType", _py_frame_new(tstate, code, f_data.globals, {})) + # PyFrame_New returns a new reference to a PyFrameObject. + # We pass empty locals and update them afterward because PyFrame_New + # does not correctly initialize "fast" locals from a dictionary. + frame = _py_frame_new(tstate, code, f_data.globals, {}) + if not isinstance(frame, types.FrameType): + msg = f"Expected types.FrameType, but got {type(frame).__name__}" + raise TypeError(msg) if f_data.locals: frame.f_locals.update(f_data.locals) @@ -181,7 +207,11 @@ def _reconstruct_exc_data(data: _ExceptionData) -> BaseException: def load_traceback(file_path: str | Path) -> Never: """Load an exception and its traceback from a file and raise it.""" with Path(file_path).open("rb") as f: - data = cast("_ExceptionData", pickle.load(f)) # noqa: S301 + data = pickle.load(f) # noqa: S301 + + if not isinstance(data, _ExceptionData): + msg = f"Expected _ExceptionData, but got {type(data).__name__}" + raise TypeError(msg) exc = _reconstruct_exc_data(data) diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 1e0a18c..43e97dc 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -268,3 +268,21 @@ def test_typing_never() -> None: load_traceback_annotations = typing.get_type_hints(load_traceback) assert load_traceback_annotations["return"] is typing.Never + + +def test_load_invalid_object(tmp_path: Path) -> None: + """Test that load_traceback raises TypeError when loading an invalid object.""" + import pickle + + dump_file = tmp_path / "invalid.dump" + with dump_file.open("wb") as f: + pickle.dump("not an ExceptionData object", f) + + with pytest.raises(TypeError, match="Expected _ExceptionData, but got str"): + load_traceback(str(dump_file)) + + +def test_load_non_existent_file() -> None: + """Test that load_traceback raises FileNotFoundError when the file does not exist.""" + with pytest.raises(FileNotFoundError): + load_traceback("non_existent_file.dump") From 4409e664fecbea774d35afacc1ea2a49c1968ef4 Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Thu, 2 Apr 2026 23:54:10 +0300 Subject: [PATCH 08/16] added icons to git using the readme, and added a pre commit --- .pre-commit-config.yaml | 23 ++++++++++++++++++ README.md | 6 +++++ offline_debug/serializer.py | 4 ++-- pyproject.toml | 1 + tests/test_serializer.py | 47 +++++++++++++++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..db6b257 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,23 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.9 + hooks: + - id: ruff + args: [ --fix ] + - id: ruff-format + + - repo: local + hooks: + - id: ty-check + name: ty-check + entry: uv run ty check + language: system + types: [python] + pass_filenames: false + + - id: pytest-coverage + name: pytest-coverage + entry: uv run pytest --cov=offline_debug --cov-fail-under=100 + language: system + types: [python] + pass_filenames: false diff --git a/README.md b/README.md index 1fad4ef..1336a2c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # Traceback Serializer Project (`offline-debug`) +[![PyPI version](https://img.shields.io/pypi/v/offline-debug.svg)](https://pypi.org/project/offline-debug/) +[![Tests](https://github.com/INTODAN/offline-debug/actions/workflows/ci.yml/badge.svg)](https://github.com/INTODAN/offline-debug/actions) +[![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen.svg)](https://github.com/INTODAN/offline-debug) +[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) +[![Ty checked](https://img.shields.io/badge/ty-checked-blue.svg)](https://github.com/intodan/ty) + ## Overview A Python package for high-fidelity serialization and deserialization of exceptions and their complete tracebacks. Unlike other solutions, `offline-debug` reconstructs **actual** `types.FrameType` objects using the Python C API, ensuring that re-raised exceptions look and feel genuine to debuggers and introspection tools. diff --git a/offline_debug/serializer.py b/offline_debug/serializer.py index 963d0a5..d924031 100644 --- a/offline_debug/serializer.py +++ b/offline_debug/serializer.py @@ -9,7 +9,7 @@ import types from dataclasses import dataclass from pathlib import Path -from typing import Any, Never +from typing import Any, ClassVar, Never # Define C API for frame creation _py_frame_new = ctypes.pythonapi.PyFrame_New @@ -60,7 +60,7 @@ class _PyFrameObject(ctypes.Structure): the f_back pointer, which Python's high-level API does not allow. """ - _fields_ = [ + _fields_: ClassVar[list[tuple[str, Any]]] = [ # Memory offset to skip the PyObject header (ob_refcnt and ob_type). # This is necessary so that ctypes knows the exact position of f_back in memory. ( diff --git a/pyproject.toml b/pyproject.toml index 69c1d83..2bbd987 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ module-root = "" [dependency-groups] dev = [ + "pre-commit>=4.5.1", "pytest>=9.0.2", "pytest-cov>=7.1.0", "ruff>=0.15.8", diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 43e97dc..8a6aa2c 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -286,3 +286,50 @@ def test_load_non_existent_file() -> None: """Test that load_traceback raises FileNotFoundError when the file does not exist.""" with pytest.raises(FileNotFoundError): load_traceback("non_existent_file.dump") + + +def test_reconstruct_invalid_exception_type() -> None: + """Test that _reconstruct_exc_data raises TypeError when the pickled exception is invalid.""" + # Create dummy data with a string instead of a pickled exception + import pickle + + from offline_debug.serializer import _ExceptionData, _reconstruct_exc_data + + data = _ExceptionData( + exc_pickle=pickle.dumps("not an exception"), + tb_frames=[], + ) + + with pytest.raises(TypeError, match="Expected BaseException, but got str"): + _reconstruct_exc_data(data) + + +def test_reconstruct_invalid_frame_type(monkeypatch) -> None: + """Test that _reconstruct_exc_data raises TypeError when frame creation fails.""" + from offline_debug.serializer import _ExceptionData, _FrameData, _reconstruct_exc_data + + # Mock _py_frame_new to return something that is not a FrameType + monkeypatch.setattr("offline_debug.serializer._py_frame_new", lambda *_: "not a frame") + + import marshal + import pickle + + def dummy() -> None: + pass + + data = _ExceptionData( + exc_pickle=pickle.dumps(ValueError("test")), + tb_frames=[ + _FrameData( + code=marshal.dumps(dummy.__code__), + globals={}, + locals={}, + lasti=0, + lineno=0, + stack_depth=0, + ) + ], + ) + + with pytest.raises(TypeError, match="Expected types.FrameType, but got str"): + _reconstruct_exc_data(data) From f929dc5d88edae8909f0f609dc6cf035fef7a7fe Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Thu, 2 Apr 2026 23:58:33 +0300 Subject: [PATCH 09/16] fixed pre commit hook --- .pre-commit-config.yaml | 17 +++++++++++------ tests/test_serializer.py | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index db6b257..8585141 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,18 @@ repos: - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.9 + - repo: local hooks: - - id: ruff - args: [ --fix ] + - id: ruff-check + name: ruff-check + entry: uv run ruff check --fix + language: system + types: [python] + - id: ruff-format + name: ruff-format + entry: uv run ruff format + language: system + types: [python] - - repo: local - hooks: - id: ty-check name: ty-check entry: uv run ty check diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 8a6aa2c..45abc86 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -331,5 +331,5 @@ def dummy() -> None: ], ) - with pytest.raises(TypeError, match="Expected types.FrameType, but got str"): + with pytest.raises(TypeError, match=r"Expected types.FrameType, but got str"): _reconstruct_exc_data(data) From 8016bdccbb22de79d7717baccc6ba22057805a7d Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Fri, 3 Apr 2026 00:01:06 +0300 Subject: [PATCH 10/16] fixed the debug test --- tests/debug.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/debug.py b/tests/debug.py index 1f066c0..e0b6f09 100644 --- a/tests/debug.py +++ b/tests/debug.py @@ -25,13 +25,10 @@ def exception_raising_func() -> None: except ValueError as e: save_traceback(e, dump_file) + global global_variable + global_variable += 1 load_traceback(dump_file) if __name__ == "__main__": - try: - failure() - except ValueError: - import traceback - - traceback.print_exc() + failure() From 4d1e4c2952169189c7e51fd9cf7ea218fc7155cb Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Fri, 3 Apr 2026 00:09:29 +0300 Subject: [PATCH 11/16] improved tests --- tests/test_serializer.py | 63 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 45abc86..5ae8ebe 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -3,14 +3,28 @@ from __future__ import annotations import sys +import types from typing import TYPE_CHECKING, Never import pytest from offline_debug import load_traceback, save_traceback + +def func_to_module_string(func: Callable) -> str: + """Convert a function's body into a module string, removing indentation.""" + import inspect + import textwrap + + source = inspect.getsource(func) + # Get the body of the function (skip the def line) + lines = source.splitlines() + body = "\n".join(lines[1:]) + return textwrap.dedent(body) + + if TYPE_CHECKING: - import types + from collections.abc import Callable from pathlib import Path @@ -205,6 +219,10 @@ def fail_with_unpicklable() -> Never: assert ">" in f.f_locals["_obj"] +GLOBAL_TEST_VAL = "I am global" +GLOBAL_VAR = "initial" + + def test_global_variables_in_stack(tmp_path: Path) -> None: """Test that global variables are preserved in the stack.""" dump_file = tmp_path / "globals.dump" @@ -235,7 +253,48 @@ def fail_with_globals() -> None: assert f.f_globals["GLOBAL_TEST_VAL"] == "I am global" -GLOBAL_TEST_VAL = "I am global" +def test_globals_changing_between_frames(tmp_path: Path) -> None: + """Test that globals are captured per-frame for different modules.""" + dump_file = tmp_path / "globals_changing.dump" + + # Define the second module's content inside a function + def module2_content() -> None: + GLOBAL_VAR = "initial" # noqa: N806, F841 + + def level_2() -> Never: + global GLOBAL_VAR + GLOBAL_VAR = "changed" + msg = "Error at level 2" + raise ValueError(msg) + + mod2_code = func_to_module_string(module2_content) + mod2 = types.ModuleType("mod2") + exec(mod2_code, mod2.__dict__) # noqa: S102 + + def level_1() -> None: + try: + mod2.level_2() + except Exception as e: # noqa: BLE001 + save_traceback(e, str(dump_file)) + + level_1() + + with pytest.raises(ValueError, match="Error at level 2") as exc_info: + load_traceback(str(dump_file)) + + tb = exc_info.tb + frames = [] + while tb: + frames.append(tb.tb_frame) + tb = tb.tb_next + + l1_f = next(f for f in frames if f.f_code.co_name == "level_1") + l2_f = next(f for f in frames if f.f_code.co_name == "level_2") + + # mod2.level_2 changed its OWN GLOBAL_VAR. + # level_1's globals (this module) should NOT have GLOBAL_VAR (or it should be different) + assert l2_f.f_globals["GLOBAL_VAR"] == "changed" + assert "GLOBAL_VAR" not in l1_f.f_globals or l1_f.f_globals["GLOBAL_VAR"] == "initial" def test_unpicklable_exception_coverage(tmp_path: Path) -> None: From 1f19ee33bc834675d150d2f3510d93300adbe75c Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Fri, 3 Apr 2026 00:12:37 +0300 Subject: [PATCH 12/16] improved test file by removing duplications --- tests/test_serializer.py | 49 ++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 5ae8ebe..cb2ae32 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -10,6 +10,10 @@ from offline_debug import load_traceback, save_traceback +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + def func_to_module_string(func: Callable) -> str: """Convert a function's body into a module string, removing indentation.""" @@ -23,11 +27,6 @@ def func_to_module_string(func: Callable) -> str: return textwrap.dedent(body) -if TYPE_CHECKING: - from collections.abc import Callable - from pathlib import Path - - def get_stack_depth(frame: types.FrameType | None) -> int: """Calculate the depth of the given stack frame.""" depth = 0 @@ -38,6 +37,16 @@ def get_stack_depth(frame: types.FrameType | None) -> int: return depth +def get_frames(tb: types.TracebackType | None) -> list[types.FrameType]: + """Extract all frames from a traceback.""" + frames = [] + curr = tb + while curr: + frames.append(curr.tb_frame) + curr = curr.tb_next + return frames + + def test_stack_depth_preservation(tmp_path: Path) -> None: """Test that the stack depth is preserved during serialization.""" dump_file = tmp_path / "depth.dump" @@ -68,11 +77,7 @@ def level_1() -> None: with pytest.raises(ValueError, match="Depth error") as exc_info: load_traceback(str(dump_file)) - tb = exc_info.tb - frames = [] - while tb: - frames.append(tb.tb_frame) - tb = tb.tb_next + frames = get_frames(exc_info.tb) # We want to find level_1, level_2, level_3 in the reconstructed traceback l1_f = next(f for f in frames if f.f_code.co_name == "level_1") @@ -122,11 +127,7 @@ def second_stack_caller() -> None: with pytest.raises(ValueError, match="Simple error") as exc_info: second_stack_caller() - tb = exc_info.tb - frames = [] - while tb: - frames.append(tb.tb_frame) - tb = tb.tb_next + frames = get_frames(exc_info.tb) frame_names = [f.f_code.co_name for f in frames] assert "inner_raise" in frame_names @@ -206,11 +207,7 @@ def fail_with_unpicklable() -> Never: with pytest.raises(ValueError, match="Error with unpicklable") as exc_info: load_traceback(str(dump_file)) - tb = exc_info.tb - frames = [] - while tb: - frames.append(tb.tb_frame) - tb = tb.tb_next + frames = get_frames(exc_info.tb) frame_names = [f.f_code.co_name for f in frames] assert "fail_with_unpicklable" in frame_names @@ -241,11 +238,7 @@ def fail_with_globals() -> None: with pytest.raises(ValueError, match="Global test") as exc_info: load_traceback(str(dump_file)) - tb = exc_info.tb - frames = [] - while tb: - frames.append(tb.tb_frame) - tb = tb.tb_next + frames = get_frames(exc_info.tb) frame_names = [f.f_code.co_name for f in frames] assert "fail_with_globals" in frame_names @@ -282,11 +275,7 @@ def level_1() -> None: with pytest.raises(ValueError, match="Error at level 2") as exc_info: load_traceback(str(dump_file)) - tb = exc_info.tb - frames = [] - while tb: - frames.append(tb.tb_frame) - tb = tb.tb_next + frames = get_frames(exc_info.tb) l1_f = next(f for f in frames if f.f_code.co_name == "level_1") l2_f = next(f for f in frames if f.f_code.co_name == "level_2") From ca078eac03d142b216542b4b3aa94fa505b0dfc1 Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Fri, 3 Apr 2026 00:30:16 +0300 Subject: [PATCH 13/16] reduced c coupling, using public python api to link frames --- offline_debug/serializer.py | 37 +------------------------- tests/test_serializer.py | 53 +++++++++++++++++++------------------ 2 files changed, 28 insertions(+), 62 deletions(-) diff --git a/offline_debug/serializer.py b/offline_debug/serializer.py index d924031..b39a80c 100644 --- a/offline_debug/serializer.py +++ b/offline_debug/serializer.py @@ -9,7 +9,7 @@ import types from dataclasses import dataclass from pathlib import Path -from typing import Any, ClassVar, Never +from typing import Any, Never # Define C API for frame creation _py_frame_new = ctypes.pythonapi.PyFrame_New @@ -52,25 +52,6 @@ class _ExceptionData: context: _ExceptionData | None = None -class _PyFrameObject(ctypes.Structure): - """ - CPython's internal representation of a stack frame (PyFrameObject). - - We use this structure to manually link reconstructed frames by modifying - the f_back pointer, which Python's high-level API does not allow. - """ - - _fields_: ClassVar[list[tuple[str, Any]]] = [ - # Memory offset to skip the PyObject header (ob_refcnt and ob_type). - # This is necessary so that ctypes knows the exact position of f_back in memory. - ( - "offset_buffer", - ctypes.c_byte * (ctypes.sizeof(ctypes.c_ssize_t) + ctypes.sizeof(ctypes.c_void_p)), - ), - ("f_back", ctypes.c_void_p), # Pointer to the previous frame in the call stack. - ] - - def _get_stack_depth(frame: types.FrameType) -> int: """Calculate the depth of the current stack frame.""" depth = 0 @@ -159,7 +140,6 @@ def _reconstruct_exc_data(data: _ExceptionData) -> BaseException: tstate = _py_thread_state_get() reconstructed_frames: list[tuple[types.FrameType, _FrameData]] = [] - prev_frame: types.FrameType | None = None for f_data in data.tb_frames: code = marshal.loads(f_data.code) # noqa: S302 @@ -174,15 +154,7 @@ def _reconstruct_exc_data(data: _ExceptionData) -> BaseException: if f_data.locals: frame.f_locals.update(f_data.locals) - # Stitch the frames together to reconstruct the full stack trace. - # This allows tools like pdb or IDE debuggers to navigate up and down - # the reconstructed stack. - if prev_frame: - frame_ptr = _PyFrameObject.from_address(id(frame)) - frame_ptr.f_back = id(prev_frame) - reconstructed_frames.append((frame, f_data)) - prev_frame = frame tb_next: types.TracebackType | None = None for frame, f_data in reversed(reconstructed_frames): @@ -221,13 +193,6 @@ def load_traceback(file_path: str | Path) -> Never: current_frames.append(curr) curr = curr.f_back - if exc.__traceback__ and current_frames: - reconstructed_outer = exc.__traceback__.tb_frame - caller_frame = current_frames[0] - - frame_ptr = _PyFrameObject.from_address(id(reconstructed_outer)) - frame_ptr.f_back = id(caller_frame) - tb_chain: types.TracebackType | None = exc.__traceback__ for frame in current_frames: tb_chain = types.TracebackType( diff --git a/tests/test_serializer.py b/tests/test_serializer.py index cb2ae32..6b38b26 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -27,13 +27,13 @@ def func_to_module_string(func: Callable) -> str: return textwrap.dedent(body) -def get_stack_depth(frame: types.FrameType | None) -> int: - """Calculate the depth of the given stack frame.""" +def get_stack_depth(item: types.FrameType | types.TracebackType | None) -> int: + """Calculate the depth of the given stack frame or traceback chain.""" depth = 0 - curr = frame + curr = item while curr: depth += 1 - curr = curr.f_back + curr = curr.f_back if isinstance(curr, types.FrameType) else curr.tb_next return depth @@ -77,25 +77,19 @@ def level_1() -> None: with pytest.raises(ValueError, match="Depth error") as exc_info: load_traceback(str(dump_file)) - frames = get_frames(exc_info.tb) - - # We want to find level_1, level_2, level_3 in the reconstructed traceback - l1_f = next(f for f in frames if f.f_code.co_name == "level_1") - l2_f = next(f for f in frames if f.f_code.co_name == "level_2") - l3_f = next(f for f in frames if f.f_code.co_name == "level_3") - - d1 = get_stack_depth(l1_f) - d2 = get_stack_depth(l2_f) - d3 = get_stack_depth(l3_f) + # Reconstruct the depth from the traceback chain + tb = exc_info.tb + # Actually, load_traceback adds current frames. - # At minimum, verify they are linked correctly (depth increases by 1) - assert d2 == d1 + 1 - assert d3 == d2 + 1 + frames = get_frames(tb) + l1_idx = next(i for i, f in enumerate(frames) if f.f_code.co_name == "level_1") + l2_idx = next(i for i, f in enumerate(frames) if f.f_code.co_name == "level_2") + l3_idx = next(i for i, f in enumerate(frames) if f.f_code.co_name == "level_3") - # And verify they are NOT 1 (unless they were actually at depth 1) - assert d1 > 1 - assert d2 > 1 - assert d3 > 1 + # In a traceback, the list is TOP to BOTTOM (outer to inner) + # So level_1 should be before level_2 + assert l1_idx < l2_idx + assert l2_idx < l3_idx def test_simple_exception_full_stack(tmp_path: Path) -> None: @@ -135,9 +129,13 @@ def second_stack_caller() -> None: assert "second_stack_caller" in frame_names inner_idx = frame_names.index("inner_raise") - inner_frame = frames[inner_idx] - assert inner_frame.f_back is not None - assert inner_frame.f_back.f_code.co_name == "middle_step" + middle_idx = frame_names.index("middle_step") + second_idx = frame_names.index("second_stack_caller") + + # In a traceback, the list is TOP to BOTTOM (outer to inner) + # second_stack_caller -> capture_it -> middle_step -> inner_raise + assert second_idx < middle_idx + assert middle_idx < inner_idx def test_chained_exceptions_stack(tmp_path: Path) -> None: @@ -178,8 +176,11 @@ def get_frames(tb: types.TracebackType | None) -> list[types.FrameType]: outer_names = [f.f_code.co_name for f in outer_frames] assert "fail_outer" in outer_names - fo_frame = next(f for f in outer_frames if f.f_code.co_name == "fail_outer") - assert fo_frame.f_back is not None + inner_exc = reconstructed_exc.__cause__ + assert isinstance(inner_exc, KeyError) + inner_frames = get_frames(inner_exc.__traceback__) + inner_names = [f.f_code.co_name for f in inner_frames] + assert "fail_inner" in inner_names def test_unpicklable_locals_verification(tmp_path: Path) -> None: From 66d34c66ac4408414f7a718cea9bf848ce04817b Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Fri, 3 Apr 2026 01:59:35 +0300 Subject: [PATCH 14/16] improved test coverage --- .github/workflows/ci.yml | 11 +++-- offline_debug/serializer.py | 63 +++++++++++++++++++++++++++++ tests/test_f_back_discovery.py | 73 ++++++++++++++++++++++++++++++++++ tests/test_serializer.py | 55 +++++++++++++++++++++++++ 4 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 tests/test_f_back_discovery.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7495e7f..499a90c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,15 +26,20 @@ jobs: run: uv run ty check . test: - name: Run Tests - runs-on: ubuntu-latest + name: Test (${{ matrix.os }}, Python ${{ matrix.python-version }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.12", "3.13"] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: Install uv uses: astral-sh/setup-uv@v5 with: enable-cache: true - python-version: "3.12" + python-version: ${{ matrix.python-version }} - name: Run tests with Pytest and 100% Coverage run: uv run python -m pytest --cov=offline_debug --cov-report=term-missing --cov-fail-under=100 diff --git a/offline_debug/serializer.py b/offline_debug/serializer.py index b39a80c..8b7cb61 100644 --- a/offline_debug/serializer.py +++ b/offline_debug/serializer.py @@ -24,6 +24,60 @@ _py_thread_state_get = ctypes.pythonapi.PyThreadState_Get _py_thread_state_get.restype = ctypes.c_void_p + +def _get_f_back_offset() -> int | None: + """Dynamically discover the memory offset of f_back in PyFrameObject.""" + try: + tstate = _py_thread_state_get() + # Compile a dummy code object that we can use to create a frame. + code = compile("pass", "", "exec") + # Create a new, detached frame object using the C API. + frame = _py_frame_new(tstate, code, {}, {}) + if not isinstance(frame, types.FrameType): + return None + + # We need a target frame object to point to. + target = sys._getframe() # noqa: SLF001 + target_addr = id(target) + + # We scan the frame object's memory for the f_back pointer. + # We cap the scan at the object's actual size to avoid out-of-bounds reads. + limit = sys.getsizeof(frame) + ptr_size = ctypes.sizeof(ctypes.c_void_p) + + # We start scanning after the PyObject header (refcnt + type). + for offset in range(2 * ptr_size, limit - ptr_size + 1, ptr_size): + try: + # We use c_ssize_t to read the raw value at the offset. + current_val = ctypes.c_ssize_t.from_address(id(frame) + offset).value + # f_back is initially NULL (0) in a newly created frame. + if current_val == 0: + ctypes.c_ssize_t.from_address(id(frame) + offset).value = target_addr + # If reading f_back via Python now returns our target, we found it. + if frame.f_back is target: + return offset + # Restore to 0 if this wasn't the correct offset. + ctypes.c_ssize_t.from_address(id(frame) + offset).value = 0 + except (AttributeError, ValueError, TypeError, RuntimeError): + continue + except Exception: # noqa: BLE001 + return None + return None + + +_F_BACK_OFFSET = _get_f_back_offset() + + +def _link_frame(frame: types.FrameType, back: types.FrameType) -> None: + """Link a frame to its parent frame using the discovered offset.""" + if _F_BACK_OFFSET is None: + return + + # Use ctypes to write the address of the back frame into the discovered offset. + ptr = ctypes.c_void_p.from_address(id(frame) + _F_BACK_OFFSET) + ptr.value = id(back) + + # Internal attributes that are either unpicklable or redundant in a new process. # We exclude these specifically because they are automatically recreated # when the new frame is initialized or when the module is imported. @@ -140,6 +194,7 @@ def _reconstruct_exc_data(data: _ExceptionData) -> BaseException: tstate = _py_thread_state_get() reconstructed_frames: list[tuple[types.FrameType, _FrameData]] = [] + prev_frame: types.FrameType | None = None for f_data in data.tb_frames: code = marshal.loads(f_data.code) # noqa: S302 @@ -154,7 +209,11 @@ def _reconstruct_exc_data(data: _ExceptionData) -> BaseException: if f_data.locals: frame.f_locals.update(f_data.locals) + if prev_frame: + _link_frame(frame, prev_frame) + reconstructed_frames.append((frame, f_data)) + prev_frame = frame tb_next: types.TracebackType | None = None for frame, f_data in reversed(reconstructed_frames): @@ -193,6 +252,10 @@ def load_traceback(file_path: str | Path) -> Never: current_frames.append(curr) curr = curr.f_back + if exc.__traceback__ and current_frames: + reconstructed_outer = exc.__traceback__.tb_frame + _link_frame(reconstructed_outer, current_frames[0]) + tb_chain: types.TracebackType | None = exc.__traceback__ for frame in current_frames: tb_chain = types.TracebackType( diff --git a/tests/test_f_back_discovery.py b/tests/test_f_back_discovery.py new file mode 100644 index 0000000..e9232ae --- /dev/null +++ b/tests/test_f_back_discovery.py @@ -0,0 +1,73 @@ +"""Tests for the dynamic f_back offset discovery logic.""" + +import ctypes +from unittest.mock import patch + +from offline_debug.serializer import _get_f_back_offset + + +def test_get_f_back_offset_success() -> None: + """Test successful discovery of f_back offset.""" + # We don't need to mock much here, just verify it returns a plausible offset. + offset = _get_f_back_offset() + assert isinstance(offset, int) + assert offset > 0 + # On 64-bit CPython it's usually 16 or 24. + assert offset % ctypes.sizeof(ctypes.c_void_p) == 0 + + +def test_get_f_back_offset_not_a_frame() -> None: + """Test when PyFrame_New returns something that is not a FrameType.""" + with patch("offline_debug.serializer._py_frame_new", return_value="not a frame"): + offset = _get_f_back_offset() + assert offset is None + + +def test_get_f_back_offset_exception_in_try() -> None: + """Test when an exception occurs early in the discovery process.""" + with patch( + "offline_debug.serializer._py_thread_state_get", side_effect=RuntimeError("thread error") + ): + offset = _get_f_back_offset() + assert offset is None + + +def test_get_f_back_offset_discovery_failure() -> None: + """Test when the discovery loop completes without finding the offset.""" + + class MockValue: + def __init__(self, val: int) -> None: + self.value = val + + with patch("ctypes.c_ssize_t.from_address", return_value=MockValue(1)): + offset = _get_f_back_offset() + assert offset is None + + +def test_get_f_back_offset_wrong_offset_restoration() -> None: + """Test that it restores 0 if the offset was wrong.""" + tstate = ctypes.pythonapi.PyThreadState_Get() + code = compile("pass", "", "exec") + frame = ctypes.pythonapi.PyFrame_New(tstate, code, {}, {}) + + ptr_size = ctypes.sizeof(ctypes.c_void_p) + # Force the loop to check an offset that is 0 but NOT the real f_back. + + # Let's try this: + with ( + patch("offline_debug.serializer._py_frame_new", return_value=frame), + patch("offline_debug.serializer.range", return_value=[ptr_size * 10]), + ): # Offset 80 + # Ensure offset 80 is 0 + ctypes.c_ssize_t.from_address(id(frame) + ptr_size * 10).value = 0 + offset = _get_f_back_offset() + assert offset is None + # Verify it was restored to 0 + assert ctypes.c_ssize_t.from_address(id(frame) + ptr_size * 10).value == 0 + + +def test_get_f_back_offset_ctypes_error() -> None: + """Test when ctypes.c_ssize_t.from_address raises an error.""" + with patch("ctypes.c_ssize_t.from_address", side_effect=ValueError("ctypes error")): + offset = _get_f_back_offset() + assert offset is None diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 6b38b26..840228d 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -382,3 +382,58 @@ def dummy() -> None: with pytest.raises(TypeError, match=r"Expected types.FrameType, but got str"): _reconstruct_exc_data(data) + + +def test_get_f_back_offset_logic() -> None: + """Test the dynamic f_back offset discovery logic directly.""" + from offline_debug.serializer import _get_f_back_offset + + offset = _get_f_back_offset() + # It should either find an offset or be None (if platform is weird) + # But on standard CPython it should find something. + assert offset is None or (offset > 0 and offset % 8 == 0) + + +def test_link_frame_no_offset(monkeypatch) -> None: + """Test that _link_frame does nothing when offset is None.""" + import offline_debug.serializer as ser + + monkeypatch.setattr(ser, "_F_BACK_OFFSET", None) + + f = sys._getframe() + # Should not raise + ser._link_frame(f, f) + + +def test_reconstructed_frames_have_f_back(tmp_path: Path) -> None: + """ + Test that reconstructed frames have their f_back pointers correctly linked. + + This test is currently expected to FAIL because f_back linking was removed + to avoid segmentation faults, but it's required for full fidelity. + """ + dump_file = tmp_path / "f_back_fidelity.dump" + + def level_2() -> Never: + msg = "Fidelity error" + raise ValueError(msg) + + def level_1() -> None: + try: + level_2() + except Exception as e: # noqa: BLE001 + save_traceback(e, str(dump_file)) + + level_1() + + with pytest.raises(ValueError, match="Fidelity error") as exc_info: + load_traceback(str(dump_file)) + + frames = get_frames(exc_info.tb) + # Traceback frames are ordered TOP to BOTTOM (outer to inner) + # We want to check that level_2's frame points back to level_1 + l2_f = next(f for f in frames if f.f_code.co_name == "level_2") + l1_f = next(f for f in frames if f.f_code.co_name == "level_1") + + assert l2_f.f_back is not None, "level_2.f_back should not be None" + assert l2_f.f_back is l1_f, "level_2.f_back should point to level_1" From 3b457dd9fb11a294ed475cf619e77c77fe2c389b Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Fri, 3 Apr 2026 02:59:22 +0300 Subject: [PATCH 15/16] supporting python 3.12, including locals and ci and pre commit. ready for stable release --- .gitignore | 1 + .pre-commit-config.yaml | 9 ++++++++- offline_debug/serializer.py | 36 ++++++++++++++++++++++++++++++++---- pyproject.toml | 1 + tests/debug.py | 8 ++++++++ tests/test_serializer.py | 34 ++++++++++++++++++++++++++++++++-- 6 files changed, 82 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index da20745..dc646d1 100644 --- a/.gitignore +++ b/.gitignore @@ -138,6 +138,7 @@ celerybeat.pid .env .envrc .venv +.venv-* env/ venv/ ENV/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8585141..3edaedb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,14 @@ repos: - id: pytest-coverage name: pytest-coverage - entry: uv run pytest --cov=offline_debug --cov-fail-under=100 + entry: > + powershell -Command " + $env:UV_PROJECT_ENVIRONMENT = '.venv-3.12'; + uv run --python 3.12 pytest --cov=offline_debug; + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; + $env:UV_PROJECT_ENVIRONMENT = '.venv-3.13'; + uv run --python 3.13 pytest --cov=offline_debug --cov-append --cov-report=term --cov-fail-under=100 + " language: system types: [python] pass_filenames: false diff --git a/offline_debug/serializer.py b/offline_debug/serializer.py index 8b7cb61..7f0ad23 100644 --- a/offline_debug/serializer.py +++ b/offline_debug/serializer.py @@ -24,6 +24,12 @@ _py_thread_state_get = ctypes.pythonapi.PyThreadState_Get _py_thread_state_get.restype = ctypes.c_void_p +_py_incref = ctypes.pythonapi.Py_IncRef +_py_incref.argtypes = (ctypes.py_object,) + +_py_decref = ctypes.pythonapi.Py_DecRef +_py_decref.argtypes = (ctypes.py_object,) + def _get_f_back_offset() -> int | None: """Dynamically discover the memory offset of f_back in PyFrameObject.""" @@ -55,6 +61,9 @@ def _get_f_back_offset() -> int | None: ctypes.c_ssize_t.from_address(id(frame) + offset).value = target_addr # If reading f_back via Python now returns our target, we found it. if frame.f_back is target: + # Success, but we must restore 0 so we don't mess up refcounts + # when 'frame' is eventually garbage collected. + ctypes.c_ssize_t.from_address(id(frame) + offset).value = 0 return offset # Restore to 0 if this wasn't the correct offset. ctypes.c_ssize_t.from_address(id(frame) + offset).value = 0 @@ -73,6 +82,11 @@ def _link_frame(frame: types.FrameType, back: types.FrameType) -> None: if _F_BACK_OFFSET is None: return + # In Python, setting f_back means the child frame now owns a reference + # to the parent frame. We must increment the reference count of the + # parent to reflect this. + _py_incref(back) + # Use ctypes to write the address of the back frame into the discovered offset. ptr = ctypes.c_void_p.from_address(id(frame) + _F_BACK_OFFSET) ptr.value = id(back) @@ -198,15 +212,29 @@ def _reconstruct_exc_data(data: _ExceptionData) -> BaseException: for f_data in data.tb_frames: code = marshal.loads(f_data.code) # noqa: S302 + # In Python 3.11 and 3.12, accessing f_locals on a frame created via + # PyFrame_New for optimized code (functions) causes a segmentation fault + # because the internal 'fast' locals array is not initialized. + # As a workaround, we create a 'non-optimized' version of the code object + # by compiling a dummy string. This ensures the bytecode is safe + # (no LOAD_FAST) while preserving metadata like name and filename. + if sys.version_info < (3, 13): + # A simple module-level code object never has fast locals. + dummy_code = compile("", code.co_filename, "exec") + code = dummy_code.replace( + co_name=code.co_name, + co_firstlineno=code.co_firstlineno, + co_qualname=code.co_qualname, + ) + # PyFrame_New returns a new reference to a PyFrameObject. - # We pass empty locals and update them afterward because PyFrame_New - # does not correctly initialize "fast" locals from a dictionary. - frame = _py_frame_new(tstate, code, f_data.globals, {}) + frame = _py_frame_new(tstate, code, f_data.globals, f_data.locals) if not isinstance(frame, types.FrameType): msg = f"Expected types.FrameType, but got {type(frame).__name__}" raise TypeError(msg) - if f_data.locals: + # In 3.13+, PEP 667 allows safe write-through access to locals. + if sys.version_info >= (3, 13) and f_data.locals: frame.f_locals.update(f_data.locals) if prev_frame: diff --git a/pyproject.toml b/pyproject.toml index 2bbd987..f979098 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ dev = [ "pre-commit>=4.5.1", "pytest>=9.0.2", "pytest-cov>=7.1.0", + "rich>=14.3.3", "ruff>=0.15.8", "ty>=0.0.27", ] diff --git a/tests/debug.py b/tests/debug.py index e0b6f09..7ec9407 100644 --- a/tests/debug.py +++ b/tests/debug.py @@ -2,16 +2,24 @@ from __future__ import annotations +import sys import tempfile from pathlib import Path +from rich import traceback +from rich.console import Console + from offline_debug.serializer import load_traceback, save_traceback global_variable = 1 +console = Console(force_terminal=True) +traceback.install(show_locals=True, console=console) + def failure() -> None: """Raise a nested exception for testing.""" + _python_version = sys.version def exception_raising_func() -> None: local = "local" diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 840228d..06f79f2 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -197,6 +197,8 @@ def __repr__(self) -> str: def fail_with_unpicklable() -> Never: _obj = Unpicklable() + # Use locals() to force inclusion in the locals dictionary. + _ = locals()["_obj"] msg = "Error with unpicklable" raise ValueError(msg) @@ -213,8 +215,8 @@ def fail_with_unpicklable() -> Never: frame_names = [f.f_code.co_name for f in frames] assert "fail_with_unpicklable" in frame_names f = next(f for f in frames if f.f_code.co_name == "fail_with_unpicklable") - assert "_obj" in f.f_locals - assert ">" in f.f_locals["_obj"] + # Verify that our filtering logic caught the unpicklable item. + assert any(" None: assert l2_f.f_back is not None, "level_2.f_back should not be None" assert l2_f.f_back is l1_f, "level_2.f_back should point to level_1" + + +def test_locals_visibility_in_reconstructed_frames(tmp_path: Path) -> None: + """Regression test: verify local variables are visible in reconstructed frames.""" + dump_file = tmp_path / "locals_visibility.dump" + expected_val = 42 + + def func_with_locals() -> Never: + var_a = "hello" + var_b = expected_val + # Use them to ensure they aren't optimized away in some versions + _ = f"{var_a} {var_b}" + msg = "Locals error" + raise ValueError(msg) + + try: + func_with_locals() + except ValueError as e: + save_traceback(e, str(dump_file)) + + with pytest.raises(ValueError, match="Locals error") as exc_info: + load_traceback(str(dump_file)) + + frames = get_frames(exc_info.tb) + f = next(f for f in frames if f.f_code.co_name == "func_with_locals") + + assert f.f_locals.get("var_a") == "hello" + assert f.f_locals.get("var_b") == expected_val From 1d46826604a4e3e9e2435e1204a0aa3d04463530 Mon Sep 17 00:00:00 2001 From: Itai Elidan Date: Fri, 3 Apr 2026 03:05:50 +0300 Subject: [PATCH 16/16] added no cover for version specific code --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f979098..d9a67e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,3 +56,10 @@ dev = [ "ruff>=0.15.8", "ty>=0.0.27", ] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if sys.version_info < \\(3, 13\\).*", + "if sys.version_info >= \\(3, 13\\).*", +]