From 39a1382011f34d692add63fd4e84453678bb6498 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Tue, 17 Nov 2026 12:27:55 +0100 Subject: [PATCH] Fix memory retained by error responses Pyramid keeps an error response's exception on the request. Its traceback reached the frame holding the pipeline result, so each failed request kept the selected metadata tree alive until a full garbage collection. Run the plumbings in their own function, re-raise HTTP errors without their traceback, and raise 409/500 outside the except clauses. HTTP errors such as 406 from content negotiation are no longer turned into 500. Closes #283. --- src/pyff/api.py | 99 +++++++++++++++++++++--------------- src/pyff/test/test_md_api.py | 91 +++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 40 deletions(-) diff --git a/src/pyff/api.py b/src/pyff/api.py index 87a2e266..37fb6fda 100644 --- a/src/pyff/api.py +++ b/src/pyff/api.py @@ -1,5 +1,6 @@ import importlib import threading +import traceback from collections.abc import Generator, Iterable, Mapping from datetime import datetime, timedelta from json import dumps @@ -265,54 +266,72 @@ def _d(x: Optional[str], do_split: bool = True) -> tuple[Optional[str], Optional else: q = new_path + # The pipeline result - often the whole selected metadata tree - only lives in _run_plumbings' frame. Pyramid keeps + # an error response's exception on the request, so an error must not reference that frame through its traceback: + # request, exception and frame would form a reference cycle, freed only by a full garbage collection (issue #283). try: - accepter = MediaAccept(accept) - for p in request.registry.plumbings: - state = { - entry: True, - 'headers': {'Content-Type': None}, - 'accept': accepter, - 'url': request.current_route_url(), - 'select': q, - 'match': match.lower() if match else match, - 'path': new_path, - 'stats': {}, - } - - r = p.process(request.registry.md, state=state, raise_exceptions=True, scheduler=request.registry.scheduler) - log.debug(f'Plumbing process result: {r}') - if r is None: - r = [] - - response = Response() - _headers = state.get('headers', {}) - response.headers.update(_headers) - ctype = _headers.get('Content-Type', None) - if not ctype: - r, t = _fmt(r, accepter) - ctype = t - - response.text = b2u(r) - response.size = len(r) - response.content_type = ctype - cache_ttl = int(state.get('cache', 0)) - response.expires = datetime.now() + timedelta(seconds=cache_ttl) - return response + response = _run_plumbings(request, entry, accept, q, match, new_path) + except exc.HTTPException as ex: + # An HTTP error from processing, eg 406 from content negotiation, is the response as is. + raise ex.with_traceback(None) except ResourceException as ex: - import traceback - log.debug(traceback.format_exc()) log.warning(f'Exception from processing pipeline: {ex}') - raise exc.exception_response(409) + status = 409 except BaseException as ex: - import traceback - log.debug(traceback.format_exc()) log.error(f'Exception from processing pipeline: {ex}') - raise exc.exception_response(500) + status = 500 + else: + if response is None and request.method == 'GET': + raise exc.exception_response(404) + return response - if request.method == 'GET': - raise exc.exception_response(404) + # Raised outside the except clauses, so that it does not chain the original exception and its traceback. + raise exc.exception_response(status) + + +def _run_plumbings( + request: Request, entry: str, accept: str, q: Optional[str], match: Optional[str], new_path: Optional[str] +) -> Optional[Response]: + """ + Run the request through the plumbings and format the result for the accepted content type. + + :return: the response, or None if there are no plumbings + """ + accepter = MediaAccept(accept) + for p in request.registry.plumbings: + state = { + entry: True, + 'headers': {'Content-Type': None}, + 'accept': accepter, + 'url': request.current_route_url(), + 'select': q, + 'match': match.lower() if match else match, + 'path': new_path, + 'stats': {}, + } + + r = p.process(request.registry.md, state=state, raise_exceptions=True, scheduler=request.registry.scheduler) + log.debug(f'Plumbing process result: {r}') + if r is None: + r = [] + + response = Response() + _headers = state.get('headers', {}) + response.headers.update(_headers) + ctype = _headers.get('Content-Type', None) + if not ctype: + r, t = _fmt(r, accepter) + ctype = t + + response.text = b2u(r) + response.size = len(r) + response.content_type = ctype + cache_ttl = int(state.get('cache', 0)) + response.expires = datetime.now() + timedelta(seconds=cache_ttl) + return response + return None def webfinger_handler(request: Request) -> Response: diff --git a/src/pyff/test/test_md_api.py b/src/pyff/test/test_md_api.py index e3c59fc1..fb2dea47 100644 --- a/src/pyff/test/test_md_api.py +++ b/src/pyff/test/test_md_api.py @@ -1,10 +1,13 @@ +import gc import json import os import shutil import tempfile import unittest +import weakref from datetime import datetime, timezone from types import SimpleNamespace +from unittest import mock from urllib.parse import quote as urlescape import pytest @@ -356,3 +359,91 @@ def test_entity_id_without_trailing_slash_is_not_found(self): with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url: assert requests.post(f'{url}/api/call/update').status_code == 200 assert self._entity_ids(url, '/entities/https:/sp.example.com/saml2/metadata') == set() + + +class _PipelineResult(list): + """A pipeline result that can be weakly referenced, to observe when it is freed""" + + +class PyFFAPITestErrorResponses(PipeLineTest): + """ + Errors while processing a request must have the right status, and must not keep the pipeline result - often + the whole selected metadata tree - alive after the request: cf. issue #283. + """ + + mdx = None + app = None + + @classmethod + def setUpClass(cls): + SignerTestCase.setUpClass() + config.local_copy_dir = tempfile.mkdtemp() + cls.mdx = tempfile.NamedTemporaryFile('w').name + with open(cls.mdx, "w") as fd: + fd.write( + f""" +- when update: + - load: + - {os.path.join(cls.datadir, 'metadata', 'test01.xml')} +- when request: + - select + - pipe: + - when accept application/xml: + - emit application/xml + - break +""" + ) + cls._app = mkapp(cls.mdx) + cls.app = lambda *args, **kwargs: cls._app + + @classmethod + def tearDownClass(cls): + SignerTestCase.tearDownClass() + if os.path.exists(cls.mdx): + os.unlink(cls.mdx) + if os.path.exists(config.local_copy_dir): + shutil.rmtree(config.local_copy_dir) + + def _request_with_pipeline_result(self, url, fail): + """ + GET /entities as text/plain with the pipeline replaced by one producing a weakly referenceable result, + then failing if ``fail``. The garbage collector is off, so only reference counting can free the result. + + :return: the HTTP status and whether the result was freed once the request was done + """ + refs = [] + + def process(*args, **kwargs): + result = _PipelineResult(['']) + refs.append(weakref.ref(result)) + if fail: + raise RuntimeError("pipeline failed") + return result + + gc.collect() + gc.disable() + try: + with mock.patch.object(self._app.registry.plumbings[0], 'process', process): + status = requests.get(f'{url}/entities', headers={'Accept': 'text/plain'}).status_code + return status, refs[0]() is None + finally: + gc.enable() + + def test_unacceptable_type_is_406(self): + """No pipeline branch and no format matches text/plain: that is 406 Not Acceptable, not a server error""" + with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url: + assert requests.post(f'{url}/api/call/update').status_code == 200 + r = requests.get(f'{url}/entities', headers={'Accept': 'text/plain'}) + assert r.status_code == 406 + + def test_406_frees_the_pipeline_result(self): + with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url: + status, freed = self._request_with_pipeline_result(url, fail=False) + assert status == 406 + assert freed, "the pipeline result outlived a 406 response" + + def test_500_frees_the_pipeline_result(self): + with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url: + status, freed = self._request_with_pipeline_result(url, fail=True) + assert status == 500 + assert freed, "the pipeline result outlived a 500 response"