Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 59 additions & 40 deletions src/pyff/api.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import importlib
import threading
import traceback
from collections.abc import Generator, Iterable, Mapping
from datetime import datetime, timedelta
from json import dumps
Expand Down Expand Up @@ -265,54 +266,72 @@ def _d(x: Optional[str], do_split: bool = True) -> tuple[Optional[str], Optional
else:
q = new_path

# The pipeline result - often the whole selected metadata tree - only lives in _run_plumbings' frame. Pyramid keeps
# an error response's exception on the request, so an error must not reference that frame through its traceback:
# request, exception and frame would form a reference cycle, freed only by a full garbage collection (issue #283).
try:
accepter = MediaAccept(accept)
for p in request.registry.plumbings:
state = {
entry: True,
'headers': {'Content-Type': None},
'accept': accepter,
'url': request.current_route_url(),
'select': q,
'match': match.lower() if match else match,
'path': new_path,
'stats': {},
}

r = p.process(request.registry.md, state=state, raise_exceptions=True, scheduler=request.registry.scheduler)
log.debug(f'Plumbing process result: {r}')
if r is None:
r = []

response = Response()
_headers = state.get('headers', {})
response.headers.update(_headers)
ctype = _headers.get('Content-Type', None)
if not ctype:
r, t = _fmt(r, accepter)
ctype = t

response.text = b2u(r)
response.size = len(r)
response.content_type = ctype
cache_ttl = int(state.get('cache', 0))
response.expires = datetime.now() + timedelta(seconds=cache_ttl)
return response
response = _run_plumbings(request, entry, accept, q, match, new_path)
except exc.HTTPException as ex:
# An HTTP error from processing, eg 406 from content negotiation, is the response as is.
raise ex.with_traceback(None)
except ResourceException as ex:
import traceback

log.debug(traceback.format_exc())
log.warning(f'Exception from processing pipeline: {ex}')
raise exc.exception_response(409)
status = 409
except BaseException as ex:
import traceback

log.debug(traceback.format_exc())
log.error(f'Exception from processing pipeline: {ex}')
raise exc.exception_response(500)
status = 500
else:
if response is None and request.method == 'GET':
raise exc.exception_response(404)
return response

if request.method == 'GET':
raise exc.exception_response(404)
# Raised outside the except clauses, so that it does not chain the original exception and its traceback.
raise exc.exception_response(status)


def _run_plumbings(
request: Request, entry: str, accept: str, q: Optional[str], match: Optional[str], new_path: Optional[str]
) -> Optional[Response]:
"""
Run the request through the plumbings and format the result for the accepted content type.

:return: the response, or None if there are no plumbings
"""
accepter = MediaAccept(accept)
for p in request.registry.plumbings:
state = {
entry: True,
'headers': {'Content-Type': None},
'accept': accepter,
'url': request.current_route_url(),
'select': q,
'match': match.lower() if match else match,
'path': new_path,
'stats': {},
}

r = p.process(request.registry.md, state=state, raise_exceptions=True, scheduler=request.registry.scheduler)
log.debug(f'Plumbing process result: {r}')
if r is None:
r = []

response = Response()
_headers = state.get('headers', {})
response.headers.update(_headers)
ctype = _headers.get('Content-Type', None)
if not ctype:
r, t = _fmt(r, accepter)
ctype = t

response.text = b2u(r)
response.size = len(r)
response.content_type = ctype
cache_ttl = int(state.get('cache', 0))
response.expires = datetime.now() + timedelta(seconds=cache_ttl)
return response
return None


def webfinger_handler(request: Request) -> Response:
Expand Down
91 changes: 91 additions & 0 deletions src/pyff/test/test_md_api.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import gc
import json
import os
import shutil
import tempfile
import unittest
import weakref
from datetime import datetime, timezone
from types import SimpleNamespace
from unittest import mock
from urllib.parse import quote as urlescape

import pytest
Expand Down Expand Up @@ -356,3 +359,91 @@ def test_entity_id_without_trailing_slash_is_not_found(self):
with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url:
assert requests.post(f'{url}/api/call/update').status_code == 200
assert self._entity_ids(url, '/entities/https:/sp.example.com/saml2/metadata') == set()


class _PipelineResult(list):
"""A pipeline result that can be weakly referenced, to observe when it is freed"""


class PyFFAPITestErrorResponses(PipeLineTest):
"""
Errors while processing a request must have the right status, and must not keep the pipeline result - often
the whole selected metadata tree - alive after the request: cf. issue #283.
"""

mdx = None
app = None

@classmethod
def setUpClass(cls):
SignerTestCase.setUpClass()
config.local_copy_dir = tempfile.mkdtemp()
cls.mdx = tempfile.NamedTemporaryFile('w').name
with open(cls.mdx, "w") as fd:
fd.write(
f"""
- when update:
- load:
- {os.path.join(cls.datadir, 'metadata', 'test01.xml')}
- when request:
- select
- pipe:
- when accept application/xml:
- emit application/xml
- break
"""
)
cls._app = mkapp(cls.mdx)
cls.app = lambda *args, **kwargs: cls._app

@classmethod
def tearDownClass(cls):
SignerTestCase.tearDownClass()
if os.path.exists(cls.mdx):
os.unlink(cls.mdx)
if os.path.exists(config.local_copy_dir):
shutil.rmtree(config.local_copy_dir)

def _request_with_pipeline_result(self, url, fail):
"""
GET /entities as text/plain with the pipeline replaced by one producing a weakly referenceable result,
then failing if ``fail``. The garbage collector is off, so only reference counting can free the result.

:return: the HTTP status and whether the result was freed once the request was done
"""
refs = []

def process(*args, **kwargs):
result = _PipelineResult(['<md:EntitiesDescriptor/>'])
refs.append(weakref.ref(result))
if fail:
raise RuntimeError("pipeline failed")
return result

gc.collect()
gc.disable()
try:
with mock.patch.object(self._app.registry.plumbings[0], 'process', process):
status = requests.get(f'{url}/entities', headers={'Accept': 'text/plain'}).status_code
return status, refs[0]() is None
finally:
gc.enable()

def test_unacceptable_type_is_406(self):
"""No pipeline branch and no format matches text/plain: that is 406 Not Acceptable, not a server error"""
with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url:
assert requests.post(f'{url}/api/call/update').status_code == 200
r = requests.get(f'{url}/entities', headers={'Accept': 'text/plain'})
assert r.status_code == 406

def test_406_frees_the_pipeline_result(self):
with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url:
status, freed = self._request_with_pipeline_result(url, fail=False)
assert status == 406
assert freed, "the pipeline result outlived a 406 response"

def test_500_frees_the_pipeline_result(self):
with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url:
status, freed = self._request_with_pipeline_result(url, fail=True)
assert status == 500
assert freed, "the pipeline result outlived a 500 response"
Loading