From bf395d9bbc318a47bc07e91ca4e6e6c72d65a079 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Wed, 22 Jul 2026 15:56:23 -0400 Subject: [PATCH 01/82] Add Privacy Guard v0 --- projects/README.md | 2 + projects/privacy-guard/BENCHMARKS.md | 65 ++ projects/privacy-guard/CONTRIBUTING.md | 50 ++ projects/privacy-guard/README.md | 242 +++++++ .../examples/openshell-e2e/README.md | 70 ++ .../examples/openshell-e2e/capture_server.py | 60 ++ .../openshell-e2e/middleware_server.py | 50 ++ .../examples/openshell-e2e/policy.yaml | 38 + .../examples/openshell-e2e/run.sh | 197 +++++ .../examples/openshell-manual/README.md | 149 ++++ .../examples/openshell-manual/gateway.toml | 12 + .../examples/openshell-manual/policy.yaml | 47 ++ .../middleware-dev-manifest.json | 7 + .../proto/supervisor_middleware.proto | 242 +++++++ projects/privacy-guard/pyproject.toml | 70 ++ .../scripts/benchmark_privacy_guard.py | 319 ++++++++ projects/privacy-guard/scripts/validate.sh | 20 + .../src/privacy_guard/__init__.py | 1 + .../src/privacy_guard/bindings/__init__.py | 1 + .../bindings/supervisor_middleware_pb2.py | 78 ++ .../bindings/supervisor_middleware_pb2.pyi | 209 ++++++ .../supervisor_middleware_pb2_grpc.py | 210 ++++++ .../src/privacy_guard/body/__init__.py | 45 ++ .../src/privacy_guard/body/base.py | 117 +++ .../src/privacy_guard/body/json.py | 300 ++++++++ .../privacy-guard/src/privacy_guard/config.py | 135 ++++ .../src/privacy_guard/constants.py | 46 ++ .../privacy-guard/src/privacy_guard/errors.py | 213 ++++++ .../src/privacy_guard/payloads/__init__.py | 8 + .../src/privacy_guard/payloads/request.py | 32 + .../src/privacy_guard/payloads/result.py | 26 + .../src/privacy_guard/processor.py | 380 ++++++++++ .../src/privacy_guard/scanners/__init__.py | 27 + .../src/privacy_guard/scanners/base.py | 156 ++++ .../src/privacy_guard/scanners/passthrough.py | 23 + .../src/privacy_guard/service/__init__.py | 9 + .../src/privacy_guard/service/server.py | 80 ++ .../src/privacy_guard/service/servicer.py | 323 +++++++++ .../src/privacy_guard/validation.py | 58 ++ projects/privacy-guard/tests/__init__.py | 1 + .../privacy-guard/tests/body/test_base.py | 122 ++++ .../privacy-guard/tests/body/test_json.py | 575 +++++++++++++++ .../tests/examples/test_openshell_e2e.py | 80 ++ .../privacy-guard/tests/scanner_helpers.py | 33 + .../tests/scanners/test_passthrough.py | 155 ++++ .../privacy-guard/tests/service/__init__.py | 1 + .../tests/service/test_server.py | 323 +++++++++ .../tests/service/test_servicer.py | 474 ++++++++++++ projects/privacy-guard/tests/test_config.py | 369 ++++++++++ projects/privacy-guard/tests/test_errors.py | 39 + .../privacy-guard/tests/test_hardening.py | 594 +++++++++++++++ projects/privacy-guard/tests/test_payloads.py | 118 +++ .../privacy-guard/tests/test_processor.py | 684 ++++++++++++++++++ .../tests/test_single_block_composition.py | 51 ++ .../privacy-guard/tests/test_typing_policy.py | 583 +++++++++++++++ projects/privacy-guard/uv.lock | 376 ++++++++++ 56 files changed, 8695 insertions(+) create mode 100644 projects/privacy-guard/BENCHMARKS.md create mode 100644 projects/privacy-guard/CONTRIBUTING.md create mode 100644 projects/privacy-guard/README.md create mode 100644 projects/privacy-guard/examples/openshell-e2e/README.md create mode 100644 projects/privacy-guard/examples/openshell-e2e/capture_server.py create mode 100644 projects/privacy-guard/examples/openshell-e2e/middleware_server.py create mode 100644 projects/privacy-guard/examples/openshell-e2e/policy.yaml create mode 100755 projects/privacy-guard/examples/openshell-e2e/run.sh create mode 100644 projects/privacy-guard/examples/openshell-manual/README.md create mode 100644 projects/privacy-guard/examples/openshell-manual/gateway.toml create mode 100644 projects/privacy-guard/examples/openshell-manual/policy.yaml create mode 100644 projects/privacy-guard/middleware-dev-manifest.json create mode 100644 projects/privacy-guard/proto/supervisor_middleware.proto create mode 100644 projects/privacy-guard/pyproject.toml create mode 100755 projects/privacy-guard/scripts/benchmark_privacy_guard.py create mode 100755 projects/privacy-guard/scripts/validate.sh create mode 100644 projects/privacy-guard/src/privacy_guard/__init__.py create mode 100644 projects/privacy-guard/src/privacy_guard/bindings/__init__.py create mode 100644 projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2.py create mode 100644 projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2.pyi create mode 100644 projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2_grpc.py create mode 100644 projects/privacy-guard/src/privacy_guard/body/__init__.py create mode 100644 projects/privacy-guard/src/privacy_guard/body/base.py create mode 100644 projects/privacy-guard/src/privacy_guard/body/json.py create mode 100644 projects/privacy-guard/src/privacy_guard/config.py create mode 100644 projects/privacy-guard/src/privacy_guard/constants.py create mode 100644 projects/privacy-guard/src/privacy_guard/errors.py create mode 100644 projects/privacy-guard/src/privacy_guard/payloads/__init__.py create mode 100644 projects/privacy-guard/src/privacy_guard/payloads/request.py create mode 100644 projects/privacy-guard/src/privacy_guard/payloads/result.py create mode 100644 projects/privacy-guard/src/privacy_guard/processor.py create mode 100644 projects/privacy-guard/src/privacy_guard/scanners/__init__.py create mode 100644 projects/privacy-guard/src/privacy_guard/scanners/base.py create mode 100644 projects/privacy-guard/src/privacy_guard/scanners/passthrough.py create mode 100644 projects/privacy-guard/src/privacy_guard/service/__init__.py create mode 100644 projects/privacy-guard/src/privacy_guard/service/server.py create mode 100644 projects/privacy-guard/src/privacy_guard/service/servicer.py create mode 100644 projects/privacy-guard/src/privacy_guard/validation.py create mode 100644 projects/privacy-guard/tests/__init__.py create mode 100644 projects/privacy-guard/tests/body/test_base.py create mode 100644 projects/privacy-guard/tests/body/test_json.py create mode 100644 projects/privacy-guard/tests/examples/test_openshell_e2e.py create mode 100644 projects/privacy-guard/tests/scanner_helpers.py create mode 100644 projects/privacy-guard/tests/scanners/test_passthrough.py create mode 100644 projects/privacy-guard/tests/service/__init__.py create mode 100644 projects/privacy-guard/tests/service/test_server.py create mode 100644 projects/privacy-guard/tests/service/test_servicer.py create mode 100644 projects/privacy-guard/tests/test_config.py create mode 100644 projects/privacy-guard/tests/test_errors.py create mode 100644 projects/privacy-guard/tests/test_hardening.py create mode 100644 projects/privacy-guard/tests/test_payloads.py create mode 100644 projects/privacy-guard/tests/test_processor.py create mode 100644 projects/privacy-guard/tests/test_single_block_composition.py create mode 100644 projects/privacy-guard/tests/test_typing_policy.py create mode 100644 projects/privacy-guard/uv.lock diff --git a/projects/README.md b/projects/README.md index 674946c..a732eff 100644 --- a/projects/README.md +++ b/projects/README.md @@ -8,6 +8,8 @@ Current projects: - `openshell-middleware-init`: Typer CLI that generates version-matched Python and Rust OpenShell supervisor middleware projects. +- `privacy-guard`: OpenShell supervisor middleware for inspecting and enforcing + policy on provider-bound requests before credentials are attached. - `python-project-template`: Minimal, production-ready Python project scaffold managed with uv. - `reachy-mini-openshell`: Reachy Mini conversation demo for OpenShell. diff --git a/projects/privacy-guard/BENCHMARKS.md b/projects/privacy-guard/BENCHMARKS.md new file mode 100644 index 0000000..a4a1b3f --- /dev/null +++ b/projects/privacy-guard/BENCHMARKS.md @@ -0,0 +1,65 @@ +# Full-path benchmark evidence + +The benchmark invokes `RequestProcessor.process` and therefore covers JSON +normalization, scanner calls, scanner-output validation, contextual finding +validation, policy application, and body reconstruction. Input bodies are exact +1 KiB, 1 MiB, and 4 MiB JSON payloads. The maximum load is the request limit of +4,096 findings, distributed across 16 blocks; multiple-scanner cases use four +scanners. Replacement cases redact matched characters to an empty string so the +4 MiB input remains within the output limit. Each scanner constructs normal +strict `Finding` models during every measured scan; no finding output is reused +between requests. The full suite also contains a separate maximum-width numeric +array: exactly 4,194,303 bytes and 2,097,151 numeric elements, with zero text +blocks and findings. It exposes the complete duplicate-aware stdlib parse and +single cached strict Pydantic adapter path without large-string scanning costs. + +JSON normalization stores the adapter's `JsonValue` output directly in private +handler state. It does not build an immutable mirror tree. Incremental text-block +traversal retains state proportional to nesting depth, no-op reconstruction +returns the original bytes without walking the tree, and replacement +reconstruction performs one `deepcopy`. + +Run the recorded suite with: + +```bash +uv run --frozen python scripts/benchmark_privacy_guard.py --suite full +``` + +Wall-clock samples are collected without tracing. Peak allocations are measured +in separate runs with `tracemalloc`, preventing allocation tracing from changing +the wall-time result. Each reported value is the median of seven runs after one +verified warm-up. `--profile profile.out` adds one verified `cProfile` pass per +scenario without contaminating either measurement. + +Reference environment: CPython 3.11.9 on Apple arm64, macOS 26.5.2. Recorded +2026-07-22 with the command above. + +| Scenario | Input bytes | Findings | Scanners | Reconstruction | Median wall (ms) | Median peak traced allocation (bytes) | +| --- | ---: | ---: | ---: | --- | ---: | ---: | +| 1KiB-zero-one-noop | 1,024 | 0 | 1 | no-op | 0.075 | 7,043 | +| 1KiB-typical-multiple-replacement | 1,024 | 8 | 4 | replacement | 0.192 | 19,563 | +| 1MiB-typical-one-noop | 1,048,576 | 8 | 1 | no-op | 55.380 | 2,101,945 | +| 1MiB-max-one-replacement | 1,048,576 | 4,096 | 1 | replacement | 116.764 | 8,252,749 | +| 4MiB-zero-multiple-noop | 4,194,304 | 0 | 4 | no-op | 434.537 | 8,393,513 | +| 4MiB-max-multiple-replacement | 4,194,304 | 4,096 | 4 | replacement | 542.377 | 20,834,860 | +| 1KiB-typical-one-noop | 1,024 | 8 | 1 | no-op | 0.093 | 14,696 | +| 1MiB-zero-one-noop | 1,048,576 | 0 | 1 | no-op | 56.848 | 2,101,890 | +| 1MiB-typical-multiple-replacement | 1,048,576 | 8 | 4 | replacement | 129.166 | 4,209,498 | +| 1MiB-max-multiple-noop | 1,048,576 | 4,096 | 4 | no-op | 126.820 | 5,117,784 | +| 4MiB-typical-one-replacement | 4,194,304 | 8 | 1 | replacement | 303.447 | 16,792,456 | +| 4MiB-typical-multiple-noop | 4,194,304 | 8 | 4 | no-op | 437.978 | 8,393,401 | +| 4MiB-max-one-noop | 4,194,304 | 4,096 | 1 | no-op | 229.557 | 8,402,617 | +| 4194303B-2097151-elements-wide-numeric-zero-one-noop | 4,194,303 | 0 | 1 | no-op | 726.707 | 38,102,178 | + +The focused 100,000-element numeric-array regression uses a 200,001-byte body. +On the same interpreter, complete normalization peaked at 1,802,636 traced +bytes (below its committed 8 MiB ceiling), while walking the already validated +tree peaked at 1,067 traced bytes (below 64 KiB), demonstrating that walker +state does not grow with container width. + +`tracemalloc` reports traced Python allocations, not process RSS, allocator +arena retention, native allocations, or transient memory outside its tracing +domain. The maximum-width peak includes the unavoidable overlap while the +cached Pydantic adapter constructs its validated output from the stdlib parse +tree; the raw parse tree becomes unreachable when the parse helper returns, +before text-block materialization begins. diff --git a/projects/privacy-guard/CONTRIBUTING.md b/projects/privacy-guard/CONTRIBUTING.md new file mode 100644 index 0000000..2873e64 --- /dev/null +++ b/projects/privacy-guard/CONTRIBUTING.md @@ -0,0 +1,50 @@ +# Contributing to Privacy Guard + +Run `scripts/validate.sh` from this directory before review. Run +`scripts/validate.sh --python 3.11` when changing interpreter-sensitive code. +The gate owns full tests, Ruff formatting and lint, curated `ty` diagnostics, +the import smoke test, and the package build. + +Trust-boundary validation has one owner: + +| Boundary | Owner | +| --- | --- | +| Scalar fields and record shape | strict frozen Pydantic models | +| Action selection and action-specific fields | `PolicyConfig.on_finding` discriminated union keyed by `action` | +| Scanner metadata and scanner output shape | `Scanner` public wrapper | +| Format metadata and normalized/reconstructed output shape | `FormatHandler` public wrappers | +| Scanner/block identity, offsets, aggregate limits, and original bytes | `RequestProcessor` | +| Recursive JSON values | cached Pydantic adapter in `JsonHandler` | +| Protobuf conversion | service adapter | + +Custom scanners and format handlers subclass the nominal ABCs, call the base +constructor, and decorate protected hook implementations with `@override`. +`RequestProcessor` accepts a scanner sequence. Scanners return block-relative +`Finding` models; the processor attaches the opaque text-block path by composing +them into `RequestBodyFinding` models. + +Scanner configuration must not depend on policy. Every `ScannerConfig` declares +the scanner's complete `entity_types` catalog, and concrete subtypes own any +additional detection behavior. Observe, block, and redact action configs own +the finding criteria applied after scanning. + +Within modules and classes, place the public API before private implementation +details whenever dependency ordering allows it. Keep private constants, helper +records, functions, and methods toward the bottom of their containing scope. + +The handwritten typing policy is AST-enforced by +`tests/test_typing_policy.py`. Generated bindings under +`src/privacy_guard/bindings` are excluded. Keep generics parameterized and +suppressions narrow and rule-specific. Production and test overrides are +explicit; example scripts omit `@override` to keep their public API surface +minimal and copyable. + +Run the committed full-path performance evidence with: + +```bash +uv run --frozen python scripts/benchmark_privacy_guard.py --suite full +``` + +Use the default quick suite during iteration. Both suites verify findings and +reconstruction outcomes before reporting median wall time and peak traced +allocation. diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md new file mode 100644 index 0000000..99bb120 --- /dev/null +++ b/projects/privacy-guard/README.md @@ -0,0 +1,242 @@ +# Privacy Guard + +An OpenShell supervisor middleware. The supervisor calls it over gRPC (the +`SupervisorMiddleware` service in `bindings/`) to inspect provider-bound HTTP +requests *before* credentials are attached. Privacy Guard parses the request +body, scans its text for sensitive values, and returns an allow/deny decision +plus an optionally rewritten body that the supervisor forwards instead of the +original. + +> Status: **hardened research project with a passthrough production default.** Strict policy parsing, +> request-level processing, the generic JSON handler, safe gRPC adaptation, and +> a loopback server are implemented. The production scanner is still the +> passthrough scanner; no real PII scanner exists yet. + +The [OpenShell E2E example](examples/openshell-e2e/README.md) launches the +middleware, attaches it to a disposable sandbox, and verifies that OpenShell +forwards a reconstructed provider-shaped request to a local capture endpoint. + +## Request flow + +`RequestProcessor` orchestrates one complete request: + +``` +proto HttpRequestEvaluation + -> payloads.InterceptedRequest proto-free capture of the request + -> body.FormatHandler.normalize() -> body.RequestBody (TextBlocks) + -> scanners.Scanner.scan(text_block) -> scanners.Finding[] + -> policy via config.PolicyConfig -> per-block replacements + -> body.FormatHandler.reconstruct() -> rewritten body (bytes) + -> payloads.ProcessingResult decision + replacement + findings + -> proto HttpRequestResult +``` + +The servicer is the only seam that touches both the proto messages and the +domain types: it translates proto -> domain on the way in and domain -> proto on +the way out. Everything below the service layer is free of `bindings/` and +gRPC. OpenShell applies an allowed mutation and forwards the provider request; +Privacy Guard does not make the provider call itself. + +Scanners continue to operate on exactly one text block. Scanner calls run on a +dedicated four-thread executor rather than the gRPC event loop; scanners must be +thread-safe and must not retain request content. Format handlers decide +which text blocks are relevant and own their addressing scheme. The processor +scans every emitted text block but treats each block path as opaque, so adding a +new request format does not require provider-specific processor logic. + +## Scanner and JSON policy + +Findings carry scanner-owned `low`, `medium`, or `high` confidence. Each action +selects the findings it applies to by `entity_types` and `minimum_confidence`: + +```json +{ + "body_format": "json", + "on_finding": { + "action": "redact", + "entity_types": ["email"], + "minimum_confidence": "high", + "template": "[{entity}]" + } +} +``` + +`PolicyConfig.on_finding` is a Pydantic discriminated union keyed by `action`. +Observe, block, and redact each carry their own finding criteria; only the +redact variant has a `template`. `entity_types: null` selects every emitted +entity type, while an empty list selects none. `minimum_confidence: null` +accepts every confidence level. + +Scanner configuration is independent of policy. A concrete `ScannerConfig` +subtype controls what and how its scanner detects; scanners can run without a +`PolicyConfig`. Action criteria filter the resulting `Finding` values for one +policy evaluation and never reconfigure the scanner. + +Every scanner config declares its complete `entity_types` catalog. When an +action selects entity types, every configured name must occur in the union of +the active scanner catalogs. Unknown names fail config validation instead of +silently producing no findings. + +Every JSON string value and every object key is scanned; JSON numbers, booleans, +and nulls are not. Keys are observable in `observe`, cause normal denial in +`block`, and cause a stable deny in `redact` because collision-safe key mutation +is not supported. Value findings may overlap for observe/block. Redaction picks +winners deterministically by confidence, span length, offsets, scanner identity, +and entity. A scanner sequence is passed to `RequestProcessor`; scanner names +must be unique and remain visible in aggregated findings. + +The default service uses `PassthroughScanner`. The deterministic email regex and +its server entry point live entirely under `examples/openshell-e2e`; production +package modules neither export nor select it. + +A scanner is a nominal extension: declare its strict configuration type and +implement only `_scan`. The base constructor validates and retains the config, +and the public `scan` wrapper validates the returned tuple and each `Finding`. + +```python +from privacy_guard.scanners import Finding, Scanner, ScannerConfig + + +class ExampleScanner(Scanner[ScannerConfig]): + def _scan(self, text_block: str) -> tuple[Finding, ...]: + return () +``` + +Applications construct the processor with a sequence, for example +`RequestProcessor([ExampleScanner(ScannerConfig(name="example", entity_types=frozenset()))])`. +The base +constructor infers and validates the scanner's declared config type. A scanner +returns block-relative +`Finding` values; the processor composes each one into a `RequestBodyFinding` +with the owning `TextBlock.path`. + +Applications serving a custom scanner can use the high-level server API, which +owns processor, middleware, gRPC, and shutdown wiring: + +```python +from privacy_guard.service import MiddlewareServer + +scanner = ExampleScanner(ScannerConfig(name="example", entity_types=frozenset())) +server = MiddlewareServer(scanner=scanner) +server.serve() # Defaults to 127.0.0.1:50051 +``` + +## Resource and failure behavior + +The service enforces the protocol's 4 MiB input and replacement-body maximum, +32 finding groups, and exact 4 KiB encoded limit for each aggregate finding. +The operator may configure a lower effective maximum; this project cannot observe +that value, so deployments must keep the registration aligned with the 4 MiB +manifest or add the lower limit to service configuration. + +JSON parsing is additionally bounded to 64 nesting levels, 4,096 text blocks, +and 4 MiB of scanned characters. Scanning is capped at 256 findings per block, +4,096 per request, four active scanner workers, and 16 concurrent gRPC calls. +Shape excess is invalid input. Finding or outbound representation excess returns +a stable `privacy_guard_limit_exceeded` deny with no body or partial findings, +avoiding a failure-mode-dependent fail open. +Redacted text is projected against the body budget before it is rendered, which +bounds template and finding amplification. The authoritative serialized-body +limit is checked immediately after format reconstruction and again at the gRPC +boundary. + +Operational logs contain request ID, duration, action, finding count, and safe +error code only—never bodies, text blocks, or matches. A cancelled RPC does not +release its scanner slot until its synchronous worker really exits. + +## Module map + +| Module | Responsibility | +| --- | --- | +| `config` | Strict Pydantic `PolicyConfig` parsing at the untrusted config boundary | +| `constants` | Package-wide limits, service metadata, and stable protocol values | +| `errors` | Closed, content-safe error catalog shared by all components | +| `payloads` | Frozen `InterceptedRequest` and `ProcessingResult` domain records | +| `body` | Nominal `FormatHandler` ABC + `JsonHandler`; strict `RequestBody`, `TextBlock` models | +| `scanners` | `Scanner` ABC + strict `ScannerConfig`, `Finding`, and `RequestBodyFinding` models | +| `processor` | Proto-free request orchestration and policy application | +| `service` | High-level `MiddlewareServer`, gRPC lifecycle, and servicer adapter | +| `bindings` | generated protobuf stubs — do not edit by hand | + +## Notes for implementers + +- **Scanner metadata.** Applications pass an explicit `ScannerConfig` subtype to + the scanner constructor; `Scanner` infers and validates the declared generic + config type. The read-only `config` property preserves that concrete type, and + `supported_entity_types` returns its required `entity_types` catalog. +- **Finding types.** A scanner returns strict block-relative `scanners.Finding` + values. The processor creates `RequestBodyFinding` values with the owning text-block + path, and the servicer aggregates those into the protocol's count-based `Finding`. +- **Text-block paths.** `Finding` has no path state. A `Scanner` sees only one text + block; the processor attaches `TextBlock.path` by creating `RequestBodyFinding`. +- **Format selection.** `PolicyConfig.body_format` (default `json`) picks a handler + from the `format_handlers` mapping supplied to `RequestProcessor`. JSON is the + only built-in format; applications can supply mappings containing other formats. + Do not infer a provider from the request body or headers. +- **Format handlers.** A custom handler subclasses `FormatHandler`, calls + `super().__init__(format_name="...")`, and implements the protected + `_normalize` and `_reconstruct` hooks with `@override`. Return normally + constructed strict `RequestBody` and `TextBlock` models; handler instances are + reused concurrently and must retain no request content or mutable request state. + + ```python + from collections.abc import Mapping + + from typing_extensions import override + + from privacy_guard.body import FormatHandler, RequestBody + from privacy_guard.config import PolicyConfig + + + class CustomHandler(FormatHandler): + def __init__(self) -> None: + super().__init__(format_name="custom") + + @override + def _normalize(self, raw_body: bytes, policy_config: PolicyConfig) -> RequestBody: + return RequestBody( + text_blocks=(), parsed_value=None, original_bytes=raw_body + ) + + @override + def _reconstruct( + self, + request_body: RequestBody, + replacements_by_path: Mapping[str, str], + ) -> bytes: + return request_body.original_bytes + ``` +- **Log safety.** Raw bodies, parsed values, and text-block content use `repr=False` to + keep content out of routine domain representations; this does not sanitize + arbitrary tracebacks. Cataloged errors and gRPC status details are + content-safe, and caught collaborator exception chains must never be logged. + +## Development validation + +Run the complete local gate from this directory: + +```bash +scripts/validate.sh +``` + +To exercise the package with the minimum supported interpreter, run +`scripts/validate.sh --python 3.11`. This project has no dedicated CI workflow, +so this committed entry point is the authoritative contributor gate. It runs the +full tests, formatting, lint, curated `ty` rules, import smoke, and package build. +The AST policy test rejects cast operations and explicit dynamic typing in +handwritten `src`, `tests`, and `examples`; only generated protobuf/gRPC bindings +are excluded. + +The deterministic benchmark gate uses three samples per median and covers every +required size, finding load, scanner shape, and reconstruction mode: + +```bash +uv run --frozen python scripts/benchmark_privacy_guard.py +``` + +For the broader scenario set and seven samples per median, pass `--suite full`. +Pass `--profile profile.out` to either suite to record a `cProfile` artifact. +The harness reports median wall time and median peak traced allocation for the +complete normalize, scan, output validation, policy, and reconstruction path. +Committed reference measurements and methodology are in +[BENCHMARKS.md](BENCHMARKS.md). diff --git a/projects/privacy-guard/examples/openshell-e2e/README.md b/projects/privacy-guard/examples/openshell-e2e/README.md new file mode 100644 index 0000000..76596ee --- /dev/null +++ b/projects/privacy-guard/examples/openshell-e2e/README.md @@ -0,0 +1,70 @@ +# Privacy Guard OpenShell E2E example + +This example proves the complete local path: + +```text +sandbox curl + -> OpenShell network policy and HTTP middleware + -> Privacy Guard gRPC service + -> reconstructed request + -> host capture endpoint +``` + +It uses no provider credentials or paid API. The sandbox sends a provider-shaped +JSON request to a local capture endpoint. Privacy Guard uses its explicit +development-only deterministic scanner to redact `user@example.com`, and verifies the +captured request body after OpenShell forwards it. + +## Prerequisites + +- macOS with the Homebrew OpenShell `0.0.86` gateway running +- Docker Desktop running with + pinned `ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:aeef1c63f00e2913ea002ccb3aaf925f338b5c5d70e63576f0d95c16a138044e` available locally +- `brew`, `docker`, `openshell`, `python3`, and `uv` +- ports `50051` and `18080` available on the host + +Run: + +```bash +./projects/privacy-guard/examples/openshell-e2e/run.sh +``` + +The pytest hook is skipped by default. From the repository root, run the same +state-mutating harness through the integration suite only with: + +```bash +PRIVACY_GUARD_RUN_OPEN_SHELL_E2E=1 \ + uv run --project projects/privacy-guard pytest -q \ + projects/privacy-guard/tests/examples/test_openshell_e2e.py +``` + +The harness temporarily appends a `privacy-guard-e2e` registration to +`~/.config/openshell/gateway.toml`, restarts the gateway, creates a disposable +sandbox, and restores the exact prior gateway configuration on exit. Existing +gateway configuration is preserved. + +If the default `en0` address is not reachable from Docker, provide the host IPv4 +address explicitly: + +```bash +PRIVACY_GUARD_E2E_HOST_ADDRESS=192.168.1.10 \ + ./projects/privacy-guard/examples/openshell-e2e/run.sh +``` + +The default reference is the locally verified immutable digest used during this +spike. Override the sandbox image when necessary with +`PRIVACY_GUARD_E2E_SANDBOX_IMAGE`. The image must already exist in the local +Docker daemon and provide `/usr/bin/curl`. + +## Interrupted-run recovery + +Normal failures and signals trigger automatic cleanup. If the process is killed +without running its trap, inspect these files before restarting OpenShell: + +- `~/.config/openshell/gateway.toml.privacy-guard-e2e.bak` means the original + configuration should replace `gateway.toml`. +- `~/.config/openshell/gateway.toml.privacy-guard-e2e.absent` means no user + configuration existed before the run; remove both the marker and the temporary + `gateway.toml`. + +Then run `brew services restart openshell` and verify `openshell status`. diff --git a/projects/privacy-guard/examples/openshell-e2e/capture_server.py b/projects/privacy-guard/examples/openshell-e2e/capture_server.py new file mode 100644 index 0000000..68a885e --- /dev/null +++ b/projects/privacy-guard/examples/openshell-e2e/capture_server.py @@ -0,0 +1,60 @@ +"""Capture one HTTP POST body for the Privacy Guard OpenShell E2E example.""" + +from __future__ import annotations + +import argparse +import os +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +MAX_CAPTURE_BYTES = 1024 * 1024 + + +class _CaptureHandler(BaseHTTPRequestHandler): + output_path: Path + + def do_POST(self) -> None: + """Persist one bounded request body and return a deterministic response.""" + content_length = int(self.headers.get("content-length", "0")) + if content_length < 0 or content_length > MAX_CAPTURE_BYTES: + self.send_error(413) + return + + request_body = self.rfile.read(content_length) + temporary_path = self.output_path.with_suffix(".tmp") + temporary_path.write_bytes(request_body) + os.replace(temporary_path, self.output_path) + + response_body = b'{"captured":true}' + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(response_body))) + self.end_headers() + self.wfile.write(response_body) + + # The example expects exactly one request. + self.server.shutdown() + + def log_message(self, format: str, *args: object) -> None: + """Suppress request content and default access-log output.""" + + +def main() -> int: + """Serve until one request is captured.""" + parser = argparse.ArgumentParser() + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=18080) + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args() + + _CaptureHandler.output_path = arguments.output + server = ThreadingHTTPServer((arguments.host, arguments.port), _CaptureHandler) + try: + server.serve_forever() + finally: + server.server_close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/projects/privacy-guard/examples/openshell-e2e/middleware_server.py b/projects/privacy-guard/examples/openshell-e2e/middleware_server.py new file mode 100644 index 0000000..3750c3f --- /dev/null +++ b/projects/privacy-guard/examples/openshell-e2e/middleware_server.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Development-only deterministic Privacy Guard server for the manual E2E.""" + +from __future__ import annotations + +import argparse +import re + +from privacy_guard.scanners import ( + Confidence, + Finding, + Scanner, + ScannerConfig, +) +from privacy_guard.service import MiddlewareServer + + +class ExampleEmailScanner(Scanner[ScannerConfig]): + """Detect example email addresses deterministically; not a production scanner.""" + + _EMAIL = re.compile( + r"(? tuple[Finding, ...]: + return tuple( + Finding( + entity="email", + scanner_name=self.scanner_name, + start_offset=match.start(), + end_offset=match.end(), + confidence=Confidence.HIGH, + ) + for match in self._EMAIL.finditer(text_block) + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--listen", default="127.0.0.1:50051") + arguments = parser.parse_args() + scanner = ExampleEmailScanner( + ScannerConfig(name="example_regex", entity_types=frozenset({"email"})) + ) + server = MiddlewareServer(scanner=scanner) + server.serve(arguments.listen) + + +if __name__ == "__main__": + main() diff --git a/projects/privacy-guard/examples/openshell-e2e/policy.yaml b/projects/privacy-guard/examples/openshell-e2e/policy.yaml new file mode 100644 index 0000000..732ee29 --- /dev/null +++ b/projects/privacy-guard/examples/openshell-e2e/policy.yaml @@ -0,0 +1,38 @@ +version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + privacy_guard_capture: + name: Privacy Guard E2E capture endpoint + endpoints: + - host: host.openshell.internal + port: 18080 + protocol: rest + enforcement: enforce + access: full + binaries: + - { path: /usr/bin/curl } + +network_middlewares: + privacy_guard_redaction: + name: Redact deterministic example email + middleware: privacy-guard-e2e + order: 0 + config: + body_format: json + on_finding: + action: redact + entity_types: [email] + minimum_confidence: high + on_error: fail_closed + endpoints: + include: [host.openshell.internal] diff --git a/projects/privacy-guard/examples/openshell-e2e/run.sh b/projects/privacy-guard/examples/openshell-e2e/run.sh new file mode 100755 index 0000000..ab6d2f7 --- /dev/null +++ b/projects/privacy-guard/examples/openshell-e2e/run.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PYTHON_PROJECT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +MIDDLEWARE_PORT=50051 +CAPTURE_PORT=18080 +GATEWAY_CONFIG="${HOME}/.config/openshell/gateway.toml" +GATEWAY_BACKUP="${GATEWAY_CONFIG}.privacy-guard-e2e.bak" +GATEWAY_ABSENT_MARKER="${GATEWAY_CONFIG}.privacy-guard-e2e.absent" +SANDBOX_NAME="privacy-guard-e2e-$$" +SANDBOX_IMAGE="${PRIVACY_GUARD_E2E_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:aeef1c63f00e2913ea002ccb3aaf925f338b5c5d70e63576f0d95c16a138044e}" +EXPECTED_TEXT="hello [email]" + +temporary_directory="" +middleware_pid="" +capture_pid="" +gateway_config_changed=false + +log() { + printf '[privacy-guard-e2e] %s\n' "$*" +} + +fail() { + printf '[privacy-guard-e2e] ERROR: %s\n' "$*" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1" +} + +wait_for_tcp_port() { + local port="$1" + local attempt + for attempt in $(seq 1 60); do + if python3 -c "import socket; s=socket.create_connection(('127.0.0.1', ${port}), 0.2); s.close()" 2>/dev/null; then + return 0 + fi + sleep 0.25 + done + return 1 +} + +wait_for_gateway() { + local attempt + for attempt in $(seq 1 60); do + if openshell status >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +restore_gateway_config() { + if [[ "${gateway_config_changed}" != true ]]; then + return + fi + + if [[ -f "${GATEWAY_BACKUP}" ]]; then + mv "${GATEWAY_BACKUP}" "${GATEWAY_CONFIG}" + elif [[ -f "${GATEWAY_ABSENT_MARKER}" ]]; then + rm -f "${GATEWAY_CONFIG}" "${GATEWAY_ABSENT_MARKER}" + else + printf '[privacy-guard-e2e] ERROR: gateway backup state is missing; not restarting gateway\n' >&2 + return + fi + gateway_config_changed=false + brew services restart openshell >/dev/null + wait_for_gateway || printf '[privacy-guard-e2e] WARNING: restored gateway did not become ready\n' >&2 +} + +cleanup() { + set +e + openshell sandbox delete "${SANDBOX_NAME}" >/dev/null 2>&1 + restore_gateway_config + if [[ -n "${middleware_pid}" ]]; then + kill "${middleware_pid}" >/dev/null 2>&1 + wait "${middleware_pid}" >/dev/null 2>&1 + fi + if [[ -n "${capture_pid}" ]]; then + kill "${capture_pid}" >/dev/null 2>&1 + wait "${capture_pid}" >/dev/null 2>&1 + fi + if [[ -n "${temporary_directory}" ]]; then + rm -rf "${temporary_directory}" + fi +} + +trap cleanup EXIT INT TERM + +if [[ "$(uname -s)" != Darwin ]]; then + fail "this automated example currently supports the Homebrew macOS gateway" +fi + +for command_name in brew docker openshell python3 uv; do + require_command "${command_name}" +done + +if [[ -e "${GATEWAY_BACKUP}" || -e "${GATEWAY_ABSENT_MARKER}" ]]; then + fail "stale gateway recovery files exist; follow README.md recovery instructions" +fi + +host_address="${PRIVACY_GUARD_E2E_HOST_ADDRESS:-$(ipconfig getifaddr en0 || true)}" +[[ "${host_address}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || \ + fail "set PRIVACY_GUARD_E2E_HOST_ADDRESS to a host IPv4 address reachable from Docker" +docker image inspect "${SANDBOX_IMAGE}" >/dev/null 2>&1 || \ + fail "sandbox image is not local; pull ${SANDBOX_IMAGE} or set PRIVACY_GUARD_E2E_SANDBOX_IMAGE" + +temporary_directory="$(mktemp -d)" +capture_file="${temporary_directory}/captured-request.json" +middleware_log="${temporary_directory}/middleware.log" +capture_log="${temporary_directory}/capture.log" + +log "starting Privacy Guard and capture endpoint on ${host_address}" +uv run --project "${PYTHON_PROJECT}" python "${SCRIPT_DIR}/middleware_server.py" \ + --listen "0.0.0.0:${MIDDLEWARE_PORT}" >"${middleware_log}" 2>&1 & +middleware_pid=$! +python3 "${SCRIPT_DIR}/capture_server.py" --port "${CAPTURE_PORT}" \ + --output "${capture_file}" >"${capture_log}" 2>&1 & +capture_pid=$! + +wait_for_tcp_port "${MIDDLEWARE_PORT}" || { + tail -n 40 "${middleware_log}" >&2 + fail "Privacy Guard did not start" +} +wait_for_tcp_port "${CAPTURE_PORT}" || { + tail -n 40 "${capture_log}" >&2 + fail "capture server did not start" +} + +mkdir -p "$(dirname "${GATEWAY_CONFIG}")" +if [[ -f "${GATEWAY_CONFIG}" ]]; then + grep -q 'name = "privacy-guard-e2e"' "${GATEWAY_CONFIG}" && \ + fail "gateway already contains a privacy-guard-e2e registration" + cp -p "${GATEWAY_CONFIG}" "${GATEWAY_BACKUP}" +else + touch "${GATEWAY_ABSENT_MARKER}" + : >"${GATEWAY_CONFIG}" +fi +gateway_config_changed=true + +cat >>"${GATEWAY_CONFIG}" </dev/null +wait_for_gateway || { + tail -n 80 /opt/homebrew/var/log/openshell/openshell-gateway.err.log >&2 || true + fail "OpenShell gateway did not accept the middleware registration" +} + +request_body='{"messages":[{"role":"user","content":"hello user@example.com"}],"model":"capture-model"}' +log "creating disposable sandbox and sending provider-shaped request" +openshell sandbox create \ + --name "${SANDBOX_NAME}" \ + --from "${SANDBOX_IMAGE}" \ + --no-auto-providers \ + --policy "${SCRIPT_DIR}/policy.yaml" \ + --no-keep \ + --no-tty \ + -- \ + curl --fail-with-body --silent --show-error \ + --request POST \ + --header 'content-type: application/json' \ + --data "${request_body}" \ + "http://host.openshell.internal:${CAPTURE_PORT}/v1/chat/completions" + +for attempt in $(seq 1 40); do + [[ -f "${capture_file}" ]] && break + sleep 0.25 +done +[[ -f "${capture_file}" ]] || fail "capture endpoint did not receive the request" + +python3 - "${capture_file}" "${EXPECTED_TEXT}" <<'PY' +import json +import pathlib +import sys + +captured = json.loads(pathlib.Path(sys.argv[1]).read_bytes()) +actual = captured["messages"][0]["content"] +if actual != sys.argv[2]: + raise SystemExit(f"unexpected reconstructed content: {actual!r}") +if captured["model"] != "capture-model": + raise SystemExit("untouched provider-shaped field changed") +PY + +log "SUCCESS: OpenShell forwarded reconstructed content: ${EXPECTED_TEXT}" diff --git a/projects/privacy-guard/examples/openshell-manual/README.md b/projects/privacy-guard/examples/openshell-manual/README.md new file mode 100644 index 0000000..063475a --- /dev/null +++ b/projects/privacy-guard/examples/openshell-manual/README.md @@ -0,0 +1,149 @@ +# Manually try Privacy Guard with Claude Code + +This example temporarily runs the installed OpenShell gateway with a config from +this directory. It does not create or modify `~/.config/openshell/gateway.toml`, +and it does not create a project-local state directory. + +The example scanner detects email addresses in Claude Code request bodies and +replaces them with `[email]` before Anthropic receives the request. + +## Prerequisites + +- macOS with Docker Desktop running +- Python 3 and `uv` +- OpenShell installed with its recommended installer: + + ```bash + curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh + ``` + +Run the example commands from this directory: + +```bash +cd projects/privacy-guard/examples/openshell-manual +``` + +## 1. Edit the example config + +Find the host address that Docker can reach: + +```bash +ipconfig getifaddr en0 +``` + +Replace `REPLACE_WITH_HOST_IP` in `gateway.toml` with that address. If `en0` +does not return an address, find the active interface with: + +```bash +route get default | grep interface +``` + +Only the checked-out `gateway.toml` is edited. Do not copy it into +`~/.config/openshell`. + +## 2. Start Privacy Guard + +In terminal 1: + +```bash +uv run --project ../.. python ../openshell-e2e/middleware_server.py \ + --listen 0.0.0.0:50051 +``` + +Leave it running. + +## 3. Run the installed gateway with the example config + +In terminal 2: + +```bash +brew services stop openshell + +OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" \ +openshell-gateway --config "$PWD/gateway.toml" +``` + +The first command stops the background service so the foreground gateway can use +the standard port. The second command reuses the credentials and state created +by the recommended macOS installation, but loads `gateway.toml` from this +directory. It should stay in the foreground; `Server listening` means it is +ready. + +Middleware registration is static. After editing `gateway.toml`, stop this +foreground process with `Ctrl-C` and run the second command again. + +## 4. Create the sandbox and run Claude + +In terminal 3: + +```bash +openshell status + +openshell sandbox create \ + --name privacy-guard-lab \ + --from base \ + --no-auto-providers \ + --policy "$PWD/policy.yaml" \ + -- claude +``` + +Choose Claude Code's subscription-account login and complete authentication. +Then enter: + +```text +Tell me something that rhymes with my email wendy@gmail.com +``` + +Privacy Guard should replace the address with `[email]` before Claude sees it. +Model output is nondeterministic, so use OpenShell's logs as the authoritative +check: + +```bash +openshell logs privacy-guard-lab --tail +``` + +Look for the `api.anthropic.com/v1/messages` request with `transformed:true` and +an email finding. + +## Change the behavior + +Edit `policy.yaml` and change `on_finding.action` to `observe`, `block`, or +`redact`, then apply it without recreating the sandbox: + +```bash +openshell policy set privacy-guard-lab --policy "$PWD/policy.yaml" --wait +``` + +- `redact` sends `[email]` to Claude. +- `observe` records the finding but sends the original email. +- `block` denies the request. + +To reconnect later: + +```bash +openshell sandbox connect privacy-guard-lab +``` + +Then run `claude` inside the sandbox. + +## Cleanup + +Exit Claude and the sandbox, then delete it: + +```bash +openshell sandbox delete privacy-guard-lab +``` + +Stop the foreground gateway with `Ctrl-C` in terminal 2, then restore the normal +background gateway: + +```bash +brew services start openshell +``` + +Stop Privacy Guard with `Ctrl-C` in terminal 1. No default OpenShell config was +changed. + +This test uses Claude Code because subscription prompts are sent in inspectable +HTTP request bodies. ChatGPT-subscription Codex currently sends prompts in +WebSocket frames, which this HTTP middleware cannot inspect. diff --git a/projects/privacy-guard/examples/openshell-manual/gateway.toml b/projects/privacy-guard/examples/openshell-manual/gateway.toml new file mode 100644 index 0000000..fb9189f --- /dev/null +++ b/projects/privacy-guard/examples/openshell-manual/gateway.toml @@ -0,0 +1,12 @@ +# OpenShell gateway configuration for the manual Privacy Guard example. +# Replace REPLACE_WITH_HOST_IP, then pass this file to openshell-gateway with +# --config. Do not copy it into ~/.config/openshell. + +[openshell] +version = 1 + +[[openshell.supervisor.middleware]] +name = "privacy-guard-manual" +grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051" +max_body_bytes = 4194304 +timeout = "5s" diff --git a/projects/privacy-guard/examples/openshell-manual/policy.yaml b/projects/privacy-guard/examples/openshell-manual/policy.yaml new file mode 100644 index 0000000..8457472 --- /dev/null +++ b/projects/privacy-guard/examples/openshell-manual/policy.yaml @@ -0,0 +1,47 @@ +version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + claude_code: + name: Claude Code subscription access + endpoints: + - host: api.anthropic.com + port: 443 + protocol: rest + enforcement: enforce + access: full + - host: platform.claude.com + port: 443 + protocol: rest + enforcement: enforce + access: full + - { host: statsig.anthropic.com, port: 443 } + - { host: sentry.io, port: 443 } + binaries: + - { path: /usr/local/bin/claude } + - { path: /usr/bin/node } + +network_middlewares: + privacy_guard_redaction: + name: Redact deterministic example emails + middleware: privacy-guard-manual + order: 0 + config: + body_format: json + on_finding: + action: redact + entity_types: [email] + minimum_confidence: high + on_error: fail_closed + endpoints: + include: + - api.anthropic.com diff --git a/projects/privacy-guard/middleware-dev-manifest.json b/projects/privacy-guard/middleware-dev-manifest.json new file mode 100644 index 0000000..6698ff1 --- /dev/null +++ b/projects/privacy-guard/middleware-dev-manifest.json @@ -0,0 +1,7 @@ +{ + "openshell_version": "v0.0.86", + "proto_source": "https://raw.githubusercontent.com/NVIDIA/OpenShell/v0.0.86/proto/supervisor_middleware.proto", + "proto_sha256": "e9d5a992ff5b50a33e9625176aaf6df8496d6774aa2ef3afe5cae7bc83c01105", + "languages": ["python"], + "python_package": "privacy_guard" +} diff --git a/projects/privacy-guard/proto/supervisor_middleware.proto b/projects/privacy-guard/proto/supervisor_middleware.proto new file mode 100644 index 0000000..dbde411 --- /dev/null +++ b/projects/privacy-guard/proto/supervisor_middleware.proto @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package openshell.middleware.v1; + +import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; + +// SupervisorMiddleware lets an operator-run service inspect and transform +// sandbox HTTP egress before OpenShell injects credentials. +service SupervisorMiddleware { + // Describe returns the service manifest and declared bindings. + rpc Describe(google.protobuf.Empty) returns (MiddlewareManifest); + + // ValidateConfig checks service-specific configuration for one binding. + rpc ValidateConfig(ValidateConfigRequest) returns (ValidateConfigResponse); + + // EvaluateHttpRequest returns an allow, deny, or mutation decision for one + // buffered HTTP request. + rpc EvaluateHttpRequest(HttpRequestEvaluation) returns (HttpRequestResult); +} + +// MiddlewareManifest describes one middleware service and the bindings it +// exposes. The service is the operator-run gRPC server implementing +// SupervisorMiddleware. +message MiddlewareManifest { + // Human-readable middleware service name used only for diagnostics. This is + // not required to match an operator-owned registration name. + string name = 1; + // Release version of the middleware service implementation, used for + // diagnostics. + string service_version = 2; + // Bindings exposed by this middleware service. + repeated MiddlewareBinding bindings = 3; +} + +// MiddlewareBinding declares one operation and phase supported by a service. +message MiddlewareBinding { + // Supported operation. V1 supports HTTP_REQUEST. + SupervisorMiddlewareOperation operation = 1; + // Supported evaluation phase. V1 supports PRE_CREDENTIALS. + SupervisorMiddlewarePhase phase = 2; + // Maximum request or replacement body this binding can process. + uint64 max_body_bytes = 3; + // Optional binding-specific RPC timeout. Empty uses the operator-configured + // service timeout, or the 500ms platform default when that is also omitted. + // A non-empty value may shorten but cannot extend the operator timeout. + // Values use an integer with an `ms` or `s` suffix and must be between + // 10ms and 30s. + string timeout = 4; +} + +// ValidateConfigRequest contains one policy configuration to validate. +message ValidateConfigRequest { + // Service-specific policy configuration. + google.protobuf.Struct config = 1; + // Built-in middleware name or operator-owned registration name. + string middleware_name = 2; +} + +// ValidateConfigResponse reports whether a policy configuration is accepted. +message ValidateConfigResponse { + // True when the service accepts the configuration. + bool valid = 1; + // Human-readable validation failure reason. Empty when valid is true. + string reason = 2; +} + +// HttpRequestEvaluation contains one buffered HTTP request to evaluate. +message HttpRequestEvaluation { + // Evaluation phase selected for this request. + SupervisorMiddlewarePhase phase = 1; + // Sandbox and request identity available to the supervisor. + // The encoded context is limited to 4 KiB. + RequestContext context = 2; + // Validated service-specific policy configuration. + // The encoded configuration is limited to 64 KiB. + google.protobuf.Struct config = 3; + // Destination and HTTP request target. + // The encoded target is limited to 32 KiB. + HttpRequestTarget target = 4; + // HTTP request headers before OpenShell injects credentials, in wire + // order. Repeated header names are preserved as separate entries. Protected + // credential, routing, framing, and hop-by-hop headers are omitted. + // At most 128 lines and 64 KiB of encoded headers are included. + repeated HttpHeader headers = 5; + // Buffered request body, limited to 4 MiB. Empty for a bodyless request. + bytes body = 6; + // Built-in middleware name or operator-owned registration name. + string middleware_name = 7; +} + +// HttpHeader is one request header line. +message HttpHeader { + // Lowercased header name. + string name = 1; + // Header value with surrounding whitespace trimmed. + string value = 2; +} + +// Supervisor operation selected for middleware evaluation. +enum SupervisorMiddlewareOperation { + SUPERVISOR_MIDDLEWARE_OPERATION_UNSPECIFIED = 0; + SUPERVISOR_MIDDLEWARE_OPERATION_HTTP_REQUEST = 1; +} + +// Ordered phase within a supervisor operation. +enum SupervisorMiddlewarePhase { + SUPERVISOR_MIDDLEWARE_PHASE_UNSPECIFIED = 0; + SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS = 1; +} + +// RequestContext identifies the sandbox request being evaluated. +message RequestContext { + // Request id used to correlate middleware and supervisor logs. + string request_id = 1; + // Sandbox id that originated the request. + string sandbox_id = 2; + // Workload process that originated the request, when available. + Process originating_process = 3; +} + +// HttpRequestTarget describes the admitted HTTP destination and request target. +message HttpRequestTarget { + // Request scheme, such as "http" or "https". + string scheme = 1; + // Destination hostname selected by network policy. + string host = 2; + // Destination TCP port. + uint32 port = 3; + // HTTP request method. + string method = 4; + // Request path without the query string. + string path = 5; + // Raw request query string without the leading question mark. + string query = 6; +} + +// Process identifies a workload process and its executable ancestry. +message Process { + // Executable path for the originating process. + string binary = 1; + // Process id within the sandbox. + uint32 pid = 2; + // Executable paths for ancestor processes, nearest parent first. + repeated string ancestors = 3; +} + +// Decision controls whether OpenShell continues processing the request. +enum Decision { + // Invalid response value handled according to the policy failure mode. + DECISION_UNSPECIFIED = 0; + // Continue processing the request and apply any returned mutations. + DECISION_ALLOW = 1; + // Deny the request before credentials are injected or data is sent upstream. + DECISION_DENY = 2; +} + +// Finding is an audit-safe observation produced during evaluation. +message Finding { + // Stable, service-defined finding type. + string type = 1; + // Human-readable finding label that does not contain request content. + string label = 2; + // Number of matching observations represented by this finding. + uint32 count = 3; + // Service-defined confidence level. + string confidence = 4; + // Service-defined severity level. + string severity = 5; +} + +// ExistingHeaderAction controls how a header write behaves when the +// case-insensitive header name is already present. Every action writes the +// value when the header is absent. +enum ExistingHeaderAction { + EXISTING_HEADER_ACTION_UNSPECIFIED = 0; + // Add another field value without changing existing values. + EXISTING_HEADER_ACTION_APPEND = 1; + // Remove every existing value, then add the new value. + EXISTING_HEADER_ACTION_OVERWRITE = 2; + // Leave the existing values unchanged. + EXISTING_HEADER_ACTION_SKIP = 3; +} + +// WriteHeader proposes one header value and defines collision behavior. +message WriteHeader { + string name = 1; + string value = 2; + ExistingHeaderAction on_existing = 3; +} + +// RemoveHeader removes every value for a case-insensitive header name. +message RemoveHeader { + string name = 1; +} + +// HeaderMutation is one ordered request-header operation. +message HeaderMutation { + oneof operation { + WriteHeader write = 1; + RemoveHeader remove = 2; + } +} + +// HttpRequestResult contains the decision and optional request mutations. +message HttpRequestResult { + // Allow or deny decision for this request. + Decision decision = 1; + // Free-form service diagnostic. OpenShell does not relay this text into + // denied responses or security logs. Limited to 4 KiB before discarding. + string reason = 2; + // Replacement request body when has_body is true. Limited to 4 MiB. + bytes body = 3; + // True when body should replace the request body, including with an empty body. + bool has_body = 4; + // Ordered request-header mutations applied before the next middleware and + // before forwarding. Header writes are restricted to the + // "x-openshell-middleware-" namespace. Removes may target other visible + // request headers, but credential, routing, framing, and hop-by-hop headers + // are always protected. A violating result is a middleware failure handled + // according to the policy failure mode. At most 64 operations, 32 KiB of + // validated name/value data, and 64 KiB encoded are accepted. + repeated HeaderMutation header_mutations = 5; + // Audit-safe findings produced during evaluation. For operator-run services, + // OpenShell logs platform-owned fields derived from the operator-owned + // registration name rather than service-provided type, label, confidence, + // or metadata text. + // At most 32 findings of at most 4 KiB encoded each are accepted per stage. + // A policy selects at most 10 stages, so one chain retains at most 320. + repeated Finding findings = 6; + // Non-secret service-defined metadata included in diagnostics. At most 64 + // entries and 32 KiB of combined key/value data are accepted. + map metadata = 7; + // Optional stable machine-readable code for a deny decision. Codes must + // start with a lowercase ASCII letter and contain only lowercase ASCII + // letters, digits, and underscores, with a maximum length of 64 bytes. + // OpenShell may return this code to the requester, unlike free-form reason. + string reason_code = 8; +} diff --git a/projects/privacy-guard/pyproject.toml b/projects/privacy-guard/pyproject.toml new file mode 100644 index 0000000..ef069c0 --- /dev/null +++ b/projects/privacy-guard/pyproject.toml @@ -0,0 +1,70 @@ +[project] +name = "privacy-guard" +version = "0.1.0" +description = "Inspect and enforce policy on provider-bound OpenShell requests." +readme = "README.md" +requires-python = ">=3.11" +license = "Apache-2.0" +authors = [ + { name = "NVIDIA CORPORATION & AFFILIATES" }, +] +dependencies = [ + "grpcio>=1.81.1,<2", + "protobuf>=6.33.5,<7", + "pydantic>=2.11,<3", + "typing-extensions>=4.12,<5", +] + +[project.scripts] +privacy-guard = "privacy_guard.service.server:main" + +[project.urls] +Repository = "https://github.com/NVIDIA/OpenShell-Research" + +[dependency-groups] +dev = [ + "pytest>=8.3,<9", + "pytest-asyncio>=0.25,<2", + "ruff>=0.12,<0.13", + "ty>=0.0.1a16,<0.1", +] + +[build-system] +requires = ["uv_build>=0.11.8,<0.12.0"] +build-backend = "uv_build" + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.ruff] +target-version = "py311" +extend-exclude = ["src/privacy_guard/bindings"] + +[tool.ruff.lint] +select = ["E", "F", "I", "UP"] + +[tool.ty.src] +# Generated protobuf/gRPC bindings are checked at their handwritten adapter seam; +# their generator-owned implementations and incomplete annotations stay excluded. +exclude = ["src/privacy_guard/bindings"] + +[tool.ty.rules] +blanket-ignore-comment = "error" +missing-override-decorator = "error" +missing-type-argument = "error" +redundant-cast = "error" +unresolved-attribute = "error" +unresolved-global = "error" +unresolved-import = "error" +unresolved-reference = "error" +unused-ignore-comment = "error" +unused-type-ignore-comment = "error" + +[[tool.ty.overrides]] +include = ["examples/**"] + +[tool.ty.overrides.rules] +missing-override-decorator = "ignore" + +[tool.uv] +required-version = ">=0.11.0" diff --git a/projects/privacy-guard/scripts/benchmark_privacy_guard.py b/projects/privacy-guard/scripts/benchmark_privacy_guard.py new file mode 100755 index 0000000..b9772ef --- /dev/null +++ b/projects/privacy-guard/scripts/benchmark_privacy_guard.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +"""Benchmark complete Privacy Guard normalize/scan/validate/reconstruct paths.""" + +from __future__ import annotations + +import argparse +import cProfile +import gc +import platform +import statistics +import time +import tracemalloc +from dataclasses import dataclass + +from pydantic import Field +from typing_extensions import override + +from privacy_guard.config import ( + ActionConfig, + ObserveActionConfig, + PolicyConfig, + RedactActionConfig, +) +from privacy_guard.constants import ( + MAX_BODY_BYTES, + MAX_FINDINGS_PER_BLOCK, + MAX_FINDINGS_PER_REQUEST, +) +from privacy_guard.payloads import InterceptedRequest, ProcessingDecision +from privacy_guard.processor import RequestProcessor +from privacy_guard.scanners import Finding, Scanner, ScannerConfig + +_KIB = 1024 +_MIB = 1024 * 1024 +_MARKER_CHARACTER = "x" +_SCANNER_COUNT_MULTIPLE = 4 +_MAXIMUM_WIDTH_NUMERIC_ELEMENT_COUNT = 2_097_151 +_MAXIMUM_WIDTH_NUMERIC_BYTES = 4_194_303 + + +class _BenchmarkScannerConfig(ScannerConfig): + findings_per_block: int = Field(ge=0, le=MAX_FINDINGS_PER_BLOCK) + + +class _BenchmarkScanner(Scanner[_BenchmarkScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + if not text_block.startswith( + _MARKER_CHARACTER * self.config.findings_per_block + ): + return () + return tuple( + Finding( + entity="benchmark", + scanner_name=self.scanner_name, + start_offset=offset, + end_offset=offset + 1, + ) + for offset in range(self.config.findings_per_block) + ) + + +@dataclass(frozen=True) +class _Scenario: + name: str + target_bytes: int + finding_load: str + scanner_count: int + reconstruction: str + body_shape: str = "text-blocks" + + +@dataclass(frozen=True) +class _PreparedScenario: + definition: _Scenario + processor: RequestProcessor + request: InterceptedRequest + expected_findings: int + + +@dataclass(frozen=True) +class _Measurement: + definition: _Scenario + actual_bytes: int + finding_count: int + median_wall_ms: float + median_peak_bytes: int + + +_QUICK_SCENARIOS = ( + _Scenario("1KiB-zero-one-noop", _KIB, "zero", 1, "noop"), + _Scenario( + "1KiB-typical-multiple-replacement", + _KIB, + "typical", + _SCANNER_COUNT_MULTIPLE, + "replacement", + ), + _Scenario("1MiB-typical-one-noop", _MIB, "typical", 1, "noop"), + _Scenario("1MiB-max-one-replacement", _MIB, "max", 1, "replacement"), + _Scenario( + "4MiB-zero-multiple-noop", + MAX_BODY_BYTES, + "zero", + _SCANNER_COUNT_MULTIPLE, + "noop", + ), + _Scenario( + "4MiB-max-multiple-replacement", + MAX_BODY_BYTES, + "max", + _SCANNER_COUNT_MULTIPLE, + "replacement", + ), +) + +_FULL_SCENARIOS = _QUICK_SCENARIOS + ( + _Scenario("1KiB-typical-one-noop", _KIB, "typical", 1, "noop"), + _Scenario("1MiB-zero-one-noop", _MIB, "zero", 1, "noop"), + _Scenario( + "1MiB-typical-multiple-replacement", + _MIB, + "typical", + _SCANNER_COUNT_MULTIPLE, + "replacement", + ), + _Scenario( + "1MiB-max-multiple-noop", + _MIB, + "max", + _SCANNER_COUNT_MULTIPLE, + "noop", + ), + _Scenario( + "4MiB-typical-one-replacement", MAX_BODY_BYTES, "typical", 1, "replacement" + ), + _Scenario( + "4MiB-typical-multiple-noop", + MAX_BODY_BYTES, + "typical", + _SCANNER_COUNT_MULTIPLE, + "noop", + ), + _Scenario("4MiB-max-one-noop", MAX_BODY_BYTES, "max", 1, "noop"), + _Scenario( + "4194303B-2097151-elements-wide-numeric-zero-one-noop", + _MAXIMUM_WIDTH_NUMERIC_BYTES, + "zero", + 1, + "noop", + "wide-numeric", + ), +) + + +def _build_json_body(target_bytes: int, block_count: int) -> bytes: + minimum_text_length = MAX_FINDINGS_PER_BLOCK + values = [_MARKER_CHARACTER * minimum_text_length for _ in range(block_count)] + encoded = ('{"blocks":["' + '","'.join(values) + '"]}').encode() + if len(encoded) > target_bytes: + raise ValueError("target body is too small for the requested finding load") + values[0] += _MARKER_CHARACTER * (target_bytes - len(encoded)) + body = ('{"blocks":["' + '","'.join(values) + '"]}').encode() + if len(body) != target_bytes: + raise AssertionError("benchmark body construction is not exact") + return body + + +def _build_maximum_width_numeric_body() -> bytes: + body = b"[" + (b"0," * (_MAXIMUM_WIDTH_NUMERIC_ELEMENT_COUNT - 1)) + b"0]" + if len(body) != _MAXIMUM_WIDTH_NUMERIC_BYTES: + raise AssertionError("maximum-width body construction is not exact") + return body + + +def _prepare_scenario(definition: _Scenario) -> _PreparedScenario: + if definition.finding_load == "zero": + total_findings_per_block = 0 + block_count = 1 + elif definition.finding_load == "typical": + total_findings_per_block = 8 + block_count = 1 + elif definition.finding_load == "max": + total_findings_per_block = MAX_FINDINGS_PER_BLOCK + block_count = MAX_FINDINGS_PER_REQUEST // MAX_FINDINGS_PER_BLOCK + else: + raise ValueError("unknown finding load") + + findings_per_scanner = total_findings_per_block // definition.scanner_count + if findings_per_scanner * definition.scanner_count != total_findings_per_block: + raise AssertionError("finding load must divide evenly across scanners") + scanners = [ + _BenchmarkScanner( + _BenchmarkScannerConfig( + name=f"benchmark-{index}", + entity_types=frozenset({"benchmark"}), + findings_per_block=findings_per_scanner, + ) + ) + for index in range(definition.scanner_count) + ] + action: ActionConfig = ( + RedactActionConfig(template="") + if definition.reconstruction == "replacement" + else ObserveActionConfig() + ) + policy = PolicyConfig(on_finding=action) + expected_findings = block_count * total_findings_per_block + if definition.body_shape == "text-blocks": + body = _build_json_body(definition.target_bytes, block_count) + elif definition.body_shape == "wide-numeric": + if block_count != 1 or expected_findings != 0: + raise AssertionError("wide numeric scenario must have zero findings") + body = _build_maximum_width_numeric_body() + else: + raise ValueError("unknown body shape") + request = InterceptedRequest(raw_body=body, policy_config=policy) + return _PreparedScenario( + definition=definition, + processor=RequestProcessor(scanners), + request=request, + expected_findings=expected_findings, + ) + + +def _run_and_verify(prepared: _PreparedScenario) -> None: + result = prepared.processor.process(prepared.request) + if result.decision is not ProcessingDecision.ALLOW: + raise AssertionError(f"{prepared.definition.name} unexpectedly denied") + if len(result.findings) != prepared.expected_findings: + raise AssertionError(f"{prepared.definition.name} finding count changed") + has_replacement = result.replacement_body is not None + if has_replacement != (prepared.definition.reconstruction == "replacement"): + raise AssertionError(f"{prepared.definition.name} reconstruction changed") + + +def _measure(prepared: _PreparedScenario, samples: int) -> _Measurement: + _run_and_verify(prepared) + wall_samples: list[float] = [] + for _ in range(samples): + started = time.perf_counter_ns() + _run_and_verify(prepared) + wall_samples.append((time.perf_counter_ns() - started) / 1_000_000) + + peak_samples: list[int] = [] + for _ in range(samples): + gc.collect() + tracemalloc.start() + try: + _run_and_verify(prepared) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + peak_samples.append(peak) + + return _Measurement( + definition=prepared.definition, + actual_bytes=len(prepared.request.raw_body), + finding_count=prepared.expected_findings, + median_wall_ms=statistics.median(wall_samples), + median_peak_bytes=int(statistics.median(peak_samples)), + ) + + +def _print_results(measurements: list[_Measurement], samples: int) -> None: + print(f"Python: {platform.python_version()} ({platform.platform()})") + print(f"Samples per median: {samples}") + print( + "| Scenario | Input bytes | Findings | Scanners | Reconstruction | " + "Median wall (ms) | Median peak traced allocation (bytes) |" + ) + print("| --- | ---: | ---: | ---: | --- | ---: | ---: |") + for measurement in measurements: + definition = measurement.definition + print( + f"| {definition.name} | {measurement.actual_bytes} | " + f"{measurement.finding_count} | {definition.scanner_count} | " + f"{definition.reconstruction} | {measurement.median_wall_ms:.3f} | " + f"{measurement.median_peak_bytes} |" + ) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--suite", choices=("quick", "full"), default="quick") + parser.add_argument("--samples", type=int) + parser.add_argument( + "--profile", + metavar="PATH", + help="write cProfile data for one verified pass through every scenario", + ) + return parser.parse_args() + + +def main() -> int: + arguments = _parse_args() + scenarios = _QUICK_SCENARIOS if arguments.suite == "quick" else _FULL_SCENARIOS + samples = ( + arguments.samples + if arguments.samples is not None + else (3 if arguments.suite == "quick" else 7) + ) + if samples < 3 or samples % 2 == 0: + raise SystemExit("--samples must be an odd integer of at least 3") + prepared = [_prepare_scenario(scenario) for scenario in scenarios] + if arguments.profile is not None: + profiler = cProfile.Profile() + profiler.enable() + for item in prepared: + _run_and_verify(item) + profiler.disable() + profiler.dump_stats(arguments.profile) + measurements = [_measure(item, samples) for item in prepared] + _print_results(measurements, samples) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/projects/privacy-guard/scripts/validate.sh b/projects/privacy-guard/scripts/validate.sh new file mode 100755 index 0000000..2eb9c60 --- /dev/null +++ b/projects/privacy-guard/scripts/validate.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +uv_run=(uv run --frozen) +if [[ $# -gt 0 ]]; then + if [[ $1 != "--python" || $# -ne 2 ]]; then + echo "usage: scripts/validate.sh [--python VERSION]" >&2 + exit 2 + fi + uv_run+=(--python "$2") +fi + +"${uv_run[@]}" pytest -q +"${uv_run[@]}" ruff format --check . +"${uv_run[@]}" ruff check . +"${uv_run[@]}" ty check +"${uv_run[@]}" python -c "import privacy_guard" +uv build diff --git a/projects/privacy-guard/src/privacy_guard/__init__.py b/projects/privacy-guard/src/privacy_guard/__init__.py new file mode 100644 index 0000000..da81cd3 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/__init__.py @@ -0,0 +1 @@ +"""Privacy Guard: an OpenShell supervisor middleware. See the package README.""" diff --git a/projects/privacy-guard/src/privacy_guard/bindings/__init__.py b/projects/privacy-guard/src/privacy_guard/bindings/__init__.py new file mode 100644 index 0000000..d168201 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/bindings/__init__.py @@ -0,0 +1 @@ +"""Generated OpenShell supervisor middleware bindings.""" diff --git a/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2.py b/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2.py new file mode 100644 index 0000000..c254b0f --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: supervisor_middleware.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'supervisor_middleware.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1bsupervisor_middleware.proto\x12\x17openshell.middleware.v1\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1cgoogle/protobuf/struct.proto\"y\n\x12MiddlewareManifest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x17\n\x0fservice_version\x18\x02 \x01(\t\x12<\n\x08\x62indings\x18\x03 \x03(\x0b\x32*.openshell.middleware.v1.MiddlewareBinding\"\xca\x01\n\x11MiddlewareBinding\x12I\n\toperation\x18\x01 \x01(\x0e\x32\x36.openshell.middleware.v1.SupervisorMiddlewareOperation\x12\x41\n\x05phase\x18\x02 \x01(\x0e\x32\x32.openshell.middleware.v1.SupervisorMiddlewarePhase\x12\x16\n\x0emax_body_bytes\x18\x03 \x01(\x04\x12\x0f\n\x07timeout\x18\x04 \x01(\t\"Y\n\x15ValidateConfigRequest\x12\'\n\x06\x63onfig\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x17\n\x0fmiddleware_name\x18\x02 \x01(\t\"7\n\x16ValidateConfigResponse\x12\r\n\x05valid\x18\x01 \x01(\x08\x12\x0e\n\x06reason\x18\x02 \x01(\t\"\xd6\x02\n\x15HttpRequestEvaluation\x12\x41\n\x05phase\x18\x01 \x01(\x0e\x32\x32.openshell.middleware.v1.SupervisorMiddlewarePhase\x12\x38\n\x07\x63ontext\x18\x02 \x01(\x0b\x32\'.openshell.middleware.v1.RequestContext\x12\'\n\x06\x63onfig\x18\x03 \x01(\x0b\x32\x17.google.protobuf.Struct\x12:\n\x06target\x18\x04 \x01(\x0b\x32*.openshell.middleware.v1.HttpRequestTarget\x12\x34\n\x07headers\x18\x05 \x03(\x0b\x32#.openshell.middleware.v1.HttpHeader\x12\x0c\n\x04\x62ody\x18\x06 \x01(\x0c\x12\x17\n\x0fmiddleware_name\x18\x07 \x01(\t\")\n\nHttpHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"w\n\x0eRequestContext\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x12\n\nsandbox_id\x18\x02 \x01(\t\x12=\n\x13originating_process\x18\x03 \x01(\x0b\x32 .openshell.middleware.v1.Process\"l\n\x11HttpRequestTarget\x12\x0e\n\x06scheme\x18\x01 \x01(\t\x12\x0c\n\x04host\x18\x02 \x01(\t\x12\x0c\n\x04port\x18\x03 \x01(\r\x12\x0e\n\x06method\x18\x04 \x01(\t\x12\x0c\n\x04path\x18\x05 \x01(\t\x12\r\n\x05query\x18\x06 \x01(\t\"9\n\x07Process\x12\x0e\n\x06\x62inary\x18\x01 \x01(\t\x12\x0b\n\x03pid\x18\x02 \x01(\r\x12\x11\n\tancestors\x18\x03 \x03(\t\"[\n\x07\x46inding\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12\r\n\x05\x63ount\x18\x03 \x01(\r\x12\x12\n\nconfidence\x18\x04 \x01(\t\x12\x10\n\x08severity\x18\x05 \x01(\t\"n\n\x0bWriteHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\x12\x42\n\x0bon_existing\x18\x03 \x01(\x0e\x32-.openshell.middleware.v1.ExistingHeaderAction\"\x1c\n\x0cRemoveHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x8d\x01\n\x0eHeaderMutation\x12\x35\n\x05write\x18\x01 \x01(\x0b\x32$.openshell.middleware.v1.WriteHeaderH\x00\x12\x37\n\x06remove\x18\x02 \x01(\x0b\x32%.openshell.middleware.v1.RemoveHeaderH\x00\x42\x0b\n\toperation\"\x81\x03\n\x11HttpRequestResult\x12\x33\n\x08\x64\x65\x63ision\x18\x01 \x01(\x0e\x32!.openshell.middleware.v1.Decision\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x0c\n\x04\x62ody\x18\x03 \x01(\x0c\x12\x10\n\x08has_body\x18\x04 \x01(\x08\x12\x41\n\x10header_mutations\x18\x05 \x03(\x0b\x32\'.openshell.middleware.v1.HeaderMutation\x12\x32\n\x08\x66indings\x18\x06 \x03(\x0b\x32 .openshell.middleware.v1.Finding\x12J\n\x08metadata\x18\x07 \x03(\x0b\x32\x38.openshell.middleware.v1.HttpRequestResult.MetadataEntry\x12\x13\n\x0breason_code\x18\x08 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01*\x82\x01\n\x1dSupervisorMiddlewareOperation\x12/\n+SUPERVISOR_MIDDLEWARE_OPERATION_UNSPECIFIED\x10\x00\x12\x30\n,SUPERVISOR_MIDDLEWARE_OPERATION_HTTP_REQUEST\x10\x01*y\n\x19SupervisorMiddlewarePhase\x12+\n\'SUPERVISOR_MIDDLEWARE_PHASE_UNSPECIFIED\x10\x00\x12/\n+SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS\x10\x01*K\n\x08\x44\x65\x63ision\x12\x18\n\x14\x44\x45\x43ISION_UNSPECIFIED\x10\x00\x12\x12\n\x0e\x44\x45\x43ISION_ALLOW\x10\x01\x12\x11\n\rDECISION_DENY\x10\x02*\xa8\x01\n\x14\x45xistingHeaderAction\x12&\n\"EXISTING_HEADER_ACTION_UNSPECIFIED\x10\x00\x12!\n\x1d\x45XISTING_HEADER_ACTION_APPEND\x10\x01\x12$\n EXISTING_HEADER_ACTION_OVERWRITE\x10\x02\x12\x1f\n\x1b\x45XISTING_HEADER_ACTION_SKIP\x10\x03\x32\xcd\x02\n\x14SupervisorMiddleware\x12O\n\x08\x44\x65scribe\x12\x16.google.protobuf.Empty\x1a+.openshell.middleware.v1.MiddlewareManifest\x12q\n\x0eValidateConfig\x12..openshell.middleware.v1.ValidateConfigRequest\x1a/.openshell.middleware.v1.ValidateConfigResponse\x12q\n\x13\x45valuateHttpRequest\x12..openshell.middleware.v1.HttpRequestEvaluation\x1a*.openshell.middleware.v1.HttpRequestResultb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'supervisor_middleware_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_HTTPREQUESTRESULT_METADATAENTRY']._loaded_options = None + _globals['_HTTPREQUESTRESULT_METADATAENTRY']._serialized_options = b'8\001' + _globals['_SUPERVISORMIDDLEWAREOPERATION']._serialized_start=2037 + _globals['_SUPERVISORMIDDLEWAREOPERATION']._serialized_end=2167 + _globals['_SUPERVISORMIDDLEWAREPHASE']._serialized_start=2169 + _globals['_SUPERVISORMIDDLEWAREPHASE']._serialized_end=2290 + _globals['_DECISION']._serialized_start=2292 + _globals['_DECISION']._serialized_end=2367 + _globals['_EXISTINGHEADERACTION']._serialized_start=2370 + _globals['_EXISTINGHEADERACTION']._serialized_end=2538 + _globals['_MIDDLEWAREMANIFEST']._serialized_start=115 + _globals['_MIDDLEWAREMANIFEST']._serialized_end=236 + _globals['_MIDDLEWAREBINDING']._serialized_start=239 + _globals['_MIDDLEWAREBINDING']._serialized_end=441 + _globals['_VALIDATECONFIGREQUEST']._serialized_start=443 + _globals['_VALIDATECONFIGREQUEST']._serialized_end=532 + _globals['_VALIDATECONFIGRESPONSE']._serialized_start=534 + _globals['_VALIDATECONFIGRESPONSE']._serialized_end=589 + _globals['_HTTPREQUESTEVALUATION']._serialized_start=592 + _globals['_HTTPREQUESTEVALUATION']._serialized_end=934 + _globals['_HTTPHEADER']._serialized_start=936 + _globals['_HTTPHEADER']._serialized_end=977 + _globals['_REQUESTCONTEXT']._serialized_start=979 + _globals['_REQUESTCONTEXT']._serialized_end=1098 + _globals['_HTTPREQUESTTARGET']._serialized_start=1100 + _globals['_HTTPREQUESTTARGET']._serialized_end=1208 + _globals['_PROCESS']._serialized_start=1210 + _globals['_PROCESS']._serialized_end=1267 + _globals['_FINDING']._serialized_start=1269 + _globals['_FINDING']._serialized_end=1360 + _globals['_WRITEHEADER']._serialized_start=1362 + _globals['_WRITEHEADER']._serialized_end=1472 + _globals['_REMOVEHEADER']._serialized_start=1474 + _globals['_REMOVEHEADER']._serialized_end=1502 + _globals['_HEADERMUTATION']._serialized_start=1505 + _globals['_HEADERMUTATION']._serialized_end=1646 + _globals['_HTTPREQUESTRESULT']._serialized_start=1649 + _globals['_HTTPREQUESTRESULT']._serialized_end=2034 + _globals['_HTTPREQUESTRESULT_METADATAENTRY']._serialized_start=1987 + _globals['_HTTPREQUESTRESULT_METADATAENTRY']._serialized_end=2034 + _globals['_SUPERVISORMIDDLEWARE']._serialized_start=2541 + _globals['_SUPERVISORMIDDLEWARE']._serialized_end=2874 +# @@protoc_insertion_point(module_scope) diff --git a/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2.pyi b/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2.pyi new file mode 100644 index 0000000..10eac7f --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2.pyi @@ -0,0 +1,209 @@ +from google.protobuf import empty_pb2 as _empty_pb2 +from google.protobuf import struct_pb2 as _struct_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Iterable as _Iterable, Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class SupervisorMiddlewareOperation(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + SUPERVISOR_MIDDLEWARE_OPERATION_UNSPECIFIED: _ClassVar[SupervisorMiddlewareOperation] + SUPERVISOR_MIDDLEWARE_OPERATION_HTTP_REQUEST: _ClassVar[SupervisorMiddlewareOperation] + +class SupervisorMiddlewarePhase(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + SUPERVISOR_MIDDLEWARE_PHASE_UNSPECIFIED: _ClassVar[SupervisorMiddlewarePhase] + SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS: _ClassVar[SupervisorMiddlewarePhase] + +class Decision(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + DECISION_UNSPECIFIED: _ClassVar[Decision] + DECISION_ALLOW: _ClassVar[Decision] + DECISION_DENY: _ClassVar[Decision] + +class ExistingHeaderAction(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + EXISTING_HEADER_ACTION_UNSPECIFIED: _ClassVar[ExistingHeaderAction] + EXISTING_HEADER_ACTION_APPEND: _ClassVar[ExistingHeaderAction] + EXISTING_HEADER_ACTION_OVERWRITE: _ClassVar[ExistingHeaderAction] + EXISTING_HEADER_ACTION_SKIP: _ClassVar[ExistingHeaderAction] +SUPERVISOR_MIDDLEWARE_OPERATION_UNSPECIFIED: SupervisorMiddlewareOperation +SUPERVISOR_MIDDLEWARE_OPERATION_HTTP_REQUEST: SupervisorMiddlewareOperation +SUPERVISOR_MIDDLEWARE_PHASE_UNSPECIFIED: SupervisorMiddlewarePhase +SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS: SupervisorMiddlewarePhase +DECISION_UNSPECIFIED: Decision +DECISION_ALLOW: Decision +DECISION_DENY: Decision +EXISTING_HEADER_ACTION_UNSPECIFIED: ExistingHeaderAction +EXISTING_HEADER_ACTION_APPEND: ExistingHeaderAction +EXISTING_HEADER_ACTION_OVERWRITE: ExistingHeaderAction +EXISTING_HEADER_ACTION_SKIP: ExistingHeaderAction + +class MiddlewareManifest(_message.Message): + __slots__ = ("name", "service_version", "bindings") + NAME_FIELD_NUMBER: _ClassVar[int] + SERVICE_VERSION_FIELD_NUMBER: _ClassVar[int] + BINDINGS_FIELD_NUMBER: _ClassVar[int] + name: str + service_version: str + bindings: _containers.RepeatedCompositeFieldContainer[MiddlewareBinding] + def __init__(self, name: _Optional[str] = ..., service_version: _Optional[str] = ..., bindings: _Optional[_Iterable[_Union[MiddlewareBinding, _Mapping]]] = ...) -> None: ... + +class MiddlewareBinding(_message.Message): + __slots__ = ("operation", "phase", "max_body_bytes", "timeout") + OPERATION_FIELD_NUMBER: _ClassVar[int] + PHASE_FIELD_NUMBER: _ClassVar[int] + MAX_BODY_BYTES_FIELD_NUMBER: _ClassVar[int] + TIMEOUT_FIELD_NUMBER: _ClassVar[int] + operation: SupervisorMiddlewareOperation + phase: SupervisorMiddlewarePhase + max_body_bytes: int + timeout: str + def __init__(self, operation: _Optional[_Union[SupervisorMiddlewareOperation, str]] = ..., phase: _Optional[_Union[SupervisorMiddlewarePhase, str]] = ..., max_body_bytes: _Optional[int] = ..., timeout: _Optional[str] = ...) -> None: ... + +class ValidateConfigRequest(_message.Message): + __slots__ = ("config", "middleware_name") + CONFIG_FIELD_NUMBER: _ClassVar[int] + MIDDLEWARE_NAME_FIELD_NUMBER: _ClassVar[int] + config: _struct_pb2.Struct + middleware_name: str + def __init__(self, config: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., middleware_name: _Optional[str] = ...) -> None: ... + +class ValidateConfigResponse(_message.Message): + __slots__ = ("valid", "reason") + VALID_FIELD_NUMBER: _ClassVar[int] + REASON_FIELD_NUMBER: _ClassVar[int] + valid: bool + reason: str + def __init__(self, valid: _Optional[bool] = ..., reason: _Optional[str] = ...) -> None: ... + +class HttpRequestEvaluation(_message.Message): + __slots__ = ("phase", "context", "config", "target", "headers", "body", "middleware_name") + PHASE_FIELD_NUMBER: _ClassVar[int] + CONTEXT_FIELD_NUMBER: _ClassVar[int] + CONFIG_FIELD_NUMBER: _ClassVar[int] + TARGET_FIELD_NUMBER: _ClassVar[int] + HEADERS_FIELD_NUMBER: _ClassVar[int] + BODY_FIELD_NUMBER: _ClassVar[int] + MIDDLEWARE_NAME_FIELD_NUMBER: _ClassVar[int] + phase: SupervisorMiddlewarePhase + context: RequestContext + config: _struct_pb2.Struct + target: HttpRequestTarget + headers: _containers.RepeatedCompositeFieldContainer[HttpHeader] + body: bytes + middleware_name: str + def __init__(self, phase: _Optional[_Union[SupervisorMiddlewarePhase, str]] = ..., context: _Optional[_Union[RequestContext, _Mapping]] = ..., config: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., target: _Optional[_Union[HttpRequestTarget, _Mapping]] = ..., headers: _Optional[_Iterable[_Union[HttpHeader, _Mapping]]] = ..., body: _Optional[bytes] = ..., middleware_name: _Optional[str] = ...) -> None: ... + +class HttpHeader(_message.Message): + __slots__ = ("name", "value") + NAME_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + name: str + value: str + def __init__(self, name: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + +class RequestContext(_message.Message): + __slots__ = ("request_id", "sandbox_id", "originating_process") + REQUEST_ID_FIELD_NUMBER: _ClassVar[int] + SANDBOX_ID_FIELD_NUMBER: _ClassVar[int] + ORIGINATING_PROCESS_FIELD_NUMBER: _ClassVar[int] + request_id: str + sandbox_id: str + originating_process: Process + def __init__(self, request_id: _Optional[str] = ..., sandbox_id: _Optional[str] = ..., originating_process: _Optional[_Union[Process, _Mapping]] = ...) -> None: ... + +class HttpRequestTarget(_message.Message): + __slots__ = ("scheme", "host", "port", "method", "path", "query") + SCHEME_FIELD_NUMBER: _ClassVar[int] + HOST_FIELD_NUMBER: _ClassVar[int] + PORT_FIELD_NUMBER: _ClassVar[int] + METHOD_FIELD_NUMBER: _ClassVar[int] + PATH_FIELD_NUMBER: _ClassVar[int] + QUERY_FIELD_NUMBER: _ClassVar[int] + scheme: str + host: str + port: int + method: str + path: str + query: str + def __init__(self, scheme: _Optional[str] = ..., host: _Optional[str] = ..., port: _Optional[int] = ..., method: _Optional[str] = ..., path: _Optional[str] = ..., query: _Optional[str] = ...) -> None: ... + +class Process(_message.Message): + __slots__ = ("binary", "pid", "ancestors") + BINARY_FIELD_NUMBER: _ClassVar[int] + PID_FIELD_NUMBER: _ClassVar[int] + ANCESTORS_FIELD_NUMBER: _ClassVar[int] + binary: str + pid: int + ancestors: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, binary: _Optional[str] = ..., pid: _Optional[int] = ..., ancestors: _Optional[_Iterable[str]] = ...) -> None: ... + +class Finding(_message.Message): + __slots__ = ("type", "label", "count", "confidence", "severity") + TYPE_FIELD_NUMBER: _ClassVar[int] + LABEL_FIELD_NUMBER: _ClassVar[int] + COUNT_FIELD_NUMBER: _ClassVar[int] + CONFIDENCE_FIELD_NUMBER: _ClassVar[int] + SEVERITY_FIELD_NUMBER: _ClassVar[int] + type: str + label: str + count: int + confidence: str + severity: str + def __init__(self, type: _Optional[str] = ..., label: _Optional[str] = ..., count: _Optional[int] = ..., confidence: _Optional[str] = ..., severity: _Optional[str] = ...) -> None: ... + +class WriteHeader(_message.Message): + __slots__ = ("name", "value", "on_existing") + NAME_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + ON_EXISTING_FIELD_NUMBER: _ClassVar[int] + name: str + value: str + on_existing: ExistingHeaderAction + def __init__(self, name: _Optional[str] = ..., value: _Optional[str] = ..., on_existing: _Optional[_Union[ExistingHeaderAction, str]] = ...) -> None: ... + +class RemoveHeader(_message.Message): + __slots__ = ("name",) + NAME_FIELD_NUMBER: _ClassVar[int] + name: str + def __init__(self, name: _Optional[str] = ...) -> None: ... + +class HeaderMutation(_message.Message): + __slots__ = ("write", "remove") + WRITE_FIELD_NUMBER: _ClassVar[int] + REMOVE_FIELD_NUMBER: _ClassVar[int] + write: WriteHeader + remove: RemoveHeader + def __init__(self, write: _Optional[_Union[WriteHeader, _Mapping]] = ..., remove: _Optional[_Union[RemoveHeader, _Mapping]] = ...) -> None: ... + +class HttpRequestResult(_message.Message): + __slots__ = ("decision", "reason", "body", "has_body", "header_mutations", "findings", "metadata", "reason_code") + class MetadataEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + DECISION_FIELD_NUMBER: _ClassVar[int] + REASON_FIELD_NUMBER: _ClassVar[int] + BODY_FIELD_NUMBER: _ClassVar[int] + HAS_BODY_FIELD_NUMBER: _ClassVar[int] + HEADER_MUTATIONS_FIELD_NUMBER: _ClassVar[int] + FINDINGS_FIELD_NUMBER: _ClassVar[int] + METADATA_FIELD_NUMBER: _ClassVar[int] + REASON_CODE_FIELD_NUMBER: _ClassVar[int] + decision: Decision + reason: str + body: bytes + has_body: bool + header_mutations: _containers.RepeatedCompositeFieldContainer[HeaderMutation] + findings: _containers.RepeatedCompositeFieldContainer[Finding] + metadata: _containers.ScalarMap[str, str] + reason_code: str + def __init__(self, decision: _Optional[_Union[Decision, str]] = ..., reason: _Optional[str] = ..., body: _Optional[bytes] = ..., has_body: _Optional[bool] = ..., header_mutations: _Optional[_Iterable[_Union[HeaderMutation, _Mapping]]] = ..., findings: _Optional[_Iterable[_Union[Finding, _Mapping]]] = ..., metadata: _Optional[_Mapping[str, str]] = ..., reason_code: _Optional[str] = ...) -> None: ... diff --git a/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2_grpc.py b/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2_grpc.py new file mode 100644 index 0000000..3800987 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2_grpc.py @@ -0,0 +1,210 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" + +import grpc +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 + +from . import supervisor_middleware_pb2 as supervisor__middleware__pb2 + +GRPC_GENERATED_VERSION = "1.81.1" +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + + _version_not_supported = first_version_is_lower( + GRPC_VERSION, GRPC_GENERATED_VERSION + ) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f"The grpc package installed is at version {GRPC_VERSION}," + + " but the generated code in supervisor_middleware_pb2_grpc.py depends on" + + f" grpcio>={GRPC_GENERATED_VERSION}." + + f" Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}" + + f" or downgrade your generated code using grpcio-tools<={GRPC_VERSION}." + ) + + +class SupervisorMiddlewareStub: + """SupervisorMiddleware lets an operator-run service inspect and transform + sandbox HTTP egress before OpenShell injects credentials. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.Describe = channel.unary_unary( + "/openshell.middleware.v1.SupervisorMiddleware/Describe", + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=supervisor__middleware__pb2.MiddlewareManifest.FromString, + _registered_method=True, + ) + self.ValidateConfig = channel.unary_unary( + "/openshell.middleware.v1.SupervisorMiddleware/ValidateConfig", + request_serializer=supervisor__middleware__pb2.ValidateConfigRequest.SerializeToString, + response_deserializer=supervisor__middleware__pb2.ValidateConfigResponse.FromString, + _registered_method=True, + ) + self.EvaluateHttpRequest = channel.unary_unary( + "/openshell.middleware.v1.SupervisorMiddleware/EvaluateHttpRequest", + request_serializer=supervisor__middleware__pb2.HttpRequestEvaluation.SerializeToString, + response_deserializer=supervisor__middleware__pb2.HttpRequestResult.FromString, + _registered_method=True, + ) + + +class SupervisorMiddlewareServicer: + """SupervisorMiddleware lets an operator-run service inspect and transform + sandbox HTTP egress before OpenShell injects credentials. + """ + + def Describe(self, request, context): + """Describe returns the service manifest and declared bindings.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def ValidateConfig(self, request, context): + """ValidateConfig checks service-specific configuration for one binding.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def EvaluateHttpRequest(self, request, context): + """EvaluateHttpRequest returns an allow, deny, or mutation decision for one + buffered HTTP request. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + +def add_SupervisorMiddlewareServicer_to_server(servicer, server): + rpc_method_handlers = { + "Describe": grpc.unary_unary_rpc_method_handler( + servicer.Describe, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=supervisor__middleware__pb2.MiddlewareManifest.SerializeToString, + ), + "ValidateConfig": grpc.unary_unary_rpc_method_handler( + servicer.ValidateConfig, + request_deserializer=supervisor__middleware__pb2.ValidateConfigRequest.FromString, + response_serializer=supervisor__middleware__pb2.ValidateConfigResponse.SerializeToString, + ), + "EvaluateHttpRequest": grpc.unary_unary_rpc_method_handler( + servicer.EvaluateHttpRequest, + request_deserializer=supervisor__middleware__pb2.HttpRequestEvaluation.FromString, + response_serializer=supervisor__middleware__pb2.HttpRequestResult.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + "openshell.middleware.v1.SupervisorMiddleware", rpc_method_handlers + ) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers( + "openshell.middleware.v1.SupervisorMiddleware", rpc_method_handlers + ) + + +# This class is part of an EXPERIMENTAL API. +class SupervisorMiddleware: + """SupervisorMiddleware lets an operator-run service inspect and transform + sandbox HTTP egress before OpenShell injects credentials. + """ + + @staticmethod + def Describe( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/openshell.middleware.v1.SupervisorMiddleware/Describe", + google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + supervisor__middleware__pb2.MiddlewareManifest.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True, + ) + + @staticmethod + def ValidateConfig( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/openshell.middleware.v1.SupervisorMiddleware/ValidateConfig", + supervisor__middleware__pb2.ValidateConfigRequest.SerializeToString, + supervisor__middleware__pb2.ValidateConfigResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True, + ) + + @staticmethod + def EvaluateHttpRequest( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/openshell.middleware.v1.SupervisorMiddleware/EvaluateHttpRequest", + supervisor__middleware__pb2.HttpRequestEvaluation.SerializeToString, + supervisor__middleware__pb2.HttpRequestResult.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True, + ) diff --git a/projects/privacy-guard/src/privacy_guard/body/__init__.py b/projects/privacy-guard/src/privacy_guard/body/__init__.py new file mode 100644 index 0000000..6611f95 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/body/__init__.py @@ -0,0 +1,45 @@ +"""Format-handler registry and format-agnostic request-body records.""" + +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType + +from privacy_guard.body.base import ( + FormatHandler, + FormatHandlerContractError, + RequestBody, + TextBlock, + parse_normalized_body, +) +from privacy_guard.body.json import JsonHandler +from privacy_guard.errors import ErrorCode, PrivacyGuardError + +DEFAULT_FORMAT_HANDLERS: Mapping[str, FormatHandler] = MappingProxyType( + {handler.format_name: handler for handler in (JsonHandler(),)} +) +"""Built-in handlers keyed by name; processors may receive a custom mapping.""" + + +def select_handler(body_format: str) -> FormatHandler: + """Return the registered FormatHandler for ``body_format`` (e.g. "json"). + + ``body_format`` comes from ``PolicyConfig.body_format``. Raise for an + unregistered format rather than silently falling back. + """ + try: + return DEFAULT_FORMAT_HANDLERS[body_format] + except KeyError: + raise PrivacyGuardError(ErrorCode.BODY_FORMAT_UNSUPPORTED) from None + + +__all__ = [ + "DEFAULT_FORMAT_HANDLERS", + "FormatHandler", + "FormatHandlerContractError", + "JsonHandler", + "RequestBody", + "TextBlock", + "parse_normalized_body", + "select_handler", +] diff --git a/projects/privacy-guard/src/privacy_guard/body/base.py b/projects/privacy-guard/src/privacy_guard/body/base.py new file mode 100644 index 0000000..2453cf0 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/body/base.py @@ -0,0 +1,117 @@ +"""Strict request-body models and the nominal format-handler contract.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Mapping + +from pydantic import Field + +from privacy_guard.config import PolicyConfig +from privacy_guard.validation import ( + ScalarString, + StrictSensitiveModel, + parse_non_empty_scalar_string, +) + + +class TextBlock(StrictSensitiveModel): + """One scannable text block, addressed structurally within the body. + + ``path`` is an opaque, format-owned address that may contain sensitive + structural keys. Callers may use it as a replacement key but must not parse + it. ``replaceable=False`` exposes observable text, such as a JSON object key, + that may be scanned but must not be rewritten. + """ + + path: ScalarString = Field(repr=False) + text: ScalarString = Field(repr=False) + replaceable: bool = True + + +class RequestBody(StrictSensitiveModel): + """A normalized body plus handler-owned reconstruction state. + + ``text_blocks`` are the independently scannable values selected by the + handler. ``parsed_value`` is opaque handler-owned state and reconstruction + must not mutate it. ``original_bytes`` contains bytes equal to the exact input; + a handler returns the stored value for a no-op reconstruction. Rewritten bodies + need only preserve untouched values semantically. + """ + + text_blocks: tuple[TextBlock, ...] + parsed_value: object = Field(repr=False) + original_bytes: bytes = Field(repr=False) + + +class FormatHandlerContractError(Exception): + """A content-safe format-handler metadata or output contract failure.""" + + +class FormatHandler(ABC): + """Nominal, concurrency-safe extension point for one request-body format. + + Processor registries reuse handler instances across requests. Implementations + must therefore retain no request content or mutable per-request state. + """ + + def __init__(self, *, format_name: str) -> None: + try: + self.__format_name = parse_non_empty_scalar_string(format_name) + except ValueError: + raise FormatHandlerContractError( + "format-handler metadata is invalid" + ) from None + + @property + def format_name(self) -> str: + """Return the immutable construction-time format identity.""" + return self.__format_name + + def normalize(self, raw_body: bytes, policy_config: PolicyConfig) -> RequestBody: + """Parse request bytes and validate the implementation's body model.""" + result: object = self._normalize(raw_body, policy_config) + return parse_normalized_body(result) + + def reconstruct( + self, + request_body: RequestBody, + replacements_by_path: Mapping[str, str], + ) -> bytes: + """Rebuild a normalized body and require a bytes result.""" + result: object = self._reconstruct(request_body, replacements_by_path) + if not isinstance(result, bytes): + raise FormatHandlerContractError( + "format-handler output is invalid" + ) from None + return result + + @abstractmethod + def _normalize(self, raw_body: bytes, policy_config: PolicyConfig) -> RequestBody: + """Implement format-specific parsing without retaining request state.""" + raise NotImplementedError + + @abstractmethod + def _reconstruct( + self, + request_body: RequestBody, + replacements_by_path: Mapping[str, str], + ) -> bytes: + """Implement format-specific reconstruction without mutating the model.""" + raise NotImplementedError + + +def parse_normalized_body(result: object) -> RequestBody: + """Require handlers to return the documented body model.""" + if not isinstance(result, RequestBody): + raise FormatHandlerContractError("format-handler output is invalid") from None + return result + + +__all__ = [ + "FormatHandler", + "FormatHandlerContractError", + "RequestBody", + "TextBlock", + "parse_normalized_body", +] diff --git a/projects/privacy-guard/src/privacy_guard/body/json.py b/projects/privacy-guard/src/privacy_guard/body/json.py new file mode 100644 index 0000000..1399c88 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/body/json.py @@ -0,0 +1,300 @@ +"""JSON FormatHandler that addresses string leaves by JSON Pointer path.""" + +from __future__ import annotations + +import json +import math +from collections.abc import Iterator, Mapping +from copy import deepcopy +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from pydantic import ConfigDict, TypeAdapter, ValidationError +from typing_extensions import TypeAliasType, override + +from privacy_guard.body.base import FormatHandler, RequestBody, TextBlock +from privacy_guard.constants import ( + MAX_JSON_NESTING, + MAX_SCANNED_CHARACTERS, + MAX_TEXT_BLOCKS, +) +from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.validation import ScalarString, parse_scalar_string + +if TYPE_CHECKING: + from privacy_guard.config import PolicyConfig + + +JsonValue = TypeAliasType( + "JsonValue", + ScalarString + | int + | float + | bool + | None + | dict[ScalarString, "JsonValue"] + | list["JsonValue"], +) + + +class JsonHandler(FormatHandler): + """Handle strict UTF-8 JSON bodies, with one TextBlock per string leaf. + + Text-block paths are JSON Pointers (RFC 6901), e.g. ``/items/0/text``, so + ``reconstruct`` can locate each replacement. Rewritten JSON preserves + untouched values semantically but may change serialization details. + """ + + def __init__(self) -> None: + super().__init__(format_name="json") + + @override + def _normalize(self, raw_body: bytes, policy_config: PolicyConfig) -> RequestBody: + """Decode UTF-8, parse JSON, and collect every string leaf as a text block.""" + try: + decoded_body = raw_body.decode("utf-8") + except UnicodeDecodeError: + raise PrivacyGuardError(ErrorCode.BODY_ENCODING_INVALID) from None + + try: + parsed_value = self._parse_json_value(decoded_body) + except _InvalidJsonError: + raise PrivacyGuardError(ErrorCode.BODY_JSON_INVALID) from None + except _InvalidJsonShapeError: + raise PrivacyGuardError(ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED) from None + + try: + text_blocks = tuple(self._iter_text_blocks(parsed_value)) + except _InvalidJsonError: + raise PrivacyGuardError(ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED) from None + return RequestBody( + text_blocks=text_blocks, + parsed_value=_JsonBodyState(parsed_value), + original_bytes=raw_body, + ) + + @override + def _reconstruct( + self, + request_body: RequestBody, + replacements_by_path: Mapping[str, str], + ) -> bytes: + """Apply JSON Pointer replacements, or return exact bytes for a no-op.""" + parsed_state = request_body.parsed_value + if not isinstance(parsed_state, _JsonBodyState): + raise PrivacyGuardError(ErrorCode.BODY_RECONSTRUCTION_INVALID) from None + if not replacements_by_path: + return request_body.original_bytes + + try: + reconstructed_value = deepcopy(parsed_state.value) + for json_pointer, replacement_text in replacements_by_path.items(): + parse_scalar_string(replacement_text) + + path_tokens = self._decode_json_pointer(json_pointer) + if not path_tokens: + if not isinstance(reconstructed_value, str): + raise _InvalidJsonError + reconstructed_value = replacement_text + continue + + parent_node = reconstructed_value + for path_token in path_tokens[:-1]: + parent_node = self._resolve_child_node(parent_node, path_token) + + final_token = path_tokens[-1] + target_value = self._resolve_child_node(parent_node, final_token) + if not isinstance(target_value, str): + raise _InvalidJsonError + + if isinstance(parent_node, dict): + parent_node[final_token] = replacement_text + elif isinstance(parent_node, list): + array_index = self._parse_array_index(final_token, len(parent_node)) + parent_node[array_index] = replacement_text + else: + raise _InvalidJsonError + except Exception: + raise PrivacyGuardError(ErrorCode.BODY_RECONSTRUCTION_INVALID) from None + + try: + return json.dumps( + reconstructed_value, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + except Exception: + raise PrivacyGuardError(ErrorCode.BODY_RECONSTRUCTION_INVALID) from None + + @staticmethod + def _build_unique_object( + object_pairs: list[tuple[str, object]], + ) -> dict[str, object]: + object_members: dict[str, object] = {} + for member_name, member_value in object_pairs: + if member_name in object_members: + raise _InvalidJsonError + object_members[member_name] = member_value + return object_members + + @staticmethod + def _reject_non_finite_constant(_: str) -> object: + raise _InvalidJsonError + + @staticmethod + def _parse_finite_float(value: str) -> float: + parsed = float(value) + if not math.isfinite(parsed): + raise _InvalidJsonError + return parsed + + @classmethod + def _parse_json_value(cls, decoded_body: str) -> JsonValue: + """Parse duplicate-aware JSON and return its one validated typed tree.""" + try: + raw_value = json.loads( + decoded_body, + object_pairs_hook=cls._build_unique_object, + parse_constant=cls._reject_non_finite_constant, + parse_float=cls._parse_finite_float, + ) + except RecursionError: + raise _InvalidJsonShapeError from None + except (json.JSONDecodeError, _InvalidJsonError, ValueError): + raise _InvalidJsonError from None + + try: + return _JSON_VALUE_ADAPTER.validate_python(raw_value, strict=True) + except ValidationError as validation_error: + if any( + error["type"] == "recursion_loop" + for error in validation_error.errors( + include_url=False, + include_context=False, + include_input=False, + ) + ): + raise _InvalidJsonShapeError from None + raise _InvalidJsonError from None + + @classmethod + def _iter_text_blocks(cls, parsed_json: JsonValue) -> Iterator[TextBlock]: + """Walk a parsed JSON value incrementally in stable depth-first order.""" + count = 0 + total_characters = 0 + for emitted in cls._walk_text_blocks(parsed_json, "", 0, True): + # Bound built-in parsing before tuple/model materialization. The + # processor repeats aggregates for arbitrary handler outputs. + count += 1 + total_characters += len(emitted.text) + if count > MAX_TEXT_BLOCKS or total_characters > MAX_SCANNED_CHARACTERS: + raise _InvalidJsonError + yield emitted + + @classmethod + def _walk_text_blocks( + cls, + node: JsonValue, + json_pointer: str, + depth: int, + replaceable: bool, + ) -> Iterator[TextBlock]: + """Yield one branch at a time so traversal state remains O(depth).""" + if depth > MAX_JSON_NESTING: + raise _InvalidJsonError + if isinstance(node, str): + yield TextBlock(path=json_pointer, text=node, replaceable=replaceable) + return + if isinstance(node, dict): + for key, child_node in node.items(): + path_token = key.replace("~", "~0").replace("/", "~1") + child_path = f"{json_pointer}/{path_token}" + # Key paths are deliberately outside the JSON Pointer namespace. + yield from cls._walk_text_blocks( + key, f"#key:{child_path}", depth, False + ) + yield from cls._walk_text_blocks( + child_node, child_path, depth + 1, True + ) + return + if isinstance(node, list): + for index, child_node in enumerate(node): + yield from cls._walk_text_blocks( + child_node, f"{json_pointer}/{index}", depth + 1, True + ) + + @staticmethod + def _decode_json_pointer(json_pointer: str) -> list[str]: + if json_pointer == "": + return [] + if not json_pointer.startswith("/"): + raise _InvalidJsonError + + decoded_tokens: list[str] = [] + for encoded_token in json_pointer[1:].split("/"): + decoded_characters: list[str] = [] + character_index = 0 + while character_index < len(encoded_token): + character = encoded_token[character_index] + if character != "~": + decoded_characters.append(character) + character_index += 1 + continue + if character_index + 1 >= len(encoded_token) or encoded_token[ + character_index + 1 + ] not in {"0", "1"}: + raise _InvalidJsonError + decoded_characters.append( + "~" if encoded_token[character_index + 1] == "0" else "/" + ) + character_index += 2 + decoded_tokens.append("".join(decoded_characters)) + return decoded_tokens + + @classmethod + def _resolve_child_node(cls, parent_node: JsonValue, path_token: str) -> JsonValue: + if isinstance(parent_node, dict): + if path_token not in parent_node: + raise _InvalidJsonError + return parent_node[path_token] + if isinstance(parent_node, list): + array_index = cls._parse_array_index(path_token, len(parent_node)) + return parent_node[array_index] + raise _InvalidJsonError + + @staticmethod + def _parse_array_index(path_token: str, array_length: int) -> int: + if not path_token.isascii() or not path_token.isdecimal(): + raise _InvalidJsonError + if len(path_token) > 1 and path_token.startswith("0"): + raise _InvalidJsonError + array_index = int(path_token) + if array_index >= array_length: + raise _InvalidJsonError + return array_index + + +_JSON_VALUE_ADAPTER = TypeAdapter( + JsonValue, + config=ConfigDict( + strict=True, + allow_inf_nan=False, + hide_input_in_errors=True, + ), +) + + +@dataclass(frozen=True) +class _JsonBodyState: + """Typed reconstruction state assembled from an already validated value.""" + + value: JsonValue = field(repr=False) + + +class _InvalidJsonError(ValueError): + """Signal a strict-JSON violation without retaining request content.""" + + +class _InvalidJsonShapeError(ValueError): + """Signal JSON nesting that exceeds a parser or validator safe limit.""" diff --git a/projects/privacy-guard/src/privacy_guard/config.py b/projects/privacy-guard/src/privacy_guard/config.py new file mode 100644 index 0000000..b032b02 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/config.py @@ -0,0 +1,135 @@ +"""Strict middleware policy configuration at the untrusted config boundary.""" + +from __future__ import annotations + +from collections.abc import Mapping +from enum import StrEnum +from string import Formatter +from typing import Literal, Self, TypeAlias + +from pydantic import Field, field_validator + +from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.scanners import Confidence +from privacy_guard.validation import ( + BoundedMetadataString, + NonEmptyScalarString, + ScalarString, + StrictSensitiveModel, + parse_scalar_string, +) + + +class PolicyAction(StrEnum): + """Supported actions for scanner findings.""" + + OBSERVE = "observe" + REDACT = "redact" + BLOCK = "block" + + +class ActionConfig(StrictSensitiveModel): + """Finding criteria shared by every policy action.""" + + action: PolicyAction + entity_types: frozenset[BoundedMetadataString] | None = Field( + default=None, repr=False + ) + minimum_confidence: Confidence | None = None + + +class ObserveActionConfig(ActionConfig): + """Observe selected findings without changing or blocking the request.""" + + action: Literal[PolicyAction.OBSERVE] = PolicyAction.OBSERVE + + +class BlockActionConfig(ActionConfig): + """Block the request when at least one selected finding is present.""" + + action: Literal[PolicyAction.BLOCK] = PolicyAction.BLOCK + + +class RedactActionConfig(ActionConfig): + """Replace selected finding spans using a bounded text template.""" + + action: Literal[PolicyAction.REDACT] = PolicyAction.REDACT + template: ScalarString = Field(default="[{entity}]", repr=False) + + @field_validator("template") + @classmethod + def _validate_template(cls, value: str) -> str: + """Allow static text and the finding entity, but no formatting features.""" + try: + parsed_fields = Formatter().parse(value) + for _, field_name, format_spec, conversion in parsed_fields: + if field_name is not None and field_name != "entity": + raise ValueError("template field is unsupported") + if format_spec or conversion is not None: + raise ValueError("template formatting options are unsupported") + except ValueError: + raise ValueError("redaction template syntax is invalid") from None + return value + + +PolicyActionConfig: TypeAlias = ( + ObserveActionConfig | BlockActionConfig | RedactActionConfig +) + + +class PolicyConfig(StrictSensitiveModel): + """Strict, immutable policy parsed from the supervisor's request config.""" + + body_format: NonEmptyScalarString = Field(default="json", repr=False) + on_finding: PolicyActionConfig = Field( + default_factory=RedactActionConfig, + discriminator="action", + repr=False, + ) + + @classmethod + def from_mapping(cls, values: object) -> Self: + """Parse untrusted values while discarding Pydantic error content.""" + try: + if not isinstance(values, Mapping): + raise TypeError("configuration input must be a mapping") + prepared = dict(values) + if "on_finding" in prepared: + prepared["on_finding"] = _prepare_action_config(prepared["on_finding"]) + return cls.model_validate(prepared) + except (TypeError, ValueError): + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None + + +def _parse_policy_action(value: object) -> PolicyAction: + return PolicyAction(parse_scalar_string(value)) + + +def _parse_confidence(value: object) -> Confidence: + return Confidence(parse_scalar_string(value)) + + +def _prepare_action_config(value: object) -> dict[str, object]: + if not isinstance(value, Mapping): + raise ValueError("action configuration must be a mapping") + prepared = dict(value) + if "action" in prepared: + prepared["action"] = _parse_policy_action(prepared["action"]) + if prepared.get("minimum_confidence") is not None: + prepared["minimum_confidence"] = _parse_confidence( + prepared["minimum_confidence"] + ) + if "entity_types" in prepared: + prepared["entity_types"] = _parse_entity_type_list(prepared["entity_types"]) + return prepared + + +def _parse_entity_type_list(value: object) -> frozenset[str] | None: + if value is None: + return None + if not isinstance(value, list): + raise ValueError("entity types must be a list of strings or null") + parsed = tuple(parse_scalar_string(item) for item in value) + if len(set(parsed)) != len(parsed): + raise ValueError("entity types must be unique") + return frozenset(parsed) diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py new file mode 100644 index 0000000..1bccc98 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -0,0 +1,46 @@ +"""Package-wide Privacy Guard constants and operational limits. + +Keep this module dependency-free within the package: it must not import from +``privacy_guard``. +""" + +from __future__ import annotations + +import re +from importlib.metadata import version + +# Middleware identity and stable response values. +SERVICE_NAME = "privacy-guard" +SERVICE_VERSION = version("privacy-guard") +BLOCK_REASON = "Privacy Guard blocked the request" +BLOCK_REASON_CODE = "privacy_guard_blocked" +LIMIT_REASON = "Privacy Guard denied a result that exceeded a safety limit" +LIMIT_REASON_CODE = "privacy_guard_limit_exceeded" + +# Request-body and JSON shape limits. +MAX_BODY_BYTES = 4 * 1024 * 1024 +MAX_JSON_NESTING = 64 +MAX_TEXT_BLOCKS = 4096 +MAX_SCANNED_CHARACTERS = 4 * 1024 * 1024 + +# Scanner and result limits. +MAX_FINDINGS_PER_BLOCK = 256 +MAX_FINDINGS_PER_REQUEST = 4096 +MAX_SCANNER_METADATA_BYTES = 1024 +MAX_PROTO_FINDING_GROUPS = 32 +MAX_PROTO_FINDING_BYTES = 4 * 1024 + +# Service concurrency and transport limits. +MAX_CONCURRENT_SCANS = 4 +MAX_CONCURRENT_RPCS = 16 +PROTOBUF_ENVELOPE_ALLOWANCE_BYTES = 1024 * 1024 +MAX_RECEIVE_MESSAGE_BYTES = MAX_BODY_BYTES + PROTOBUF_ENVELOPE_ALLOWANCE_BYTES + +# Protocol validation values. +UINT32_MAX = 2**32 - 1 +REASON_CODE_PATTERN = re.compile(r"[a-z][a-z0-9_]{0,63}\Z") +CONFIDENCE_RANK = { + "low": 0, + "medium": 1, + "high": 2, +} diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py new file mode 100644 index 0000000..561641c --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -0,0 +1,213 @@ +"""Content-safe failures shared across Privacy Guard trust boundaries.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + +from typing_extensions import override + + +class ErrorKind(StrEnum): + """Whether a failure is attributable to input or middleware internals.""" + + INVALID_INPUT = "invalid_input" + INTERNAL = "internal" + + +class ErrorComponent(StrEnum): + """The Privacy Guard component responsible for a failure.""" + + CONFIG = "config" + FORMAT_HANDLER = "format_handler" + SCANNER = "scanner" + PROCESSOR = "processor" + SERVICE = "service" + SERVER = "server" + + +class ErrorCode(StrEnum): + """Stable identifiers for cataloged production failures.""" + + CONFIG_INVALID = "config_invalid" + REQUEST_PHASE_INVALID = "request_phase_invalid" + REQUEST_BODY_TOO_LARGE = "request_body_too_large" + REQUEST_SHAPE_LIMIT_EXCEEDED = "request_shape_limit_exceeded" + BODY_FORMAT_UNSUPPORTED = "body_format_unsupported" + BODY_ENCODING_INVALID = "body_encoding_invalid" + BODY_JSON_INVALID = "body_json_invalid" + BODY_RECONSTRUCTION_INVALID = "body_reconstruction_invalid" + FORMAT_HANDLER_OUTPUT_INVALID = "format_handler_output_invalid" + FORMAT_HANDLER_EXECUTION_FAILED = "format_handler_execution_failed" + SCANNER_OUTPUT_INVALID = "scanner_output_invalid" + SCANNER_EXECUTION_FAILED = "scanner_execution_failed" + FINDING_LIMIT_EXCEEDED = "finding_limit_exceeded" + RESULT_LIMIT_EXCEEDED = "result_limit_exceeded" + SERVER_BIND_FAILED = "server_bind_failed" + UNEXPECTED_SERVICE_FAILURE = "unexpected_service_failure" + + +@dataclass(frozen=True) +class ErrorSpec: + """Immutable, developer-authored classification and remediation text.""" + + kind: ErrorKind + component: ErrorComponent + operation: str + summary: str + hint: str + + +class PrivacyGuardError(Exception): + """A catalog-only failure whose public representation is content-safe.""" + + def __init__(self, code: ErrorCode) -> None: + self.code = code + self._spec = _ERROR_SPECS[code] + super().__init__(str(self)) + + @property + def kind(self) -> ErrorKind: + return self._spec.kind + + @property + def component(self) -> ErrorComponent: + return self._spec.component + + @property + def operation(self) -> str: + return self._spec.operation + + @property + def summary(self) -> str: + return self._spec.summary + + @property + def hint(self) -> str: + return self._spec.hint + + @override + def __str__(self) -> str: + return ( + f"[{self.code.value}] {self.component.value}.{self.operation}: " + f"{self.summary} Hint: {self.hint}" + ) + + +_ERROR_SPECS: dict[ErrorCode, ErrorSpec] = { + ErrorCode.CONFIG_INVALID: ErrorSpec( + ErrorKind.INVALID_INPUT, + ErrorComponent.CONFIG, + "parse", + "Policy configuration is invalid.", + "Check allowed fields, strict string types, finding action, confidence, " + "entity filters, and redact template syntax.", + ), + ErrorCode.REQUEST_PHASE_INVALID: ErrorSpec( + ErrorKind.INVALID_INPUT, + ErrorComponent.SERVICE, + "validate_phase", + "Request evaluation phase is invalid.", + "Use the advertised pre-credentials phase.", + ), + ErrorCode.REQUEST_BODY_TOO_LARGE: ErrorSpec( + ErrorKind.INVALID_INPUT, + ErrorComponent.SERVICE, + "validate_body_size", + "Request body exceeds the advertised size limit.", + "Reduce the request body to the maximum size in the middleware manifest.", + ), + ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED: ErrorSpec( + ErrorKind.INVALID_INPUT, + ErrorComponent.FORMAT_HANDLER, + "normalize", + "Request body exceeds a structural scanning limit.", + "Reduce JSON nesting, text fields, or total text content.", + ), + ErrorCode.BODY_FORMAT_UNSUPPORTED: ErrorSpec( + ErrorKind.INVALID_INPUT, + ErrorComponent.FORMAT_HANDLER, + "select", + "Request body format is unsupported.", + "Register or select a supported body format.", + ), + ErrorCode.BODY_ENCODING_INVALID: ErrorSpec( + ErrorKind.INVALID_INPUT, + ErrorComponent.FORMAT_HANDLER, + "normalize", + "Request body encoding is invalid.", + "Supply a valid UTF-8 request body.", + ), + ErrorCode.BODY_JSON_INVALID: ErrorSpec( + ErrorKind.INVALID_INPUT, + ErrorComponent.FORMAT_HANDLER, + "normalize", + "Request body is not valid JSON.", + "Check UTF-8 JSON syntax, duplicate keys, non-finite numbers, Unicode " + "scalars, and the configured body format.", + ), + ErrorCode.BODY_RECONSTRUCTION_INVALID: ErrorSpec( + ErrorKind.INTERNAL, + ErrorComponent.FORMAT_HANDLER, + "reconstruct", + "Request body reconstruction failed validation.", + "Check processor-generated paths and replacement text.", + ), + ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID: ErrorSpec( + ErrorKind.INTERNAL, + ErrorComponent.PROCESSOR, + "validate_handler", + "Format handler output is invalid.", + "Check the FormatHandler ABC and normalized-body contract.", + ), + ErrorCode.FORMAT_HANDLER_EXECUTION_FAILED: ErrorSpec( + ErrorKind.INTERNAL, + ErrorComponent.FORMAT_HANDLER, + "call", + "Format handler execution failed.", + "Run handler unit tests for normalization and reconstruction.", + ), + ErrorCode.SCANNER_OUTPUT_INVALID: ErrorSpec( + ErrorKind.INTERNAL, + ErrorComponent.PROCESSOR, + "validate_scanner", + "Scanner output is invalid.", + "Check scanner identity, supported entities, result tuple, labels, spans, " + "confidence, and paths.", + ), + ErrorCode.SCANNER_EXECUTION_FAILED: ErrorSpec( + ErrorKind.INTERNAL, + ErrorComponent.SCANNER, + "scan", + "Scanner execution failed.", + "Run the scanner single-block unit tests.", + ), + ErrorCode.FINDING_LIMIT_EXCEEDED: ErrorSpec( + ErrorKind.INTERNAL, + ErrorComponent.PROCESSOR, + "scan", + "Scanner finding limit was exceeded.", + "Tune scanner cardinality or the policy before enabling traffic.", + ), + ErrorCode.RESULT_LIMIT_EXCEEDED: ErrorSpec( + ErrorKind.INTERNAL, + ErrorComponent.SERVICE, + "serialize_result", + "A safe middleware result could not be represented.", + "Tune redaction size or scanner finding cardinality.", + ), + ErrorCode.SERVER_BIND_FAILED: ErrorSpec( + ErrorKind.INTERNAL, + ErrorComponent.SERVER, + "bind", + "Server could not bind its listen address.", + "Check the listen address and port availability.", + ), + ErrorCode.UNEXPECTED_SERVICE_FAILURE: ErrorSpec( + ErrorKind.INTERNAL, + ErrorComponent.SERVICE, + "evaluate_http_request", + "The middleware encountered an unexpected failure.", + "Reproduce with focused service and processor tests.", + ), +} diff --git a/projects/privacy-guard/src/privacy_guard/payloads/__init__.py b/projects/privacy-guard/src/privacy_guard/payloads/__init__.py new file mode 100644 index 0000000..18cee29 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/payloads/__init__.py @@ -0,0 +1,8 @@ +"""Data records carrying one request and its processing result.""" + +from __future__ import annotations + +from privacy_guard.payloads.request import InterceptedRequest +from privacy_guard.payloads.result import ProcessingDecision, ProcessingResult + +__all__ = ["InterceptedRequest", "ProcessingDecision", "ProcessingResult"] diff --git a/projects/privacy-guard/src/privacy_guard/payloads/request.py b/projects/privacy-guard/src/privacy_guard/payloads/request.py new file mode 100644 index 0000000..9bdafbe --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/payloads/request.py @@ -0,0 +1,32 @@ +"""The provider-bound request captured pre-credentials, free of proto types.""" + +from __future__ import annotations + +from pydantic import Field, field_validator + +from privacy_guard.config import PolicyConfig +from privacy_guard.validation import StrictSensitiveModel + + +class InterceptedRequest(StrictSensitiveModel): + """A captured HTTP request, decoupled from the proto ``HttpRequestEvaluation``. + + The servicer builds this from the proto message so that nothing below the + service layer depends on ``bindings/``. ``raw_body`` is kept out of ``repr`` + so routine object representations do not include the sensitive payload. + """ + + raw_body: bytes = Field(repr=False) + # Retained as normalized request context for future format negotiation; the + # JSON handler is selected explicitly by policy and does not inspect it. + content_type: str = "application/json" + policy_config: PolicyConfig = Field(default_factory=PolicyConfig, repr=False) + request_id: str = "" + + @field_validator("policy_config", mode="before") + @classmethod + def require_parsed_policy_config(cls, value: object) -> object: + """Require policy input to cross its content-safe parser first.""" + if not isinstance(value, PolicyConfig): + raise ValueError("policy config must already be parsed") + return value diff --git a/projects/privacy-guard/src/privacy_guard/payloads/result.py b/projects/privacy-guard/src/privacy_guard/payloads/result.py new file mode 100644 index 0000000..48b1ed3 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/payloads/result.py @@ -0,0 +1,26 @@ +"""The proto-free outcome of processing one intercepted request.""" + +from __future__ import annotations + +from enum import StrEnum + +from pydantic import Field + +from privacy_guard.scanners import RequestBodyFinding +from privacy_guard.validation import StrictSensitiveModel + + +class ProcessingDecision(StrEnum): + """Whether the supervisor should continue or stop the request.""" + + ALLOW = "allow" + DENY = "deny" + + +class ProcessingResult(StrictSensitiveModel): + """A decision, safe findings, and bytes that replace the body when present.""" + + decision: ProcessingDecision + replacement_body: bytes | None = Field(default=None, repr=False) + findings: tuple[RequestBodyFinding, ...] = () + reason_code: str | None = None diff --git a/projects/privacy-guard/src/privacy_guard/processor.py b/projects/privacy-guard/src/privacy_guard/processor.py new file mode 100644 index 0000000..a3b6310 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/processor.py @@ -0,0 +1,380 @@ +"""Proto-free processing for one complete intercepted request.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from string import Formatter + +from privacy_guard.body import ( + DEFAULT_FORMAT_HANDLERS, + FormatHandler, + FormatHandlerContractError, + RequestBody, + TextBlock, +) +from privacy_guard.config import ( + ActionConfig, + BlockActionConfig, + PolicyConfig, + RedactActionConfig, +) +from privacy_guard.constants import ( + BLOCK_REASON_CODE, + CONFIDENCE_RANK, + LIMIT_REASON_CODE, + MAX_BODY_BYTES, + MAX_FINDINGS_PER_BLOCK, + MAX_FINDINGS_PER_REQUEST, + MAX_SCANNED_CHARACTERS, + MAX_TEXT_BLOCKS, +) +from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.payloads import ( + InterceptedRequest, + ProcessingDecision, + ProcessingResult, +) +from privacy_guard.scanners import ( + Finding, + RequestBodyFinding, + Scanner, + ScannerConfig, + ScannerContractError, + ScannerFindingLimitExceeded, +) + + +class RequestProcessor: + """Coordinate normalization, scanners, policy, and reconstruction per request. + + Processor instances may be invoked concurrently by the service. Every configured + scanner must therefore make ``scan`` thread-safe and retain no request content. + """ + + def __init__( + self, + scanners: Sequence[Scanner[ScannerConfig]], + format_handlers: Mapping[str, FormatHandler] = DEFAULT_FORMAT_HANDLERS, + ) -> None: + scanner_tuple = tuple(scanners) + if not scanner_tuple: + raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) + names: set[str] = set() + supported_entities: set[str] = set() + for item in scanner_tuple: + if not isinstance(item, Scanner): + raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) + scanner_name = item.scanner_name + if scanner_name in names: + raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) + names.add(scanner_name) + supported_entities.update(item.supported_entity_types) + self._scanners: tuple[Scanner[ScannerConfig], ...] = scanner_tuple + self._supported_entities = frozenset(supported_entities) + self._format_handlers: dict[str, FormatHandler] = {} + for format_name, handler in format_handlers.items(): + if not isinstance(handler, FormatHandler): + raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID) + if handler.format_name != format_name: + raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID) + self._format_handlers[format_name] = handler + + def validate_policy_config(self, policy_config: PolicyConfig) -> None: + self._select_handler(policy_config.body_format) + self._validate_entity_filter(policy_config.on_finding) + + def process(self, request: InterceptedRequest) -> ProcessingResult: + policy = request.policy_config + action = policy.on_finding + handler = self._select_handler(policy.body_format) + self._validate_entity_filter(action) + if request.raw_body == b"": + return ProcessingResult(decision=ProcessingDecision.ALLOW) + + request_body = _normalize_request_body(handler, request) + verified_body = _validate_normalized_body(request_body, request.raw_body) + scan_result = self._scan_text_blocks(verified_body.text_blocks, action) + if scan_result.limit_exceeded: + return ProcessingResult( + decision=ProcessingDecision.DENY, reason_code=LIMIT_REASON_CODE + ) + if isinstance(action, BlockActionConfig) and scan_result.findings: + return ProcessingResult( + decision=ProcessingDecision.DENY, + findings=scan_result.findings, + reason_code=BLOCK_REASON_CODE, + ) + if isinstance(action, RedactActionConfig) and any( + not block.replaceable + and scan_result.findings_by_text_block_path[block.path] + for block in verified_body.text_blocks + ): + # JSON keys are scanned, but never rewritten: key mutation could collide. + return ProcessingResult( + decision=ProcessingDecision.DENY, + findings=scan_result.findings, + reason_code=BLOCK_REASON_CODE, + ) + + replacements: dict[str, str] = {} + redacted_text_bytes = 0 + if isinstance(action, RedactActionConfig): + for block in verified_body.text_blocks: + findings = scan_result.findings_by_text_block_path[block.path] + if findings: + redaction = _build_bounded_redaction( + block.text, + findings, + action.template, + MAX_BODY_BYTES - redacted_text_bytes, + ) + if redaction is None: + return ProcessingResult( + decision=ProcessingDecision.DENY, + reason_code=LIMIT_REASON_CODE, + ) + replacement, projected_size = redaction + redacted_text_bytes += projected_size + replacements[block.path] = replacement + reconstructed = _reconstruct_body(handler, verified_body.source, replacements) + if len(reconstructed) > MAX_BODY_BYTES: + return ProcessingResult( + decision=ProcessingDecision.DENY, reason_code=LIMIT_REASON_CODE + ) + return ProcessingResult( + decision=ProcessingDecision.ALLOW, + replacement_body=reconstructed + if reconstructed != request.raw_body + else None, + findings=scan_result.findings, + ) + + def _select_handler(self, requested_name: str) -> FormatHandler: + try: + handler = self._format_handlers[requested_name] + except KeyError: + raise PrivacyGuardError(ErrorCode.BODY_FORMAT_UNSUPPORTED) from None + return handler + + def _validate_entity_filter(self, action: ActionConfig) -> None: + if not action.entity_types: + return + if not action.entity_types.issubset(self._supported_entities): + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) + + def _scan_and_validate( + self, scanner: Scanner[ScannerConfig], text_block: TextBlock + ) -> tuple[Finding, ...]: + try: + result = scanner.scan(text_block.text) + except ScannerFindingLimitExceeded: + raise PrivacyGuardError(ErrorCode.FINDING_LIMIT_EXCEEDED) from None + except ScannerContractError: + raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) from None + except Exception: + raise PrivacyGuardError(ErrorCode.SCANNER_EXECUTION_FAILED) from None + for finding in result: + if ( + finding.scanner_name != scanner.scanner_name + or finding.entity not in scanner.supported_entity_types + or finding.end_offset > len(text_block.text) + ): + raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) + return tuple( + sorted(result, key=lambda item: (item.start_offset, item.end_offset)) + ) + + def _scan_text_blocks( + self, text_blocks: tuple[TextBlock, ...], action: ActionConfig + ) -> _ScanResult: + by_path: dict[str, tuple[Finding, ...]] = {} + all_findings: list[RequestBodyFinding] = [] + enabled = action.entity_types + threshold = ( + None + if action.minimum_confidence is None + else CONFIDENCE_RANK[action.minimum_confidence] + ) + for block in text_blocks: + block_findings: list[Finding] = [] + try: + for scanner in self._scanners: + for finding in self._scan_and_validate(scanner, block): + if (enabled is None or finding.entity in enabled) and ( + threshold is None + or CONFIDENCE_RANK[finding.confidence] >= threshold + ): + block_findings.append(finding) + if ( + len(block_findings) > MAX_FINDINGS_PER_BLOCK + or len(all_findings) + len(block_findings) + > MAX_FINDINGS_PER_REQUEST + ): + return _ScanResult((), {}, True) + except PrivacyGuardError as error: + if error.code is ErrorCode.FINDING_LIMIT_EXCEEDED: + return _ScanResult((), {}, True) + raise + ordered = tuple( + sorted( + block_findings, + key=lambda item: ( + item.start_offset, + item.end_offset, + item.scanner_name, + item.entity, + ), + ) + ) + by_path[block.path] = ordered + all_findings.extend( + RequestBodyFinding(finding=finding, text_block_path=block.path) + for finding in ordered + ) + return _ScanResult(tuple(all_findings), by_path) + + +@dataclass(frozen=True) +class _ScanResult: + findings: tuple[RequestBodyFinding, ...] + findings_by_text_block_path: Mapping[str, tuple[Finding, ...]] + limit_exceeded: bool = False + + +@dataclass(frozen=True) +class _VerifiedRequestBody: + """A body paired with the processor-validated request-relative view.""" + + source: RequestBody + text_blocks: tuple[TextBlock, ...] + + +def _normalize_request_body( + handler: FormatHandler, request: InterceptedRequest +) -> RequestBody: + try: + return handler.normalize(request.raw_body, request.policy_config) + except FormatHandlerContractError: + raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID) from None + except PrivacyGuardError: + raise + except Exception: + raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_EXECUTION_FAILED) from None + + +def _validate_normalized_body( + request_body: RequestBody, original_body: bytes +) -> _VerifiedRequestBody: + if request_body.original_bytes != original_body: + raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID) + text_blocks = request_body.text_blocks + if len(text_blocks) > MAX_TEXT_BLOCKS: + raise PrivacyGuardError(ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED) + + seen_paths: set[str] = set() + total_characters = 0 + for text_block in text_blocks: + if text_block.path in seen_paths: + raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID) + seen_paths.add(text_block.path) + total_characters += len(text_block.text) + if total_characters > MAX_SCANNED_CHARACTERS: + raise PrivacyGuardError(ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED) + return _VerifiedRequestBody(source=request_body, text_blocks=text_blocks) + + +def _resolve_overlaps(findings: tuple[Finding, ...]) -> tuple[Finding, ...]: + """Choose deterministic redaction winners; observation retains all findings.""" + winners: list[Finding] = [] + ranked = sorted( + findings, + key=lambda item: ( + -CONFIDENCE_RANK[item.confidence], + -(item.end_offset - item.start_offset), + item.start_offset, + item.end_offset, + item.scanner_name, + item.entity, + ), + ) + for candidate in ranked: + if all( + candidate.end_offset <= winner.start_offset + or candidate.start_offset >= winner.end_offset + for winner in winners + ): + winners.append(candidate) + return tuple(sorted(winners, key=lambda item: (item.start_offset, item.end_offset))) + + +def _redact_text( + original_text: str, findings: tuple[Finding, ...], template: str +) -> str: + parts: list[str] = [] + cursor = 0 + for finding in findings: + parts.append(original_text[cursor : finding.start_offset]) + parts.append(template.format(entity=finding.entity)) + cursor = finding.end_offset + parts.append(original_text[cursor:]) + return "".join(parts) + + +def _formatted_template_size(template: str, entity: str) -> int: + """Return UTF-8 size without constructing a potentially large rendering.""" + size = 0 + entity_size = len(entity.encode("utf-8")) + for literal, field_name, _, _ in Formatter().parse(template): + size += len(literal.encode("utf-8")) + if field_name is not None: + size += entity_size + return size + + +def _project_redacted_text_size( + original_text: str, + findings: tuple[Finding, ...], + template: str, + remaining_bytes: int, +) -> int | None: + """Bound rendered text before allocating it; serialization is checked later.""" + size = 0 + cursor = 0 + for finding in findings: + size += len(original_text[cursor : finding.start_offset].encode("utf-8")) + size += _formatted_template_size(template, finding.entity) + if size > remaining_bytes: + return None + cursor = finding.end_offset + size += len(original_text[cursor:].encode("utf-8")) + return size if size <= remaining_bytes else None + + +def _build_bounded_redaction( + original_text: str, + findings: tuple[Finding, ...], + template: str, + remaining_bytes: int, +) -> tuple[str, int] | None: + """Resolve overlaps once, then render only bounded intermediate text.""" + resolved = _resolve_overlaps(findings) + projected_size = _project_redacted_text_size( + original_text, resolved, template, remaining_bytes + ) + if projected_size is None: + return None + return _redact_text(original_text, resolved, template), projected_size + + +def _reconstruct_body( + handler: FormatHandler, request_body: RequestBody, replacements: Mapping[str, str] +) -> bytes: + try: + return handler.reconstruct(request_body, replacements) + except FormatHandlerContractError: + raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID) from None + except PrivacyGuardError: + raise + except Exception: + raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_EXECUTION_FAILED) from None diff --git a/projects/privacy-guard/src/privacy_guard/scanners/__init__.py b/projects/privacy-guard/src/privacy_guard/scanners/__init__.py new file mode 100644 index 0000000..2334b6b --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/scanners/__init__.py @@ -0,0 +1,27 @@ +"""Scan individual text blocks into safe findings without copying matches.""" + +from __future__ import annotations + +from privacy_guard.scanners.base import ( + Confidence, + Finding, + RequestBodyFinding, + Scanner, + ScannerConfig, + ScannerContractError, + ScannerFindingLimitExceeded, + parse_scanner_output, +) +from privacy_guard.scanners.passthrough import PassthroughScanner + +__all__ = [ + "Confidence", + "Finding", + "PassthroughScanner", + "RequestBodyFinding", + "Scanner", + "ScannerConfig", + "ScannerContractError", + "ScannerFindingLimitExceeded", + "parse_scanner_output", +] diff --git a/projects/privacy-guard/src/privacy_guard/scanners/base.py b/projects/privacy-guard/src/privacy_guard/scanners/base.py new file mode 100644 index 0000000..85533a0 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/scanners/base.py @@ -0,0 +1,156 @@ +"""Strict finding models and the nominal scanner extension contract.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from enum import StrEnum +from typing import Generic, Self, TypeVar, get_args, get_origin + +from pydantic import ( + Field, + ValidationError, + field_validator, + model_validator, +) +from typing_extensions import TypeIs + +from privacy_guard.constants import MAX_FINDINGS_PER_BLOCK, MAX_SCANNER_METADATA_BYTES +from privacy_guard.validation import ScalarString, StrictDomainModel + + +class Confidence(StrEnum): + """Scanner-owned confidence assigned to a finding.""" + + LOW = "low" + MEDIUM = "medium" + HIGH = "high" + + +class Finding(StrictDomainModel): + """A scanner-owned, block-relative sensitive-data detection.""" + + entity: str = Field(..., min_length=1, max_length=MAX_SCANNER_METADATA_BYTES) + scanner_name: str = Field(..., min_length=1, max_length=MAX_SCANNER_METADATA_BYTES) + start_offset: int = Field(..., ge=0) + end_offset: int + confidence: Confidence = Confidence.HIGH + + @field_validator("entity", "scanner_name") + @classmethod + def validate_metadata_boundary(cls, value: str) -> str: + """Require valid Unicode metadata that fits the UTF-8 byte limit.""" + if any("\ud800" <= character <= "\udfff" for character in value): + raise ValueError("finding metadata must contain valid Unicode") + if len(value.encode("utf-8")) > MAX_SCANNER_METADATA_BYTES: + raise ValueError("finding metadata exceeds the UTF-8 byte limit") + return value + + @model_validator(mode="after") + def validate_non_empty_span(self) -> Self: + """Require the finding to cover at least one character.""" + if self.end_offset <= self.start_offset: + raise ValueError("finding span must be non-empty") + return self + + +class RequestBodyFinding(StrictDomainModel): + """Place a block-relative Finding at its path within a normalized RequestBody.""" + + finding: Finding + text_block_path: ScalarString = Field(repr=False) + + +class ScannerConfig(StrictDomainModel): + """Immutable scanner identity and complete entity catalog.""" + + name: str = Field(..., min_length=1) + entity_types: frozenset[str] + + +class ScannerContractError(Exception): + """A content-safe scanner configuration or output contract failure.""" + + +class ScannerFindingLimitExceeded(Exception): + """The scanner exceeded the bounded per-block finding count.""" + + +def parse_scanner_output(result: object) -> tuple[Finding, ...]: + """Validate a scanner's tuple output against the Finding contract.""" + if not isinstance(result, tuple): + raise ScannerContractError("scanner output is invalid") + if len(result) > MAX_FINDINGS_PER_BLOCK: + raise ScannerFindingLimitExceeded + if not _contains_only_findings(result): + raise ScannerContractError("scanner output is invalid") + return result + + +def _contains_only_findings( + result: tuple[object, ...], +) -> TypeIs[tuple[Finding, ...]]: + return all(isinstance(finding, Finding) for finding in result) + + +# This private type variable must precede the public generic classes that use it. +_ScannerConfigT = TypeVar("_ScannerConfigT", bound=ScannerConfig, covariant=True) + + +class Scanner(ABC, Generic[_ScannerConfigT]): + """Nominal, concurrency-safe extension point for scanning one text block.""" + + def __init__(self, config: _ScannerConfigT) -> None: + """Validate and retain the scanner's declared concrete configuration.""" + try: + validated_config = self.get_config_type().model_validate(config) + except ValidationError: + raise ScannerContractError("scanner configuration is invalid") from None + self.__config = validated_config + self.__scanner_name = validated_config.name + + @classmethod + def get_config_type(cls) -> type[_ScannerConfigT]: + """Return the ScannerConfig type declared by the subclass.""" + for base in getattr(cls, "__orig_bases__", ()): + for argument in get_args(base): + origin = get_origin(argument) or argument + if isinstance(origin, type) and issubclass(origin, ScannerConfig): + return origin + raise TypeError(f"{cls.__name__} must declare a ScannerConfig generic argument") + + @property + def config(self) -> _ScannerConfigT: + """Return the scanner's immutable concrete configuration.""" + return self.__config + + @property + def scanner_name(self) -> str: + """Return the immutable stable scanner name.""" + return self.__scanner_name + + @property + def supported_entity_types(self) -> frozenset[str]: + """Return the complete entity catalog declared by the scanner config.""" + return self.config.entity_types + + def scan(self, text_block: ScalarString) -> tuple[Finding, ...]: + """Run the implementation and validate its observable output shape.""" + result: object = self._scan(text_block) + return parse_scanner_output(result) + + @abstractmethod + def _scan(self, text_block: str) -> tuple[Finding, ...]: + """Return block-relative findings for one block of text.""" + raise NotImplementedError + + +__all__ = [ + "Confidence", + "Finding", + "RequestBodyFinding", + "Scanner", + "ScannerConfig", + "ScannerContractError", + "ScannerFindingLimitExceeded", + "parse_scanner_output", +] diff --git a/projects/privacy-guard/src/privacy_guard/scanners/passthrough.py b/projects/privacy-guard/src/privacy_guard/scanners/passthrough.py new file mode 100644 index 0000000..1b48714 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/scanners/passthrough.py @@ -0,0 +1,23 @@ +"""Scanner that deliberately reports no findings for a text block.""" + +from __future__ import annotations + +from typing_extensions import override + +from privacy_guard.scanners.base import ( + Finding, + Scanner, + ScannerConfig, +) + + +class PassthroughScanner(Scanner[ScannerConfig]): + """Scan one text block without reporting sensitive values.""" + + def __init__(self) -> None: + super().__init__(ScannerConfig(name="passthrough", entity_types=frozenset())) + + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + """Return no findings for the supplied text block.""" + return () diff --git a/projects/privacy-guard/src/privacy_guard/service/__init__.py b/projects/privacy-guard/src/privacy_guard/service/__init__.py new file mode 100644 index 0000000..582585e --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/service/__init__.py @@ -0,0 +1,9 @@ +"""gRPC transport and servicer for the Privacy Guard middleware.""" + +from privacy_guard.service.server import MiddlewareServer +from privacy_guard.service.servicer import ( + PrivacyGuardMiddleware, + RequestProcessorLike, +) + +__all__ = ["MiddlewareServer", "PrivacyGuardMiddleware", "RequestProcessorLike"] diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py new file mode 100644 index 0000000..0aa4b0c --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -0,0 +1,80 @@ +"""gRPC server process that hosts Privacy Guard at a configured endpoint. + +This is pure transport lifecycle -- it has no counterpart in an in-process +(built-in) middleware. It exists only because Privacy Guard runs out-of-process +and the supervisor reaches it over gRPC. The default endpoint is loopback. +""" + +from __future__ import annotations + +import argparse +import asyncio +from collections.abc import Sequence + +import grpc + +from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc +from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES +from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.processor import RequestProcessor +from privacy_guard.scanners import PassthroughScanner, Scanner, ScannerConfig +from privacy_guard.service.servicer import PrivacyGuardMiddleware + + +class MiddlewareServer: + """High-level server that wires a scanner into the Privacy Guard service.""" + + def __init__(self, *, scanner: Scanner[ScannerConfig]) -> None: + self._servicer = PrivacyGuardMiddleware(RequestProcessor([scanner])) + + def serve(self, listen: str = "127.0.0.1:50051") -> None: + """Serve until termination using a managed synchronous entry point.""" + asyncio.run(serve(listen, self._servicer)) + + +def create_server(servicer: PrivacyGuardMiddleware | None = None) -> grpc.aio.Server: + """Build an unstarted gRPC server with the servicer mounted (no port bound). + + Accepts a servicer for tests; defaults to a fresh ``PrivacyGuardMiddleware``. + The receive limit reserves bounded space around the advertised body maximum + for the protobuf envelope; the servicer enforces the body limit itself. + """ + server = grpc.aio.server( + maximum_concurrent_rpcs=MAX_CONCURRENT_RPCS, + options=(("grpc.max_receive_message_length", MAX_RECEIVE_MESSAGE_BYTES),), + ) + pb2_grpc.add_SupervisorMiddlewareServicer_to_server( + servicer if servicer is not None else PrivacyGuardMiddleware(), server + ) + return server + + +async def serve( + listen: str = "127.0.0.1:50051", + servicer: PrivacyGuardMiddleware | None = None, +) -> None: + """Bind ``listen``, start the server, and serve until terminated.""" + effective_servicer = servicer if servicer is not None else PrivacyGuardMiddleware() + server = create_server(effective_servicer) + try: + bound_port = server.add_insecure_port(listen) + if bound_port == 0: + raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) + await server.start() + await server.wait_for_termination() + finally: + await server.stop(grace=0) + await effective_servicer.close() + + +def main(argv: Sequence[str] | None = None) -> int: + """CLI entry point: parse args and run the high-level middleware server.""" + parser = argparse.ArgumentParser(description="Run the Privacy Guard middleware") + parser.add_argument("--listen", default="127.0.0.1:50051") + arguments = parser.parse_args(argv) + MiddlewareServer(scanner=PassthroughScanner()).serve(arguments.listen) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py new file mode 100644 index 0000000..dff52bb --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -0,0 +1,323 @@ +"""Thin gRPC adapter for the proto-free Privacy Guard processor.""" + +from __future__ import annotations + +import asyncio +import logging +import time +from collections import OrderedDict +from concurrent.futures import ThreadPoolExecutor +from typing import Never, Protocol, TypedDict + +import grpc +from google.protobuf import json_format +from google.protobuf.message import Message +from pydantic import ValidationError +from typing_extensions import override + +from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 +from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc +from privacy_guard.config import PolicyConfig +from privacy_guard.constants import ( + BLOCK_REASON, + BLOCK_REASON_CODE, + LIMIT_REASON, + LIMIT_REASON_CODE, + MAX_BODY_BYTES, + MAX_CONCURRENT_SCANS, + MAX_PROTO_FINDING_BYTES, + MAX_PROTO_FINDING_GROUPS, + REASON_CODE_PATTERN, + SERVICE_NAME, + SERVICE_VERSION, + UINT32_MAX, +) +from privacy_guard.errors import ErrorCode, ErrorKind, PrivacyGuardError +from privacy_guard.payloads import ( + InterceptedRequest, + ProcessingDecision, + ProcessingResult, +) +from privacy_guard.processor import RequestProcessor +from privacy_guard.scanners import PassthroughScanner, RequestBodyFinding + + +class RequestProcessorLike(Protocol): + """Narrow structural seam for injecting request processors into the service.""" + + def validate_policy_config(self, policy_config: PolicyConfig) -> None: ... + + def process(self, request: InterceptedRequest) -> ProcessingResult: ... + + +class PrivacyGuardMiddleware(pb2_grpc.SupervisorMiddlewareServicer): + """Translate protobuf requests and responses at the transport boundary.""" + + def __init__(self, processor: RequestProcessorLike | None = None) -> None: + self._processor = ( + processor + if processor is not None + else RequestProcessor([PassthroughScanner()]) + ) + self._scan_slots = asyncio.Semaphore(MAX_CONCURRENT_SCANS) + self._scan_executor = ThreadPoolExecutor( + max_workers=MAX_CONCURRENT_SCANS, + thread_name_prefix="privacy-guard-scan", + ) + + async def close(self) -> None: + """Stop accepting worker jobs and wait for active synchronous scans.""" + await asyncio.to_thread( + self._scan_executor.shutdown, wait=True, cancel_futures=True + ) + + @override + async def Describe( + self, + request: object, + context: grpc.aio.ServicerContext[object, pb2.MiddlewareManifest], + ) -> pb2.MiddlewareManifest: + """Advertise the HTTP pre-credentials binding and maximum body size.""" + return self._describe() + + @override + async def ValidateConfig( + self, + request: pb2.ValidateConfigRequest, + context: grpc.aio.ServicerContext[ + pb2.ValidateConfigRequest, pb2.ValidateConfigResponse + ], + ) -> pb2.ValidateConfigResponse: + """Parse and validate policy configuration without processing a body.""" + return self._validate_config(request) + + @override + async def EvaluateHttpRequest( + self, + request: pb2.HttpRequestEvaluation, + context: grpc.aio.ServicerContext[ + pb2.HttpRequestEvaluation, pb2.HttpRequestResult + ], + ) -> pb2.HttpRequestResult: + """Validate transport input, delegate valid requests, and map the result.""" + return await self._evaluate_rpc(request, context) + + def _describe(self) -> pb2.MiddlewareManifest: + """Build the transport manifest without requiring an RPC context.""" + return pb2.MiddlewareManifest( + name=SERVICE_NAME, + service_version=SERVICE_VERSION, + bindings=[ + pb2.MiddlewareBinding( + operation=pb2.SUPERVISOR_MIDDLEWARE_OPERATION_HTTP_REQUEST, + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + max_body_bytes=MAX_BODY_BYTES, + ) + ], + ) + + def _validate_config( + self, request: pb2.ValidateConfigRequest + ) -> pb2.ValidateConfigResponse: + """Validate config through the narrow processor seam.""" + try: + policy_config = _policy_from_proto(request.config) + self._processor.validate_policy_config(policy_config) + except PrivacyGuardError as error: + return pb2.ValidateConfigResponse(valid=False, reason=str(error)) + except Exception: + error = PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) + return pb2.ValidateConfigResponse(valid=False, reason=str(error)) + return pb2.ValidateConfigResponse(valid=True) + + async def _evaluate_rpc( + self, request: pb2.HttpRequestEvaluation, context: _AbortContext + ) -> pb2.HttpRequestResult: + """Run one evaluation using only the context's abort capability.""" + started = time.monotonic() + request_id = request.context.request_id + failure: PrivacyGuardError | None = None + action = "error" + finding_count = 0 + try: + response = await self._evaluate_http_request(request) + action = "allow" if response.decision == pb2.DECISION_ALLOW else "deny" + finding_count = sum(finding.count for finding in response.findings) + return response + except PrivacyGuardError as error: + failure = error + except Exception: + failure = PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) + finally: + log_extra = _evaluation_log_extra( + request_id=request_id, + started=started, + action=action, + finding_count=finding_count, + failure=failure, + ) + _LOGGER.info( + "privacy_guard_evaluation", + extra=log_extra, + ) + + status = ( + grpc.StatusCode.INVALID_ARGUMENT + if failure.kind is ErrorKind.INVALID_INPUT + else grpc.StatusCode.INTERNAL + ) + await context.abort(status, str(failure)) + + async def _evaluate_http_request( + self, request: pb2.HttpRequestEvaluation + ) -> pb2.HttpRequestResult: + """Evaluate a request through the context-free application seam.""" + if request.phase != pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS: + raise PrivacyGuardError(ErrorCode.REQUEST_PHASE_INVALID) + if len(request.body) > MAX_BODY_BYTES: + raise PrivacyGuardError(ErrorCode.REQUEST_BODY_TOO_LARGE) + + intercepted = _request_from_proto(request) + # Cancellation cannot stop a Python worker already running. The done + # callback keeps its slot occupied until it really finishes, while the + # RPC can still observe cancellation immediately. + await self._scan_slots.acquire() + try: + future = asyncio.get_running_loop().run_in_executor( + self._scan_executor, self._processor.process, intercepted + ) + except BaseException: + self._scan_slots.release() + raise + future.add_done_callback(lambda _: self._scan_slots.release()) + result = await asyncio.shield(future) + return _result_to_proto(result) + + +_LOGGER = logging.getLogger(__name__) + + +class _AbortContext(Protocol): + async def abort(self, code: grpc.StatusCode, details: str) -> Never: ... + + +class _EvaluationLogExtra(TypedDict): + request_id: str + duration_ms: float + action: str + finding_count: int + error_code: str | None + + +def _evaluation_log_extra( + *, + request_id: str, + started: float, + action: str, + finding_count: int, + failure: PrivacyGuardError | None, +) -> _EvaluationLogExtra: + """Build the typed operational fields attached to an evaluation log.""" + return { + "request_id": request_id, + "duration_ms": round((time.monotonic() - started) * 1000, 3), + "action": action, + "finding_count": finding_count, + "error_code": failure.code.value if failure is not None else None, + } + + +def _policy_from_proto(config: Message) -> PolicyConfig: + try: + values = json_format.MessageToDict(config) + return PolicyConfig.from_mapping(values) + except Exception: + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None + + +def _request_from_proto(request: pb2.HttpRequestEvaluation) -> InterceptedRequest: + policy_config = _policy_from_proto(request.config) + try: + return InterceptedRequest( + raw_body=request.body, + content_type=next( + ( + header.value + for header in request.headers + if header.name == "content-type" + ), + "", + ), + request_id=request.context.request_id, + policy_config=policy_config, + ) + except ValidationError: + raise PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) from None + + +def _result_to_proto(result: ProcessingResult) -> pb2.HttpRequestResult: + if ( + result.replacement_body is not None + and len(result.replacement_body) > MAX_BODY_BYTES + ): + return _limit_deny() + try: + findings = _aggregate_findings(result.findings) + except PrivacyGuardError as error: + if error.code is ErrorCode.RESULT_LIMIT_EXCEEDED: + return _limit_deny() + raise + if result.decision is ProcessingDecision.ALLOW: + has_body = result.replacement_body is not None + return pb2.HttpRequestResult( + decision=pb2.DECISION_ALLOW, + body=result.replacement_body if has_body else b"", + has_body=has_body, + findings=findings, + ) + if result.decision is ProcessingDecision.DENY: + reason_code = result.reason_code or BLOCK_REASON_CODE + if REASON_CODE_PATTERN.fullmatch(reason_code) is None: + return _limit_deny() + return pb2.HttpRequestResult( + decision=pb2.DECISION_DENY, + reason=LIMIT_REASON if reason_code == LIMIT_REASON_CODE else BLOCK_REASON, + reason_code=reason_code, + has_body=False, + findings=findings, + ) + raise PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) + + +def _limit_deny() -> pb2.HttpRequestResult: + return pb2.HttpRequestResult( + decision=pb2.DECISION_DENY, + reason=LIMIT_REASON, + reason_code=LIMIT_REASON_CODE, + has_body=False, + ) + + +def _aggregate_findings(findings: tuple[RequestBodyFinding, ...]) -> list[pb2.Finding]: + groups: OrderedDict[tuple[str, str, str], int] = OrderedDict() + for request_body_finding in findings: + finding = request_body_finding.finding + key = (finding.scanner_name, finding.entity, finding.confidence.value) + groups[key] = groups.get(key, 0) + 1 + if len(groups) > MAX_PROTO_FINDING_GROUPS: + raise PrivacyGuardError(ErrorCode.RESULT_LIMIT_EXCEEDED) + aggregated = [ + pb2.Finding( + type=scanner_name, + label=entity, + confidence=confidence, + count=min(count, UINT32_MAX), + ) + for (scanner_name, entity, confidence), count in groups.items() + ] + if any(finding.ByteSize() > MAX_PROTO_FINDING_BYTES for finding in aggregated): + raise PrivacyGuardError(ErrorCode.RESULT_LIMIT_EXCEEDED) + return aggregated + + +__all__ = ["PrivacyGuardMiddleware", "RequestProcessorLike"] diff --git a/projects/privacy-guard/src/privacy_guard/validation.py b/projects/privacy-guard/src/privacy_guard/validation.py new file mode 100644 index 0000000..37e0289 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/validation.py @@ -0,0 +1,58 @@ +"""Shared strict, content-safe validation primitives.""" + +from __future__ import annotations + +from typing import Annotated + +from pydantic import BaseModel, BeforeValidator, ConfigDict + +from privacy_guard.constants import MAX_SCANNER_METADATA_BYTES + + +def parse_scalar_string(value: object) -> str: + """Return a string containing only valid Unicode scalar values.""" + if not isinstance(value, str): + raise ValueError("value must be a string") + if any("\ud800" <= character <= "\udfff" for character in value): + raise ValueError("string must contain valid Unicode scalar values") + return value + + +def parse_non_empty_scalar_string(value: object) -> str: + """Return a non-empty Unicode scalar string.""" + parsed = parse_scalar_string(value) + if not parsed: + raise ValueError("string must not be empty") + return parsed + + +def parse_bounded_metadata_string(value: object) -> str: + """Return metadata whose UTF-8 representation is within the package limit.""" + parsed = parse_non_empty_scalar_string(value) + if ( + len(parsed) > MAX_SCANNER_METADATA_BYTES + or len(parsed.encode("utf-8")) > MAX_SCANNER_METADATA_BYTES + ): + raise ValueError("metadata exceeds the UTF-8 byte limit") + return parsed + + +ScalarString = Annotated[str, BeforeValidator(parse_scalar_string)] +NonEmptyScalarString = Annotated[str, BeforeValidator(parse_non_empty_scalar_string)] +BoundedMetadataString = Annotated[str, BeforeValidator(parse_bounded_metadata_string)] + + +class StrictDomainModel(BaseModel): + """Base for immutable domain values parsed without implicit coercion.""" + + model_config = ConfigDict( + strict=True, + frozen=True, + extra="forbid", + hide_input_in_errors=True, + validate_default=True, + ) + + +class StrictSensitiveModel(StrictDomainModel): + """Semantic base for strict models containing repr-hidden sensitive fields.""" diff --git a/projects/privacy-guard/tests/__init__.py b/projects/privacy-guard/tests/__init__.py new file mode 100644 index 0000000..2017165 --- /dev/null +++ b/projects/privacy-guard/tests/__init__.py @@ -0,0 +1 @@ +"""Privacy Guard test package.""" diff --git a/projects/privacy-guard/tests/body/test_base.py b/projects/privacy-guard/tests/body/test_base.py new file mode 100644 index 0000000..337c81e --- /dev/null +++ b/projects/privacy-guard/tests/body/test_base.py @@ -0,0 +1,122 @@ +from collections.abc import Mapping +from dataclasses import FrozenInstanceError + +import pytest +from pydantic import ValidationError +from typing_extensions import override + +from privacy_guard.body import ( + FormatHandler, + FormatHandlerContractError, + RequestBody, + TextBlock, + parse_normalized_body, +) +from privacy_guard.config import PolicyConfig + + +class OpaqueHandler(FormatHandler): + def __init__(self, format_name: str = "opaque") -> None: + super().__init__(format_name=format_name) + + @override + def _normalize(self, raw_body: bytes, policy_config: PolicyConfig) -> RequestBody: + return RequestBody(text_blocks=(), parsed_value=None, original_bytes=raw_body) + + @override + def _reconstruct( + self, + request_body: RequestBody, + replacements_by_path: Mapping[str, str], + ) -> bytes: + return request_body.original_bytes + + +@pytest.mark.parametrize( + "values", + [ + {"path": 3, "text": "text"}, + {"path": "path", "text": 3}, + {"path": "path", "text": "text", "replaceable": 1}, + {"path": "bad\ud800", "text": "text"}, + {"path": "path", "text": "bad\ud800"}, + {"path": "path", "text": "text", "extra": "forbidden"}, + ], +) +def test_text_block_rejects_invalid_fields(values: dict[str, object]) -> None: + with pytest.raises(ValidationError): + TextBlock.model_validate(values) + + +def test_text_block_is_frozen_and_hides_sensitive_fields() -> None: + sentinel = "sensitive-block-8472" + block = TextBlock(path=f"/{sentinel}", text=sentinel) + + with pytest.raises((FrozenInstanceError, ValidationError)): + setattr(block, "text", "changed") + + assert sentinel not in repr(block) + + +@pytest.mark.parametrize( + "values", + [ + {"text_blocks": [], "parsed_value": None, "original_bytes": b"body"}, + {"text_blocks": (object(),), "parsed_value": None, "original_bytes": b"body"}, + {"text_blocks": (), "parsed_value": None, "original_bytes": bytearray(b"body")}, + { + "text_blocks": (), + "parsed_value": None, + "original_bytes": b"body", + "extra": "forbidden", + }, + ], +) +def test_request_body_rejects_invalid_fields(values: dict[str, object]) -> None: + with pytest.raises(ValidationError): + RequestBody.model_validate(values) + + +def test_request_body_is_frozen_and_hides_sensitive_fields() -> None: + sentinel = "sensitive-body-8472" + body = RequestBody( + text_blocks=(TextBlock(path=f"/{sentinel}", text=sentinel),), + parsed_value={"value": sentinel}, + original_bytes=sentinel.encode(), + ) + + with pytest.raises((FrozenInstanceError, ValidationError)): + setattr(body, "original_bytes", b"changed") + + assert sentinel not in repr(body) + + +def test_parse_normalized_body_reuses_exact_instance() -> None: + body = RequestBody(text_blocks=(), parsed_value=None, original_bytes=b"body") + + assert parse_normalized_body(body) is body + + +@pytest.mark.parametrize( + "result", + [ + object(), + {"text_blocks": (), "parsed_value": None, "original_bytes": b"body"}, + ], +) +def test_parse_normalized_body_rejects_non_models(result: object) -> None: + with pytest.raises(FormatHandlerContractError): + parse_normalized_body(result) + + +@pytest.mark.parametrize("format_name", ["", "bad\ud800"]) +def test_handler_rejects_invalid_format_identity(format_name: str) -> None: + with pytest.raises(FormatHandlerContractError): + OpaqueHandler(format_name) + + +def test_handler_identity_is_read_only() -> None: + handler = OpaqueHandler() + + with pytest.raises(AttributeError): + setattr(handler, "format_name", "changed") diff --git a/projects/privacy-guard/tests/body/test_json.py b/projects/privacy-guard/tests/body/test_json.py new file mode 100644 index 0000000..12fd3a0 --- /dev/null +++ b/projects/privacy-guard/tests/body/test_json.py @@ -0,0 +1,575 @@ +import json +import sys +from copy import deepcopy +from dataclasses import FrozenInstanceError + +import pytest +from pydantic import ValidationError + +import privacy_guard.body.json as json_module +from privacy_guard.body import JsonHandler, RequestBody, TextBlock, select_handler +from privacy_guard.config import PolicyConfig +from privacy_guard.errors import ErrorCode, PrivacyGuardError + + +def _assert_safe_error( + error: PrivacyGuardError, expected_code: ErrorCode, sentinel: str +) -> None: + assert error.code is expected_code + assert error.__cause__ is None + assert sentinel not in str(error) + assert sentinel not in repr(error) + assert all(sentinel not in str(argument) for argument in error.args) + + +def _json_state_value(request_body: RequestBody) -> object: + assert type(request_body.parsed_value) is json_module._JsonBodyState + return request_body.parsed_value.value + + +def test_body_domain_reprs_hide_sensitive_content() -> None: + sensitive_text = "sensitive-value-8472" + text_block = TextBlock(path="/message", text=sensitive_text) + request_body = RequestBody( + text_blocks=(text_block,), + parsed_value={"message": sensitive_text}, + original_bytes=sensitive_text.encode(), + ) + + assert sensitive_text not in repr(text_block) + assert sensitive_text not in repr(request_body) + + +def test_body_domain_reprs_hide_sensitive_json_pointer_tokens() -> None: + sensitive_key = "secret-person@example.com" + text_block = TextBlock(path=f"/{sensitive_key}", text="safe") + request_body = RequestBody( + text_blocks=(text_block,), parsed_value={}, original_bytes=b"{}" + ) + + assert sensitive_key not in repr(text_block) + assert sensitive_key not in repr(request_body) + + +def test_json_body_state_repr_hides_recursive_value() -> None: + sensitive_text = "distinctive-state-value-8472" + request_body = JsonHandler().normalize( + f'{{"message":"{sensitive_text}"}}'.encode(), PolicyConfig() + ) + + assert type(request_body.parsed_value) is json_module._JsonBodyState + assert sensitive_text not in repr(request_body.parsed_value) + + +def test_normalized_json_state_stores_the_validated_json_value_directly() -> None: + request_body = JsonHandler().normalize( + b'{"items":[{"message":"original"}]}', PolicyConfig() + ) + + assert type(request_body.parsed_value) is json_module._JsonBodyState + assert type(request_body.parsed_value.value) is dict + with pytest.raises(FrozenInstanceError): + setattr(request_body.parsed_value, "value", None) + assert type(request_body.parsed_value.value["items"]) is list + + +@pytest.mark.parametrize( + "invalid_value", + [ + b"bytes-are-not-json-strings", + ("tuples", "are", "not", "arrays"), + {1: "object keys must be strings"}, + {"nested": object()}, + float("inf"), + ], +) +def test_json_value_adapter_rejects_non_json_types_strictly( + invalid_value: object, +) -> None: + with pytest.raises(ValidationError): + json_module._JSON_VALUE_ADAPTER.validate_python(invalid_value, strict=True) + + +@pytest.mark.parametrize( + ("raw_body", "expected_value"), + [ + (b"null", None), + (b"true", True), + (b"42", 42), + (b"3.5", 3.5), + (b'"root"', "root"), + (b"[]", []), + (b"{}", {}), + ( + b'{"nested":[null,false,1,2.5,"text"]}', + {"nested": [None, False, 1, 2.5, "text"]}, + ), + ], +) +def test_normalize_stores_strict_recursive_json_roots( + raw_body: bytes, expected_value: object +) -> None: + request_body = JsonHandler().normalize(raw_body, PolicyConfig()) + + assert type(request_body.parsed_value) is json_module._JsonBodyState + assert _json_state_value(request_body) == expected_value + if not isinstance(expected_value, dict | list): + assert type(_json_state_value(request_body)) is type(expected_value) + + +def test_normalize_flat_object_collects_string_leaves() -> None: + raw_body = b'{"message":"hello","model":"model-a","count":2}' + + request_body = JsonHandler().normalize(raw_body, PolicyConfig()) + + assert request_body.original_bytes is raw_body + assert type(request_body.parsed_value) is json_module._JsonBodyState + assert _json_state_value(request_body) == { + "message": "hello", + "model": "model-a", + "count": 2, + } + assert tuple(block for block in request_body.text_blocks if block.replaceable) == ( + TextBlock(path="/message", text="hello"), + TextBlock(path="/model", text="model-a"), + ) + + +@pytest.mark.parametrize( + ("raw_body", "expected_text_blocks"), + [ + ( + b'{"outer":{"inner":"value"}}', + (TextBlock(path="/outer/inner", text="value"),), + ), + ( + b'["first",["nested",3],{"last":"value"}]', + ( + TextBlock(path="/0", text="first"), + TextBlock(path="/1/0", text="nested"), + TextBlock(path="/2/last", text="value"), + ), + ), + ( + b'[null,true,false,3.5,"only-string"]', + (TextBlock(path="/4", text="only-string"),), + ), + (b'"root string"', (TextBlock(path="", text="root string"),)), + (b"{}", ()), + (b"[]", ()), + ( + b'{"z":"first","a":{"y":"second","x":"third"},"m":"fourth"}', + ( + TextBlock(path="/z", text="first"), + TextBlock(path="/a/y", text="second"), + TextBlock(path="/a/x", text="third"), + TextBlock(path="/m", text="fourth"), + ), + ), + ( + b'{"a/b":"slash","til~de":"tilde","both~/":"both"}', + ( + TextBlock(path="/a~1b", text="slash"), + TextBlock(path="/til~0de", text="tilde"), + TextBlock(path="/both~0~1", text="both"), + ), + ), + ( + '{"greeting":"こんにちは","emoji":"🐍"}'.encode(), + ( + TextBlock(path="/greeting", text="こんにちは"), + TextBlock(path="/emoji", text="🐍"), + ), + ), + ( + b'{"escaped_emoji":"\\ud83d\\udc0d"}', + (TextBlock(path="/escaped_emoji", text="🐍"),), + ), + ], +) +def test_normalize_walks_arbitrary_json_in_stable_depth_first_order( + raw_body: bytes, expected_text_blocks: tuple[TextBlock, ...] +) -> None: + request_body = JsonHandler().normalize(raw_body, PolicyConfig()) + + assert ( + tuple(block for block in request_body.text_blocks if block.replaceable) + == expected_text_blocks + ) + + +def test_normalize_exposes_object_keys_as_nonreplaceable_blocks() -> None: + blocks = ( + JsonHandler() + .normalize(b'{"user@example.com":"safe"}', PolicyConfig()) + .text_blocks + ) + + assert blocks == ( + TextBlock( + path="#key:/user@example.com", + text="user@example.com", + replaceable=False, + ), + TextBlock(path="/user@example.com", text="safe"), + ) + + +def test_normalize_rejects_invalid_utf8() -> None: + sensitive_text = "distinctive-sensitive-utf8-8472" + + with pytest.raises(PrivacyGuardError) as exception_info: + JsonHandler().normalize( + f'"{sensitive_text}'.encode() + b'\xff"', PolicyConfig() + ) + + _assert_safe_error( + exception_info.value, ErrorCode.BODY_ENCODING_INVALID, sensitive_text + ) + + +def test_normalize_rejects_invalid_json() -> None: + sensitive_text = "distinctive-sensitive-json-8472" + + with pytest.raises(PrivacyGuardError) as exception_info: + JsonHandler().normalize( + f'{{"message":"{sensitive_text}",}}'.encode(), PolicyConfig() + ) + + _assert_safe_error( + exception_info.value, ErrorCode.BODY_JSON_INVALID, sensitive_text + ) + + +def test_normalize_shape_failure_drops_sensitive_exception_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import privacy_guard.body.json as json_module + + sensitive_text = "distinctive-sensitive-shape-8472" + monkeypatch.setattr(json_module, "MAX_SCANNED_CHARACTERS", len(sensitive_text) - 1) + + with pytest.raises(PrivacyGuardError) as exception_info: + JsonHandler().normalize(f'"{sensitive_text}"'.encode(), PolicyConfig()) + + _assert_safe_error( + exception_info.value, + ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED, + sensitive_text, + ) + + +@pytest.mark.parametrize("non_finite_value", [b"NaN", b"Infinity", b"-Infinity"]) +def test_normalize_rejects_non_standard_numeric_constants( + non_finite_value: bytes, +) -> None: + with pytest.raises(PrivacyGuardError) as exception_info: + JsonHandler().normalize(b'{"number":' + non_finite_value + b"}", PolicyConfig()) + + assert exception_info.value.code is ErrorCode.BODY_JSON_INVALID + + +def test_normalize_rejects_finite_syntax_that_overflows_to_infinity() -> None: + with pytest.raises(PrivacyGuardError) as exception_info: + JsonHandler().normalize(b'{"number":1e400}', PolicyConfig()) + + assert exception_info.value.code is ErrorCode.BODY_JSON_INVALID + + +def test_normalize_maps_python_huge_integer_limit_to_safe_json_error() -> None: + digit_limit = sys.int_info.default_max_str_digits + raw_body = b'{"number":' + (b"9" * (digit_limit + 1)) + b"}" + + with pytest.raises(PrivacyGuardError) as exception_info: + JsonHandler().normalize(raw_body, PolicyConfig()) + + assert exception_info.value.code is ErrorCode.BODY_JSON_INVALID + assert "999999" not in str(exception_info.value) + + +def test_normalize_rejects_duplicate_object_members_without_leaking_content() -> None: + sensitive_text = "distinctive-duplicate-value-8472" + + with pytest.raises(PrivacyGuardError) as exception_info: + JsonHandler().normalize( + (f'{{"message":"{sensitive_text}","message":"benign"}}').encode(), + PolicyConfig(), + ) + + assert exception_info.value.code is ErrorCode.BODY_JSON_INVALID + assert sensitive_text not in str(exception_info.value) + assert sensitive_text not in repr(exception_info.value) + + +def test_normalize_rejects_unpaired_surrogates_without_leaking_content() -> None: + sensitive_text = "distinctive-surrogate-neighbor-8472" + + with pytest.raises(PrivacyGuardError) as exception_info: + JsonHandler().normalize( + f'{{"message":"{sensitive_text}","invalid":"\\ud800"}}'.encode(), + PolicyConfig(), + ) + + _assert_safe_error( + exception_info.value, + ErrorCode.BODY_JSON_INVALID, + sensitive_text, + ) + + +def test_normalize_rejects_unpaired_surrogate_object_key_without_leaking_content() -> ( + None +): + sensitive_text = "distinctive-surrogate-key-neighbor-8472" + + with pytest.raises(PrivacyGuardError) as exception_info: + JsonHandler().normalize( + f'{{"{sensitive_text}":"safe","\\ud800":"invalid"}}'.encode(), + PolicyConfig(), + ) + + _assert_safe_error( + exception_info.value, + ErrorCode.BODY_JSON_INVALID, + sensitive_text, + ) + + +def test_reconstruct_without_replacements_returns_exact_original_bytes() -> None: + raw_body = b'{\n "message": "hello", "escaped": "\\u263a"\n}\n ' + json_handler = JsonHandler() + request_body = json_handler.normalize(raw_body, PolicyConfig()) + + assert json_handler.reconstruct(request_body, {}) is raw_body + + +def test_reconstruct_replaces_one_string_leaf() -> None: + json_handler = JsonHandler() + request_body = json_handler.normalize( + b'{"message":"hello","count":2}', PolicyConfig() + ) + + reconstructed_body = json_handler.reconstruct(request_body, {"/message": "goodbye"}) + + assert json.loads(reconstructed_body) == {"message": "goodbye", "count": 2} + assert reconstructed_body == b'{"message":"goodbye","count":2}' + + +def test_reconstruct_with_replacements_deepcopies_state_exactly_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + copy_count = 0 + + def counting_deepcopy(value: object) -> object: + nonlocal copy_count + copy_count += 1 + return deepcopy(value) + + json_handler = JsonHandler() + request_body = json_handler.normalize( + b'{"first":"one","nested":{"second":"two"}}', PolicyConfig() + ) + monkeypatch.setattr(json_module, "deepcopy", counting_deepcopy) + + reconstructed_body = json_handler.reconstruct( + request_body, {"/first": "ONE", "/nested/second": "TWO"} + ) + + assert json.loads(reconstructed_body) == { + "first": "ONE", + "nested": {"second": "TWO"}, + } + assert copy_count == 1 + + +@pytest.mark.parametrize( + ("raw_body", "replacements_by_path", "expected_value"), + [ + ( + b'{"first":"one","nested":{"second":"two"}}', + {"/first": "ONE", "/nested/second": "TWO"}, + {"first": "ONE", "nested": {"second": "TWO"}}, + ), + ( + b'{"message":"hello","other":"keep"}', + {"/message": ""}, + {"message": "", "other": "keep"}, + ), + (b'"root"', {"": "replaced"}, "replaced"), + ( + b'{"a/b":"slash","til~de":"tilde"}', + {"/a~1b": "new slash", "/til~0de": "new tilde"}, + {"a/b": "new slash", "til~de": "new tilde"}, + ), + ( + '{"message":"こんにちは"}'.encode(), + {"/message": "さようなら"}, + {"message": "さようなら"}, + ), + ], +) +def test_reconstruct_applies_explicit_string_replacements( + raw_body: bytes, + replacements_by_path: dict[str, str], + expected_value: object, +) -> None: + json_handler = JsonHandler() + request_body = json_handler.normalize(raw_body, PolicyConfig()) + + reconstructed_body = json_handler.reconstruct(request_body, replacements_by_path) + + assert json.loads(reconstructed_body) == expected_value + assert b"\\u3055" not in reconstructed_body + + +@pytest.mark.parametrize( + ("raw_body", "json_pointer"), + [ + (b'{"message":"secret-8472"}', "/unknown"), + (b'{"message":"secret-8472"}', "message"), + (b'{"~bad":"secret-8472"}', "/~2bad"), + (b'["secret-8472"]', "/-"), + (b'["secret-8472"]', "/01"), + (b'["secret-8472"]', "/1"), + (b'{"count":2}', "/count"), + (b'{"message":"secret-8472"}', ""), + ], +) +def test_reconstruct_rejects_invalid_or_non_string_targets_without_leaking_content( + raw_body: bytes, json_pointer: str +) -> None: + json_handler = JsonHandler() + request_body = json_handler.normalize(raw_body, PolicyConfig()) + + with pytest.raises(PrivacyGuardError) as exception_info: + json_handler.reconstruct(request_body, {json_pointer: "replacement"}) + + assert exception_info.value.code is ErrorCode.BODY_RECONSTRUCTION_INVALID + assert "secret-8472" not in str(exception_info.value) + assert "secret-8472" not in repr(exception_info.value) + assert exception_info.value.__cause__ is None + assert all( + "secret-8472" not in str(argument) for argument in exception_info.value.args + ) + + +@pytest.mark.parametrize("parsed_value", [{"edit": "original"}, object()]) +@pytest.mark.parametrize("replacements_by_path", [{}, {"/edit": "changed"}]) +def test_reconstruct_rejects_foreign_state_without_leaking_content( + parsed_value: object, + replacements_by_path: dict[str, str], +) -> None: + sensitive_text = "distinctive-sensitive-foreign-state-8472" + request_body = RequestBody( + text_blocks=(TextBlock(path="/edit", text="original"),), + parsed_value=parsed_value, + original_bytes=sensitive_text.encode(), + ) + + with pytest.raises(PrivacyGuardError) as exception_info: + JsonHandler().reconstruct(request_body, replacements_by_path) + + _assert_safe_error( + exception_info.value, + ErrorCode.BODY_RECONSTRUCTION_INVALID, + sensitive_text, + ) + + +def test_reconstruct_copy_failure_drops_sensitive_exception_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sensitive_text = "distinctive-sensitive-copy-8472" + json_handler = JsonHandler() + request_body = json_handler.normalize(b'{"edit":"original"}', PolicyConfig()) + + def fail_copy(*values: object) -> None: + raise RuntimeError(sensitive_text) + + monkeypatch.setattr(json_module, "deepcopy", fail_copy) + + with pytest.raises(PrivacyGuardError) as exception_info: + json_handler.reconstruct(request_body, {"/edit": "changed"}) + + _assert_safe_error( + exception_info.value, + ErrorCode.BODY_RECONSTRUCTION_INVALID, + sensitive_text, + ) + + +def test_reconstruct_serialization_failure_drops_sensitive_exception_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sensitive_text = "distinctive-sensitive-serialization-8472" + json_handler = JsonHandler() + request_body = json_handler.normalize(b'{"edit":"original"}', PolicyConfig()) + + def fail_serialization( + value: object, + *, + ensure_ascii: bool, + separators: tuple[str, str], + allow_nan: bool, + ) -> str: + raise TypeError(sensitive_text) + + monkeypatch.setattr(json_module.json, "dumps", fail_serialization) + + with pytest.raises(PrivacyGuardError) as exception_info: + json_handler.reconstruct(request_body, {"/edit": "changed"}) + + _assert_safe_error( + exception_info.value, + ErrorCode.BODY_RECONSTRUCTION_INVALID, + sensitive_text, + ) + + +def test_reconstruct_rejects_unpaired_surrogate_replacement_without_leaking_body() -> ( + None +): + sensitive_text = "distinctive-reconstruction-neighbor-8472" + json_handler = JsonHandler() + request_body = json_handler.normalize( + f'{{"edit":"original","neighbor":"{sensitive_text}"}}'.encode(), + PolicyConfig(), + ) + + with pytest.raises(PrivacyGuardError) as exception_info: + json_handler.reconstruct(request_body, {"/edit": "\ud800"}) + + assert exception_info.value.code is ErrorCode.BODY_RECONSTRUCTION_INVALID + assert sensitive_text not in str(exception_info.value) + assert sensitive_text not in repr(exception_info.value) + + +def test_reconstruct_does_not_mutate_original_parsed_value() -> None: + json_handler = JsonHandler() + request_body = json_handler.normalize( + b'{"items":[{"message":"original"}],"other":"keep"}', PolicyConfig() + ) + original_value = {"items": [{"message": "original"}], "other": "keep"} + + json_handler.reconstruct(request_body, {"/items/0/message": "changed"}) + + assert type(request_body.parsed_value) is json_module._JsonBodyState + assert _json_state_value(request_body) == original_value + + +def test_select_handler_returns_registered_json_singleton() -> None: + json_handler = select_handler("json") + + assert isinstance(json_handler, JsonHandler) + assert select_handler("json") is json_handler + + +def test_select_handler_rejects_unknown_kind_without_fallback() -> None: + sentinel = "unknown-sensitive-format-8472" + + with pytest.raises(PrivacyGuardError) as exception_info: + select_handler(sentinel) + + assert exception_info.value.code is ErrorCode.BODY_FORMAT_UNSUPPORTED + assert sentinel not in str(exception_info.value) + assert sentinel not in repr(exception_info.value) diff --git a/projects/privacy-guard/tests/examples/test_openshell_e2e.py b/projects/privacy-guard/tests/examples/test_openshell_e2e.py new file mode 100644 index 0000000..16991e9 --- /dev/null +++ b/projects/privacy-guard/tests/examples/test_openshell_e2e.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import os +import socket +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +import pytest + +EXAMPLE_DIRECTORY = Path(__file__).parents[2] / "examples" / "openshell-e2e" + + +def test_capture_server_persists_exact_post_body(tmp_path: Path) -> None: + output_path = tmp_path / "captured.json" + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + port = reservation.getsockname()[1] + process = subprocess.Popen( + [ + sys.executable, + str(EXAMPLE_DIRECTORY / "capture_server.py"), + "--host", + "127.0.0.1", + "--port", + str(port), + "--output", + str(output_path), + ] + ) + request_body = b'{"message":"exact bytes"}' + + try: + for _ in range(40): + try: + request = urllib.request.Request( + f"http://127.0.0.1:{port}/capture", + data=request_body, + method="POST", + ) + with urllib.request.urlopen(request, timeout=1) as response: + assert response.read() == b'{"captured":true}' + break + except OSError: + time.sleep(0.05) + else: + raise AssertionError("capture server did not become ready") + + assert output_path.read_bytes() == request_body + assert process.wait(timeout=2) == 0 + finally: + process.terminate() + process.wait(timeout=2) + + +def test_harness_shell_is_valid_and_policy_targets_privacy_guard() -> None: + subprocess.run( + ["bash", "-n", str(EXAMPLE_DIRECTORY / "run.sh")], + check=True, + ) + policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() + + assert "middleware: privacy-guard-e2e" in policy + assert "on_finding:" in policy + assert "action: redact" in policy + assert "entity_types: [email]" in policy + harness = (EXAMPLE_DIRECTORY / "run.sh").read_text() + assert 'python "${SCRIPT_DIR}/middleware_server.py"' in harness + assert "base@sha256:" in harness + assert "base:latest" not in harness + + +@pytest.mark.skipif( + os.environ.get("PRIVACY_GUARD_RUN_OPEN_SHELL_E2E") != "1", + reason="state-mutating OpenShell harness is explicitly opt-in", +) +def test_opt_in_full_openshell_harness() -> None: + subprocess.run([str(EXAMPLE_DIRECTORY / "run.sh")], check=True) diff --git a/projects/privacy-guard/tests/scanner_helpers.py b/projects/privacy-guard/tests/scanner_helpers.py new file mode 100644 index 0000000..845ae6e --- /dev/null +++ b/projects/privacy-guard/tests/scanner_helpers.py @@ -0,0 +1,33 @@ +"""Deterministic scanners used only by Privacy Guard tests.""" + +from __future__ import annotations + +import re + +from typing_extensions import override + +from privacy_guard.scanners import ( + Confidence, + Finding, + Scanner, + ScannerConfig, +) + + +class DeterministicEmailScanner(Scanner[ScannerConfig]): + _EMAIL = re.compile( + r"(? tuple[Finding, ...]: + return tuple( + Finding( + entity="email", + scanner_name=self.scanner_name, + start_offset=match.start(), + end_offset=match.end(), + confidence=Confidence.HIGH, + ) + for match in self._EMAIL.finditer(text_block) + ) diff --git a/projects/privacy-guard/tests/scanners/test_passthrough.py b/projects/privacy-guard/tests/scanners/test_passthrough.py new file mode 100644 index 0000000..ccdf478 --- /dev/null +++ b/projects/privacy-guard/tests/scanners/test_passthrough.py @@ -0,0 +1,155 @@ +import pytest +from pydantic import ValidationError + +from privacy_guard.scanners import ( + Finding, + PassthroughScanner, + RequestBodyFinding, + ScannerConfig, +) + + +def test_finding_names_the_scanner_that_produced_it() -> None: + finding = Finding( + entity="email", + scanner_name="example-scanner", + start_offset=4, + end_offset=18, + ) + + assert finding.scanner_name == "example-scanner" + + +def test_passthrough_scanner_has_stable_name() -> None: + scanner = PassthroughScanner() + + assert scanner.scanner_name == "passthrough" + assert scanner.supported_entity_types == frozenset() + + +def test_scanner_config_owns_identity_and_entity_types() -> None: + config = ScannerConfig(name="s" * 2048, entity_types=frozenset({"email", "phone"})) + + assert config.name == "s" * 2048 + assert config.entity_types == frozenset({"email", "phone"}) + with pytest.raises(ValidationError): + setattr(config, "name", "changed") + + +def test_passthrough_uses_minimal_config_with_fixed_empty_catalog() -> None: + scanner = PassthroughScanner() + + assert type(scanner.config) is ScannerConfig + assert scanner.supported_entity_types == frozenset() + + +@pytest.mark.parametrize( + "values", + [ + {"name": "", "entity_types": frozenset()}, + {"name": 3, "entity_types": frozenset()}, + {"name": "scanner"}, + {"name": "scanner", "entity_types": "email"}, + {"name": "scanner", "entity_types": frozenset({3})}, + ], +) +def test_scanner_config_rejects_invalid_metadata(values: dict[str, object]) -> None: + with pytest.raises(ValidationError): + ScannerConfig.model_validate(values) + + +@pytest.mark.parametrize( + "values", + [ + {"entity": "", "scanner_name": "scanner", "start_offset": 0, "end_offset": 1}, + {"entity": "email", "scanner_name": "", "start_offset": 0, "end_offset": 1}, + { + "entity": "\U0001f4a3" * 257, + "scanner_name": "scanner", + "start_offset": 0, + "end_offset": 1, + }, + { + "entity": "email", + "scanner_name": "\U0001f4a3" * 257, + "start_offset": 0, + "end_offset": 1, + }, + { + "entity": "email", + "scanner_name": "scanner", + "start_offset": True, + "end_offset": 1, + }, + { + "entity": "email", + "scanner_name": "scanner", + "start_offset": -1, + "end_offset": 1, + }, + { + "entity": "email", + "scanner_name": "scanner", + "start_offset": 1, + "end_offset": 1, + }, + { + "entity": "email", + "scanner_name": "scanner", + "start_offset": 0, + "end_offset": 1, + "confidence": "high", + }, + { + "entity": "email", + "scanner_name": "scanner", + "start_offset": 0, + "end_offset": 1, + "text_block_path": "path", + }, + ], +) +def test_finding_rejects_invalid_or_extra_fields(values: dict[str, object]) -> None: + with pytest.raises(ValidationError): + Finding.model_validate(values) + + +def test_request_body_finding_requires_hidden_path_and_models_are_frozen() -> None: + finding = Finding( + entity="email", scanner_name="scanner", start_offset=0, end_offset=1 + ) + request_body_finding = RequestBodyFinding( + finding=finding, text_block_path="sensitive/path" + ) + + assert request_body_finding.text_block_path == "sensitive/path" + assert "sensitive/path" not in repr(request_body_finding) + assert not hasattr(finding, "text_block_path") + with pytest.raises(ValidationError): + setattr(finding, "entity", "token") + with pytest.raises(ValidationError): + RequestBodyFinding.model_validate( + { + "finding": finding, + } + ) + + +@pytest.mark.parametrize( + "text_block", + [ + "", + "ordinary ASCII text", + "Unicode: café 🐍", + "first line\nsecond line", + '{"looks": "like JSON"}', + "Contact alice@example.com or call 555-0100", + ], +) +def test_passthrough_scanner_always_returns_immutable_empty_tuple( + text_block: str, +) -> None: + findings = PassthroughScanner().scan(text_block) + + assert findings == () + assert type(findings) is tuple diff --git a/projects/privacy-guard/tests/service/__init__.py b/projects/privacy-guard/tests/service/__init__.py new file mode 100644 index 0000000..c8634e6 --- /dev/null +++ b/projects/privacy-guard/tests/service/__init__.py @@ -0,0 +1 @@ +"""Privacy Guard service tests.""" diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py new file mode 100644 index 0000000..1160b2d --- /dev/null +++ b/projects/privacy-guard/tests/service/test_server.py @@ -0,0 +1,323 @@ +from collections.abc import Sequence + +import grpc +import pytest +from google.protobuf import empty_pb2, message_factory +from google.protobuf.message import Message +from typing_extensions import override + +from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 +from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc +from privacy_guard.constants import MAX_BODY_BYTES +from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.processor import RequestProcessor +from privacy_guard.scanners import ScannerConfig +from privacy_guard.service import server as server_module +from privacy_guard.service.server import MiddlewareServer, create_server, main, serve +from privacy_guard.service.servicer import PrivacyGuardMiddleware + +from ..scanner_helpers import DeterministicEmailScanner + + +class LifecycleServerFake(grpc.aio.Server): + """Nominal gRPC server fake for lifecycle-only server tests.""" + + def __init__(self) -> None: + self.stopped = False + + @override + def add_generic_rpc_handlers( + self, generic_rpc_handlers: Sequence[grpc.GenericRpcHandler] + ) -> None: + raise AssertionError("generic handler registration is not under test") + + @override + def add_insecure_port(self, address: str) -> int: + raise NotImplementedError + + @override + def add_secure_port( + self, address: str, server_credentials: grpc.ServerCredentials + ) -> int: + raise AssertionError("secure binding is not under test") + + @override + async def start(self) -> None: + raise NotImplementedError + + @override + async def stop(self, grace: float | None) -> None: + self.stopped = True + + @override + async def wait_for_termination(self, timeout: float | None = None) -> bool: + raise NotImplementedError + + +def test_middleware_server_wires_scanner_and_has_a_default_listen_address( + monkeypatch: pytest.MonkeyPatch, +) -> None: + served: list[tuple[str, PrivacyGuardMiddleware]] = [] + + async def record_serve( + listen: str, servicer: PrivacyGuardMiddleware | None = None + ) -> None: + assert servicer is not None + served.append((listen, servicer)) + + monkeypatch.setattr(server_module, "serve", record_serve) + scanner = DeterministicEmailScanner( + ScannerConfig(name="test_email", entity_types=frozenset({"email"})) + ) + + MiddlewareServer(scanner=scanner).serve() + + assert len(served) == 1 + assert served[0][0] == "127.0.0.1:50051" + assert isinstance(served[0][1], PrivacyGuardMiddleware) + + +@pytest.mark.asyncio +async def test_create_server_accepts_injected_servicer_and_serves_loopback_rpcs() -> ( + None +): + class FalseyMiddleware(PrivacyGuardMiddleware): + def __bool__(self) -> bool: + return False + + @override + async def Describe( + self, + request: object, + context: grpc.aio.ServicerContext[object, pb2.MiddlewareManifest], + ) -> pb2.MiddlewareManifest: + manifest = await super().Describe(request, context) + manifest.name = "injected-falsey-middleware" + return manifest + + scanner = DeterministicEmailScanner( + ScannerConfig(name="test_email", entity_types=frozenset({"email"})) + ) + middleware = FalseyMiddleware(RequestProcessor([scanner])) + server = create_server(middleware) + port = server.add_insecure_port("127.0.0.1:0") + assert port != 0 + await server.start() + + try: + async with grpc.aio.insecure_channel(f"127.0.0.1:{port}") as channel: + stub = pb2_grpc.SupervisorMiddlewareStub(channel) + empty_message_type: object = message_factory.GetMessageClass( + empty_pb2.DESCRIPTOR.message_types_by_name["Empty"] + ) + if not isinstance(empty_message_type, type): + raise AssertionError("Empty factory returned a non-type") + empty_message: object = empty_message_type() + if not isinstance(empty_message, Message): + raise AssertionError("Empty factory returned a non-message") + if ( + empty_message.DESCRIPTOR + is not empty_pb2.DESCRIPTOR.message_types_by_name["Empty"] + ): + raise AssertionError("Empty factory returned the wrong message type") + manifest = await stub.Describe(empty_message) + result = await stub.EvaluateHttpRequest( + pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + body=b'{"message":"user@example.com"}', + ) + ) + + assert manifest.name == "injected-falsey-middleware" + assert result.decision == pb2.DECISION_ALLOW + assert result.has_body is True + assert result.body == b'{"message":"[email]"}' + finally: + await server.stop(grace=0) + await middleware.close() + + +@pytest.mark.asyncio +async def test_loopback_accepts_body_at_advertised_limit_and_rejects_larger_body() -> ( + None +): + middleware = PrivacyGuardMiddleware() + server = create_server(middleware) + port = server.add_insecure_port("127.0.0.1:0") + assert port != 0 + await server.start() + + try: + async with grpc.aio.insecure_channel(f"127.0.0.1:{port}") as channel: + stub = pb2_grpc.SupervisorMiddlewareStub(channel) + at_limit = b'"' + (b"x" * (MAX_BODY_BYTES - 2)) + b'"' + allowed = await stub.EvaluateHttpRequest( + pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + body=at_limit, + ) + ) + + assert allowed.decision == pb2.DECISION_ALLOW + + with pytest.raises(grpc.aio.AioRpcError) as exception_info: + await stub.EvaluateHttpRequest( + pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + body=at_limit + b" ", + ) + ) + + assert exception_info.value.code() is grpc.StatusCode.INVALID_ARGUMENT + details = exception_info.value.details() + assert details is not None + assert ErrorCode.REQUEST_BODY_TOO_LARGE.value in details + finally: + await server.stop(grace=0) + await middleware.close() + + +@pytest.mark.asyncio +async def test_loopback_real_findings_cover_observe_redact_and_block() -> None: + scanner = DeterministicEmailScanner( + ScannerConfig(name="test_email", entity_types=frozenset({"email"})) + ) + middleware = PrivacyGuardMiddleware(RequestProcessor([scanner])) + server = create_server(middleware) + port = server.add_insecure_port("127.0.0.1:0") + await server.start() + body = b'{"first":"a@example.com","second":"b@example.com"}' + try: + async with grpc.aio.insecure_channel(f"127.0.0.1:{port}") as channel: + stub = pb2_grpc.SupervisorMiddlewareStub(channel) + observe = await stub.EvaluateHttpRequest( + pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + config={"on_finding": {"action": "observe"}}, + body=body, + ) + ) + redact = await stub.EvaluateHttpRequest( + pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + config={"on_finding": {"action": "redact"}}, + body=body, + ) + ) + block = await stub.EvaluateHttpRequest( + pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + config={"on_finding": {"action": "block"}}, + body=body, + ) + ) + + assert observe.decision == pb2.DECISION_ALLOW + assert not observe.has_body + assert [(item.type, item.label, item.count) for item in observe.findings] == [ + ("test_email", "email", 2) + ] + assert redact.decision == pb2.DECISION_ALLOW + assert redact.has_body + assert redact.body == b'{"first":"[email]","second":"[email]"}' + assert redact.findings[0].count == 2 + assert block.decision == pb2.DECISION_DENY + assert block.reason_code == "privacy_guard_blocked" + assert not block.has_body and block.body == b"" + assert block.findings[0].count == 2 + finally: + await server.stop(grace=0) + await middleware.close() + + +@pytest.mark.asyncio +async def test_serve_rejects_bind_failure_and_stops_server( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class BindFailureServer(LifecycleServerFake): + @override + def add_insecure_port(self, address: str) -> int: + return 0 + + @override + async def start(self) -> None: + raise AssertionError("start must not run after bind failure") + + @override + async def wait_for_termination(self, timeout: float | None = None) -> bool: + raise AssertionError("wait must not run after bind failure") + + fake_server = BindFailureServer() + monkeypatch.setattr( + server_module, + "create_server", + lambda _: fake_server, + ) + + with pytest.raises(PrivacyGuardError) as exception_info: + await serve("invalid-sensitive-listen-8472") + + assert exception_info.value.code is ErrorCode.SERVER_BIND_FAILED + assert "Hint:" in str(exception_info.value) + assert "8472" not in str(exception_info.value) + assert fake_server.stopped is True + + +@pytest.mark.asyncio +async def test_startup_failure_stops_server_and_propagates( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class StartFailureServer(LifecycleServerFake): + @override + def add_insecure_port(self, address: str) -> int: + return 12345 + + @override + async def start(self) -> None: + raise RuntimeError("startup failed") + + @override + async def wait_for_termination(self, timeout: float | None = None) -> bool: + raise AssertionError("wait must not run after startup failure") + + fake_server = StartFailureServer() + monkeypatch.setattr( + server_module, + "create_server", + lambda _: fake_server, + ) + + with pytest.raises(RuntimeError, match="startup failed"): + await serve("127.0.0.1:12345") + + assert fake_server.stopped is True + + +def test_main_parses_custom_and_default_listen_without_permanent_server( + monkeypatch: pytest.MonkeyPatch, +) -> None: + listens: list[str] = [] + + def recording_serve(server: MiddlewareServer, listen: str) -> None: + listens.append(listen) + + monkeypatch.setattr(MiddlewareServer, "serve", recording_serve) + + assert main(["--listen", "127.0.0.1:54321"]) == 0 + assert main([]) == 0 + assert listens == ["127.0.0.1:54321", "127.0.0.1:50051"] + + +def test_main_accepts_a_general_sequence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + received: list[str] = [] + + def recording_serve(server: MiddlewareServer, listen: str) -> None: + received.append(listen) + + monkeypatch.setattr(MiddlewareServer, "serve", recording_serve) + argv: Sequence[str] = ("--listen", "127.0.0.1:50052") + + assert main(argv) == 0 + assert received == ["127.0.0.1:50052"] diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py new file mode 100644 index 0000000..ce10bf4 --- /dev/null +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -0,0 +1,474 @@ +import json +import traceback +from collections.abc import Mapping +from typing import Never + +import grpc +import pytest +from pydantic import ValidationError +from typing_extensions import override + +import privacy_guard.service.servicer as servicer_module +from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 +from privacy_guard.config import PolicyAction, PolicyConfig +from privacy_guard.constants import ( + BLOCK_REASON, + BLOCK_REASON_CODE, + MAX_BODY_BYTES, + SERVICE_NAME, + SERVICE_VERSION, +) +from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.payloads import ( + InterceptedRequest, + ProcessingDecision, + ProcessingResult, +) +from privacy_guard.processor import RequestProcessor +from privacy_guard.scanners import Finding, RequestBodyFinding, Scanner, ScannerConfig +from privacy_guard.service.servicer import PrivacyGuardMiddleware + +from ..scanner_helpers import DeterministicEmailScanner + + +class AbortedRpc(Exception): + pass + + +class RecordingContext: + def __init__(self) -> None: + self.code: grpc.StatusCode | None = None + self.details: str | None = None + + async def abort(self, code: grpc.StatusCode, details: str) -> Never: + self.code = code + self.details = details + raise AbortedRpc + + +class FakeProcessor: + def __init__(self, result: ProcessingResult | None = None) -> None: + self.result = result or ProcessingResult(decision=ProcessingDecision.ALLOW) + self.validated_configs: list[PolicyConfig] = [] + self.requests: list[InterceptedRequest] = [] + + def __bool__(self) -> bool: + return False + + def validate_policy_config(self, policy_config: PolicyConfig) -> None: + self.validated_configs.append(policy_config) + + def process(self, request: InterceptedRequest) -> ProcessingResult: + self.requests.append(request) + return self.result + + +def _assert_safe_translation(error: PrivacyGuardError, sentinel: str) -> None: + assert error.__cause__ is None + assert sentinel not in str(error) + assert sentinel not in repr(error) + assert sentinel not in repr(error.args) + assert sentinel not in "".join(traceback.format_exception(error)) + + +def _evaluation( + *, + body: bytes = b'{"message":"hello"}', + config: Mapping[str, object] | None = None, + phase: pb2.SupervisorMiddlewarePhase = ( + pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS + ), + headers: list[pb2.HttpHeader] | None = None, +) -> pb2.HttpRequestEvaluation: + return pb2.HttpRequestEvaluation( + phase=phase, + config=config or {}, + headers=headers or [], + body=body, + ) + + +def test_describe_advertises_one_pre_credentials_http_binding() -> None: + manifest = PrivacyGuardMiddleware()._describe() + + assert manifest.name == SERVICE_NAME == "privacy-guard" + assert manifest.service_version == SERVICE_VERSION == "0.1.0" + assert len(manifest.bindings) == 1 + binding = manifest.bindings[0] + assert binding.operation == pb2.SUPERVISOR_MIDDLEWARE_OPERATION_HTTP_REQUEST + assert binding.phase == pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS + assert binding.max_body_bytes == MAX_BODY_BYTES == 4 * 1024 * 1024 + assert binding.timeout == "" + + +@pytest.mark.parametrize("action", [None, *PolicyAction]) +def test_validate_config_accepts_defaults_and_actions( + action: PolicyAction | None, +) -> None: + values: dict[str, object] = {} + if action is not None: + values["on_finding"] = {"action": action.value} + + response = PrivacyGuardMiddleware()._validate_config( + pb2.ValidateConfigRequest(config=values) + ) + + assert response.valid is True + assert response.reason == "" + + +@pytest.mark.parametrize( + "config", + [ + {"unknown": "sensitive-config-value-8472"}, + {"on_finding": {"action": "invalid-sensitive-action-8472"}}, + {"debug_inject_text": "sensitive-injection-value-8472"}, + {"body_format": "unknown-sensitive-format-8472"}, + { + "on_finding": { + "action": "redact", + "entity_types": ["sensitive-entity-typo-8472"], + } + }, + ], +) +def test_validate_config_rejects_malformed_values_without_echoing_them( + config: Mapping[str, object], +) -> None: + response = PrivacyGuardMiddleware()._validate_config( + pb2.ValidateConfigRequest(config=config) + ) + + assert response.valid is False + assert "Hint:" in response.reason + assert "8472" not in response.reason + + +def test_proto_policy_translation_discards_sensitive_exception_chain( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sentinel = "sensitive-proto-config-8472" + + def reject_proto(config: object) -> dict[str, object]: + raise ValueError(sentinel) + + monkeypatch.setattr(servicer_module.json_format, "MessageToDict", reject_proto) + + with pytest.raises(PrivacyGuardError) as exception_info: + servicer_module._policy_from_proto(pb2.ValidateConfigRequest().config) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + _assert_safe_translation(exception_info.value, sentinel) + + +def test_intercepted_request_translation_discards_sensitive_validation_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sentinel = "sensitive-intercepted-request-8472" + validation_error = ValidationError.from_exception_data( + "InterceptedRequest", + [ + { + "type": "value_error", + "loc": ("raw_body",), + "input": sentinel, + "ctx": {"error": ValueError(sentinel)}, + } + ], + ) + + def reject_request(**values: object) -> Never: + raise validation_error + + monkeypatch.setattr(servicer_module, "InterceptedRequest", reject_request) + + with pytest.raises(PrivacyGuardError) as exception_info: + servicer_module._request_from_proto(_evaluation()) + + assert exception_info.value.code is ErrorCode.UNEXPECTED_SERVICE_FAILURE + _assert_safe_translation(exception_info.value, sentinel) + + +def test_validate_config_uses_domain_parser_and_processor_validation() -> None: + processor = FakeProcessor() + servicer = PrivacyGuardMiddleware(processor) + + response = servicer._validate_config( + pb2.ValidateConfigRequest(config={"on_finding": {"action": "observe"}}) + ) + + assert response.valid is True + assert processor.validated_configs[0].on_finding.action is PolicyAction.OBSERVE + + +@pytest.mark.asyncio +async def test_passthrough_and_bodyless_requests_allow_without_replacement() -> None: + servicer = PrivacyGuardMiddleware() + + ordinary = await servicer._evaluate_http_request(_evaluation()) + bodyless = await servicer._evaluate_http_request(_evaluation(body=b"")) + + for response in (ordinary, bodyless): + assert response.decision == pb2.DECISION_ALLOW + assert response.has_body is False + assert response.body == b"" + assert response.reason == "" + assert response.reason_code == "" + + +@pytest.mark.asyncio +async def test_body_larger_than_advertised_limit_is_rejected_before_processing() -> ( + None +): + processor = FakeProcessor() + context = RecordingContext() + servicer = PrivacyGuardMiddleware(processor) + + with pytest.raises(AbortedRpc): + await servicer._evaluate_rpc( + _evaluation(body=b"x" * (MAX_BODY_BYTES + 1)), + context, + ) + + assert context.code is grpc.StatusCode.INVALID_ARGUMENT + assert context.details is not None + assert ErrorCode.REQUEST_BODY_TOO_LARGE.value in context.details + assert processor.requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("body", "expected"), + [ + ( + b'{"messages":[{"content":"user@example.com"}]}', + {"messages": [{"content": "[email]"}]}, + ), + ( + b'{"contents":[{"parts":[{"text":"user@example.com"}]}]}', + {"contents": [{"parts": [{"text": "[email]"}]}]}, + ), + ], +) +async def test_deterministic_finding_crosses_full_proto_domain_boundary( + body: bytes, expected: object +) -> None: + response = await PrivacyGuardMiddleware( + RequestProcessor( + [ + DeterministicEmailScanner( + ScannerConfig(name="test_email", entity_types=frozenset({"email"})) + ) + ] + ) + )._evaluate_http_request(_evaluation(body=body)) + + assert response.decision == pb2.DECISION_ALLOW + assert response.has_body is True + assert json.loads(response.body) == expected + + +@pytest.mark.asyncio +async def test_first_content_type_header_is_passed_to_processor() -> None: + processor = FakeProcessor() + servicer = PrivacyGuardMiddleware(processor) + + await servicer._evaluate_http_request( + _evaluation( + headers=[ + pb2.HttpHeader(name="content-type", value="first/type"), + pb2.HttpHeader(name="content-type", value="second/type"), + ] + ) + ) + + assert processor.requests[0].content_type == "first/type" + + +@pytest.mark.asyncio +async def test_explicit_empty_replacement_sets_has_body() -> None: + processor = FakeProcessor( + ProcessingResult(decision=ProcessingDecision.ALLOW, replacement_body=b"") + ) + + response = await PrivacyGuardMiddleware(processor)._evaluate_http_request( + _evaluation() + ) + + assert response.decision == pb2.DECISION_ALLOW + assert response.has_body is True + assert response.body == b"" + + +@pytest.mark.asyncio +async def test_deny_has_stable_reason_code_and_never_returns_a_body() -> None: + processor = FakeProcessor( + ProcessingResult( + decision=ProcessingDecision.DENY, + replacement_body=None, + reason_code=BLOCK_REASON_CODE, + ) + ) + + response = await PrivacyGuardMiddleware(processor)._evaluate_http_request( + _evaluation() + ) + + assert response.decision == pb2.DECISION_DENY + assert response.reason == BLOCK_REASON + assert response.reason_code == BLOCK_REASON_CODE + assert response.has_body is False + assert response.body == b"" + + +@pytest.mark.asyncio +async def test_findings_aggregate_in_first_observed_group_order() -> None: + findings = tuple( + RequestBodyFinding(finding=finding, text_block_path=path) + for finding, path in ( + ( + Finding( + entity="email", + scanner_name="scanner-a", + start_offset=0, + end_offset=1, + ), + "a", + ), + ( + Finding( + entity="token", + scanner_name="scanner-b", + start_offset=1, + end_offset=2, + ), + "a", + ), + ( + Finding( + entity="email", + scanner_name="scanner-a", + start_offset=2, + end_offset=3, + ), + "b", + ), + ) + ) + processor = FakeProcessor( + ProcessingResult(decision=ProcessingDecision.ALLOW, findings=findings) + ) + + response = await PrivacyGuardMiddleware(processor)._evaluate_http_request( + _evaluation() + ) + + assert [ + (finding.type, finding.label, finding.count) for finding in response.findings + ] == [ + ("scanner-a", "email", 2), + ("scanner-b", "token", 1), + ] + assert all(finding.confidence == "high" for finding in response.findings) + assert all(not finding.severity for finding in response.findings) + + +@pytest.mark.asyncio +async def test_more_than_32_finding_groups_stably_denies() -> None: + findings = tuple( + RequestBodyFinding( + finding=Finding( + entity=f"entity-{index}", + scanner_name="scanner", + start_offset=0, + end_offset=1, + ), + text_block_path="path", + ) + for index in range(33) + ) + processor = FakeProcessor( + ProcessingResult(decision=ProcessingDecision.ALLOW, findings=findings) + ) + response = await PrivacyGuardMiddleware(processor)._evaluate_http_request( + _evaluation() + ) + + assert response.decision == pb2.DECISION_DENY + assert response.reason_code == "privacy_guard_limit_exceeded" + assert not response.findings + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("evaluation", "expected_code"), + [ + ( + _evaluation(phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_UNSPECIFIED), + ErrorCode.REQUEST_PHASE_INVALID, + ), + (_evaluation(body=b'"bad\xff"'), ErrorCode.BODY_ENCODING_INVALID), + (_evaluation(body=b'{"bad":}'), ErrorCode.BODY_JSON_INVALID), + (_evaluation(config={"body_format": "xml"}), ErrorCode.BODY_FORMAT_UNSUPPORTED), + ], +) +async def test_invalid_input_maps_to_invalid_argument_with_safe_catalog_details( + evaluation: pb2.HttpRequestEvaluation, expected_code: ErrorCode +) -> None: + context = RecordingContext() + + with pytest.raises(AbortedRpc): + await PrivacyGuardMiddleware()._evaluate_rpc(evaluation, context) + + assert context.code is grpc.StatusCode.INVALID_ARGUMENT + assert context.details is not None + assert expected_code.value in context.details + assert "Hint:" in context.details + assert "8472" not in context.details + + +@pytest.mark.asyncio +async def test_processor_runtime_failure_maps_to_internal_catalog_error() -> None: + class RaisingProcessor(FakeProcessor): + @override + def process(self, request: InterceptedRequest) -> ProcessingResult: + raise RuntimeError("sensitive-runtime-failure-8472") + + context = RecordingContext() + + with pytest.raises(AbortedRpc): + await PrivacyGuardMiddleware(RaisingProcessor())._evaluate_rpc( + _evaluation(), context + ) + + assert context.code is grpc.StatusCode.INTERNAL + assert context.details is not None + assert ErrorCode.UNEXPECTED_SERVICE_FAILURE.value in context.details + assert "8472" not in context.details + + +@pytest.mark.asyncio +async def test_cataloged_scanner_failure_maps_to_internal_status() -> None: + class RaisingScanner(Scanner[ScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + raise RuntimeError("sensitive-scanner-failure-8472") + + context = RecordingContext() + scanner = RaisingScanner(ScannerConfig(name="raising", entity_types=frozenset())) + servicer = PrivacyGuardMiddleware(RequestProcessor([scanner])) + + with pytest.raises(AbortedRpc): + await servicer._evaluate_rpc(_evaluation(), context) + + assert context.code is grpc.StatusCode.INTERNAL + assert context.details is not None + assert ErrorCode.SCANNER_EXECUTION_FAILED.value in context.details + assert "8472" not in context.details + + +def test_servicer_repr_does_not_retain_request_or_config_content() -> None: + servicer = PrivacyGuardMiddleware() + + assert "sensitive-body-8472" not in repr(servicer) + assert "sensitive-injection-8472" not in repr(servicer) diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py new file mode 100644 index 0000000..676e34f --- /dev/null +++ b/projects/privacy-guard/tests/test_config.py @@ -0,0 +1,369 @@ +import pytest +from pydantic import ValidationError + +from privacy_guard.body import select_handler +from privacy_guard.config import ( + ActionConfig, + BlockActionConfig, + ObserveActionConfig, + PolicyAction, + PolicyActionConfig, + PolicyConfig, + RedactActionConfig, +) +from privacy_guard.constants import MAX_SCANNER_METADATA_BYTES +from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.scanners import Confidence +from privacy_guard.validation import ( + BoundedMetadataString, + NonEmptyScalarString, + ScalarString, + StrictDomainModel, +) + + +class _ValidationFixture(StrictDomainModel): + scalar: ScalarString + non_empty: NonEmptyScalarString + metadata: BoundedMetadataString + + +class _InvalidDefaultFixture(StrictDomainModel): + value: NonEmptyScalarString = "" + + +def _wire_on_finding(action: str, **values: object) -> dict[str, object]: + return {"on_finding": {"action": action, **values}} + + +def test_defaults_use_a_complete_redact_action() -> None: + config = PolicyConfig() + + assert config.body_format == "json" + assert type(config.on_finding) is RedactActionConfig + assert config.on_finding.action is PolicyAction.REDACT + assert config.on_finding.entity_types is None + assert config.on_finding.minimum_confidence is None + assert config.on_finding.template == "[{entity}]" + assert [action.value for action in PolicyAction] == ["observe", "redact", "block"] + + +@pytest.mark.parametrize( + ("action", "expected_type"), + [ + (PolicyAction.OBSERVE, ObserveActionConfig), + (PolicyAction.REDACT, RedactActionConfig), + (PolicyAction.BLOCK, BlockActionConfig), + ], +) +def test_wire_parser_selects_discriminated_action_model( + action: PolicyAction, expected_type: type[ActionConfig] +) -> None: + config = PolicyConfig.from_mapping(_wire_on_finding(action.value)) + + assert type(config.on_finding) is expected_type + assert config.on_finding.action is action + + +@pytest.mark.parametrize( + "action", + [ + ObserveActionConfig(), + BlockActionConfig( + entity_types=frozenset({"email"}), + minimum_confidence=Confidence.HIGH, + ), + RedactActionConfig(template="[{entity}]"), + ], +) +def test_discriminated_action_serialization_round_trips( + action: PolicyActionConfig, +) -> None: + serialized = PolicyConfig(on_finding=action).model_dump(mode="json") + reparsed = PolicyConfig.from_mapping(serialized) + + assert serialized["on_finding"]["action"] == action.action.value + assert type(reparsed.on_finding) is type(action) + assert reparsed == PolicyConfig(on_finding=action) + + +def test_on_finding_schema_declares_action_discriminator() -> None: + action_schema = PolicyConfig.model_json_schema()["properties"]["on_finding"] + + assert action_schema["discriminator"] == { + "propertyName": "action", + "mapping": { + "observe": "#/$defs/ObserveActionConfig", + "block": "#/$defs/BlockActionConfig", + "redact": "#/$defs/RedactActionConfig", + }, + } + + +@pytest.mark.parametrize("confidence", list(Confidence)) +def test_wire_parser_accepts_confidence_strings(confidence: Confidence) -> None: + config = PolicyConfig.from_mapping( + _wire_on_finding("observe", minimum_confidence=confidence.value) + ) + + assert config.on_finding.minimum_confidence is confidence + + +def test_null_minimum_confidence_selects_every_confidence() -> None: + config = PolicyConfig.from_mapping( + _wire_on_finding("observe", minimum_confidence=None) + ) + + assert config.on_finding.minimum_confidence is None + + +@pytest.mark.parametrize( + "values", + [ + {"unknown": "value"}, + {"action": {"kind": "observe"}}, + {"on_finding": "observe"}, + {"on_finding": {"kind": "observe"}}, + {"on_finding": {"action": "audit"}}, + {"on_finding": {}}, + {"body_format": ""}, + {"debug_inject_path": "/message"}, + {"debug_inject_text": " suffix"}, + {"redaction_template": "[redacted]"}, + {"entity_types": ["email"]}, + {"minimum_confidence": "high"}, + ], +) +def test_rejects_invalid_or_misplaced_fields(values: dict[str, object]) -> None: + with pytest.raises(PrivacyGuardError) as exception_info: + PolicyConfig.from_mapping(values) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + assert exception_info.value.__cause__ is None + + +@pytest.mark.parametrize( + "values", + [ + {"body_format": True}, + _wire_on_finding("redact", template=True), + _wire_on_finding("observe", minimum_confidence=True), + {"on_finding": {"action": True}}, + ], +) +def test_does_not_coerce_wrong_scalar_types(values: dict[str, object]) -> None: + with pytest.raises(PrivacyGuardError) as exception_info: + PolicyConfig.from_mapping(values) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + + +@pytest.mark.parametrize( + "values", + [ + {"body_format": "safe\ud800sentinel"}, + {"on_finding": {"action": "safe\ud800sentinel"}}, + _wire_on_finding("redact", template="safe\ud800sentinel"), + _wire_on_finding("observe", minimum_confidence="safe\ud800sentinel"), + ], +) +def test_rejects_unpaired_unicode_surrogates(values: dict[str, object]) -> None: + with pytest.raises(PrivacyGuardError) as exception_info: + PolicyConfig.from_mapping(values) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + + +def test_accepts_action_finding_criteria() -> None: + config = PolicyConfig.from_mapping( + _wire_on_finding( + "block", + entity_types=["email", "api_key"], + minimum_confidence="high", + ) + ) + + assert config.on_finding.entity_types == frozenset({"email", "api_key"}) + assert config.on_finding.minimum_confidence is Confidence.HIGH + + +@pytest.mark.parametrize( + "values", + [ + {"on_finding": PolicyAction.BLOCK}, + _wire_on_finding("block", entity_types=("email",)), + _wire_on_finding("block", entity_types={"email"}), + _wire_on_finding("block", entity_types="email"), + _wire_on_finding("block", entity_types=["email", 1]), + ], +) +def test_wire_parser_rejects_non_wire_forms(values: dict[str, object]) -> None: + with pytest.raises(PrivacyGuardError) as exception_info: + PolicyConfig.from_mapping(values) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + + +@pytest.mark.parametrize( + "values", + [ + [("on_finding", {"action": "block"})], + (("on_finding", {"action": "block"}),), + iter([("on_finding", {"action": "block"})]), + ], +) +def test_wire_parser_rejects_non_mapping_pair_iterables(values: object) -> None: + with pytest.raises(PrivacyGuardError) as exception_info: + PolicyConfig.from_mapping(values) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + assert exception_info.value.__cause__ is None + + +def test_direct_construction_is_strict_model_to_model_use() -> None: + action = BlockActionConfig( + entity_types=frozenset({"email"}), + minimum_confidence=Confidence.HIGH, + ) + config = PolicyConfig(on_finding=action) + + assert config.on_finding is action + + discriminated = PolicyConfig.model_validate({"on_finding": {"action": "block"}}) + assert type(discriminated.on_finding) is BlockActionConfig + with pytest.raises(ValidationError): + BlockActionConfig.model_validate({"entity_types": ["email"]}) + with pytest.raises(ValidationError): + BlockActionConfig.model_validate({"minimum_confidence": "high"}) + + +def test_template_is_valid_only_for_redact_action() -> None: + redact = PolicyConfig.from_mapping( + _wire_on_finding("redact", template="[redacted]") + ) + + assert type(redact.on_finding) is RedactActionConfig + assert redact.on_finding.template == "[redacted]" + for action in ("observe", "block"): + with pytest.raises(PrivacyGuardError): + PolicyConfig.from_mapping(_wire_on_finding(action, template="[redacted]")) + + +@pytest.mark.parametrize( + "entities", + [ + [""], + ["email", "email"], + ["safe\ud800sentinel"], + ["x" * (MAX_SCANNER_METADATA_BYTES + 1)], + ["😀" * (MAX_SCANNER_METADATA_BYTES // 4 + 1)], + ], +) +def test_rejects_invalid_entity_names(entities: list[str]) -> None: + with pytest.raises(PrivacyGuardError): + PolicyConfig.from_mapping(_wire_on_finding("observe", entity_types=entities)) + + +def test_none_selects_all_entities_and_empty_set_selects_none() -> None: + all_entities = PolicyConfig.from_mapping( + _wire_on_finding("observe", entity_types=None) + ) + no_entities = PolicyConfig.from_mapping( + _wire_on_finding("observe", entity_types=[]) + ) + + assert all_entities.on_finding.entity_types is None + assert no_entities.on_finding.entity_types == frozenset() + + +def test_accepts_entity_name_at_exact_utf8_metadata_limit() -> None: + entity = "😀" * (MAX_SCANNER_METADATA_BYTES // 4) + + config = PolicyConfig.from_mapping( + _wire_on_finding("observe", entity_types=[entity]) + ) + + assert config.on_finding.entity_types == frozenset({entity}) + + +def test_shared_validation_types_are_strict_scalar_safe_and_validate_defaults() -> None: + fixture = _ValidationFixture( + scalar="", + non_empty="value", + metadata="metadata", + ) + + assert fixture.scalar == "" + with pytest.raises(ValidationError): + _ValidationFixture.model_validate( + {"scalar": 1, "non_empty": "value", "metadata": "metadata"} + ) + with pytest.raises(ValidationError): + _ValidationFixture(scalar="\ud800", non_empty="value", metadata="metadata") + with pytest.raises(ValidationError): + _InvalidDefaultFixture() + + +@pytest.mark.parametrize("template", ["[redacted]", "[{entity}]", "{{{entity}}}"]) +def test_accepts_static_and_label_only_templates(template: str) -> None: + config = PolicyConfig.from_mapping(_wire_on_finding("redact", template=template)) + + assert type(config.on_finding) is RedactActionConfig + assert config.on_finding.template == template + + +@pytest.mark.parametrize( + "template", + ["{unknown}", "{entity!r}", "{entity:>10}", "{", "}", "{}", "{0}"], +) +def test_rejects_unsafe_or_malformed_templates(template: str) -> None: + with pytest.raises(PrivacyGuardError) as exception_info: + PolicyConfig.from_mapping(_wire_on_finding("redact", template=template)) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + + +def test_models_are_frozen_and_sensitive_fields_are_hidden_from_repr() -> None: + sentinel = "sensitive-config-value-8472" + config = PolicyConfig.from_mapping( + { + "body_format": sentinel, + **_wire_on_finding( + "redact", + template=sentinel, + entity_types=[sentinel], + ), + } + ) + + with pytest.raises(ValidationError): + setattr(config, "body_format", "json") + with pytest.raises(ValidationError): + setattr(config.on_finding, "minimum_confidence", Confidence.HIGH) + assert sentinel not in repr(config) + assert sentinel not in repr(config.on_finding) + + +def test_validation_failure_does_not_leak_input_or_pydantic_error() -> None: + sentinel = "sensitive-invalid-config-value-8472" + + with pytest.raises(PrivacyGuardError) as exception_info: + PolicyConfig.from_mapping({"body_format": sentinel + "\ud800"}) + + error = exception_info.value + assert error.code is ErrorCode.CONFIG_INVALID + assert sentinel not in str(error) + assert sentinel not in repr(error) + assert sentinel not in repr(error.args) + assert error.__cause__ is None + + +def test_select_handler_error_does_not_leak_unknown_format() -> None: + sentinel = "sensitive-unknown-format-8472" + + with pytest.raises(PrivacyGuardError) as exception_info: + select_handler(sentinel) + + assert exception_info.value.code is ErrorCode.BODY_FORMAT_UNSUPPORTED + assert sentinel not in str(exception_info.value) + assert sentinel not in repr(exception_info.value) diff --git a/projects/privacy-guard/tests/test_errors.py b/projects/privacy-guard/tests/test_errors.py new file mode 100644 index 0000000..8aaf539 --- /dev/null +++ b/projects/privacy-guard/tests/test_errors.py @@ -0,0 +1,39 @@ +import inspect + +from privacy_guard.errors import ( + ErrorCode, + ErrorComponent, + ErrorKind, + PrivacyGuardError, +) + + +def test_every_error_code_has_one_safe_complete_specification() -> None: + sentinel = "sensitive-request-value-8472" + + assert len({code.value for code in ErrorCode}) == len(ErrorCode) + for code in ErrorCode: + error = PrivacyGuardError(code) + message = str(error) + + assert f"[{code.value}]" in message + assert error.component.value in message + assert error.operation in message + assert error.summary in message + assert error.hint in message + assert sentinel not in message + assert repr(error) == f"PrivacyGuardError({message!r})" + + +def test_error_kinds_distinguish_invalid_input_from_internal_failures() -> None: + assert PrivacyGuardError(ErrorCode.CONFIG_INVALID).kind is ErrorKind.INVALID_INPUT + assert ( + PrivacyGuardError(ErrorCode.SCANNER_EXECUTION_FAILED).kind is ErrorKind.INTERNAL + ) + assert ( + PrivacyGuardError(ErrorCode.CONFIG_INVALID).component is ErrorComponent.CONFIG + ) + + +def test_privacy_guard_error_exposes_only_a_catalog_code_parameter() -> None: + assert list(inspect.signature(PrivacyGuardError).parameters) == ["code"] diff --git a/projects/privacy-guard/tests/test_hardening.py b/projects/privacy-guard/tests/test_hardening.py new file mode 100644 index 0000000..5875770 --- /dev/null +++ b/projects/privacy-guard/tests/test_hardening.py @@ -0,0 +1,594 @@ +from __future__ import annotations + +import asyncio +import logging +import threading +import tracemalloc +from typing import Never + +import grpc +import pytest +from pydantic import ValidationError +from typing_extensions import override + +import privacy_guard.service.servicer as servicer_module +from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 +from privacy_guard.body import JsonHandler +from privacy_guard.config import PolicyConfig +from privacy_guard.constants import MAX_BODY_BYTES +from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.payloads import ( + InterceptedRequest, + ProcessingDecision, + ProcessingResult, +) +from privacy_guard.processor import RequestProcessor +from privacy_guard.scanners import ( + Confidence, + Finding, + PassthroughScanner, + RequestBodyFinding, + Scanner, + ScannerConfig, +) +from privacy_guard.service.servicer import PrivacyGuardMiddleware + +from .scanner_helpers import DeterministicEmailScanner + + +def _request( + body: bytes, + *, + action_kind: str = "redact", + entity_types: list[str] | None = None, + minimum_confidence: str | None = None, +) -> InterceptedRequest: + return InterceptedRequest( + raw_body=body, + policy_config=PolicyConfig.from_mapping( + { + "on_finding": { + "action": action_kind, + "entity_types": entity_types, + "minimum_confidence": minimum_confidence, + } + } + ), + ) + + +class UnexpectedAbortContext: + async def abort(self, code: grpc.StatusCode, details: str) -> Never: + raise AssertionError("successful evaluation must not abort") + + +@pytest.mark.parametrize( + ("depth", "allowed"), + [(64, True), (65, False)], +) +def test_json_nesting_limit_exact_boundary(depth: int, allowed: bool) -> None: + body = (b"[" * depth) + b'"safe"' + (b"]" * depth) + if allowed: + assert ( + JsonHandler().normalize(body, PolicyConfig()).text_blocks[-1].text == "safe" + ) + else: + with pytest.raises(PrivacyGuardError) as exception_info: + JsonHandler().normalize(body, PolicyConfig()) + assert exception_info.value.code is ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED + + +@pytest.mark.parametrize("depth", [255, 300, 500, 900]) +def test_json_nesting_beyond_adapter_recursion_limit_is_shape_error(depth: int) -> None: + body = (b"[" * depth) + b'"safe"' + (b"]" * depth) + + with pytest.raises(PrivacyGuardError) as exception_info: + JsonHandler().normalize(body, PolicyConfig()) + + assert exception_info.value.code is ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED + assert exception_info.value.__cause__ is None + + +def test_json_walker_bounds_allocation_at_exact_aggregate_boundaries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import privacy_guard.body.json as json_module + + monkeypatch.setattr(json_module, "MAX_TEXT_BLOCKS", 2) + monkeypatch.setattr(json_module, "MAX_SCANNED_CHARACTERS", 2) + assert len(JsonHandler().normalize(b'{"a":"b"}', PolicyConfig()).text_blocks) == 2 + + monkeypatch.setattr(json_module, "MAX_TEXT_BLOCKS", 1) + with pytest.raises(PrivacyGuardError) as block_error: + JsonHandler().normalize(b'{"a":"b"}', PolicyConfig()) + assert block_error.value.code is ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED + + monkeypatch.setattr(json_module, "MAX_TEXT_BLOCKS", 2) + monkeypatch.setattr(json_module, "MAX_SCANNED_CHARACTERS", 1) + with pytest.raises(PrivacyGuardError) as text_error: + JsonHandler().normalize(b'{"a":"b"}', PolicyConfig()) + assert text_error.value.code is ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED + + +def test_json_walker_keeps_wide_container_traversal_allocation_bounded() -> None: + import privacy_guard.body.json as json_module + + raw_body = b"[" + (b"0," * 99_999) + b"0]" + handler = JsonHandler() + policy = PolicyConfig() + + tracemalloc.start() + try: + request_body = handler.normalize(raw_body, policy) + _, peak_bytes = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert request_body.text_blocks == () + # Parsing and the one strict typed boundary peaked near 1.8 MiB locally. + assert peak_bytes < 8 * 1024 * 1024 + + assert type(request_body.parsed_value) is json_module._JsonBodyState + tracemalloc.start() + try: + assert tuple(handler._iter_text_blocks(request_body.parsed_value.value)) == () + _, walker_peak_bytes = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + # The incremental walker retains iterator state only for nesting depth. + assert walker_peak_bytes < 64 * 1024 + + +def test_processor_checks_handler_aggregate_boundaries_contextually( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import privacy_guard.processor as processor_module + + processor = RequestProcessor([PassthroughScanner()]) + monkeypatch.setattr(processor_module, "MAX_TEXT_BLOCKS", 2) + monkeypatch.setattr(processor_module, "MAX_SCANNED_CHARACTERS", 2) + assert ( + processor.process(_request(b'{"a":"b"}')).decision is ProcessingDecision.ALLOW + ) + + monkeypatch.setattr(processor_module, "MAX_TEXT_BLOCKS", 1) + with pytest.raises(PrivacyGuardError) as block_error: + processor.process(_request(b'{"a":"b"}')) + assert block_error.value.code is ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED + + monkeypatch.setattr(processor_module, "MAX_TEXT_BLOCKS", 2) + monkeypatch.setattr(processor_module, "MAX_SCANNED_CHARACTERS", 1) + with pytest.raises(PrivacyGuardError) as text_error: + processor.process(_request(b'{"a":"b"}')) + assert text_error.value.code is ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED + + +@pytest.mark.parametrize( + ("action", "expected"), + [ + ("observe", ProcessingDecision.ALLOW), + ("block", ProcessingDecision.DENY), + ("redact", ProcessingDecision.DENY), + ], +) +def test_json_key_findings_are_observed_blocked_and_never_rewritten( + action: str, expected: ProcessingDecision +) -> None: + scanner = DeterministicEmailScanner( + ScannerConfig(name="test_email", entity_types=frozenset({"email"})) + ) + result = RequestProcessor([scanner]).process( + _request(b'{"user@example.com":"safe","model":"safe"}', action_kind=action) + ) + + assert result.decision is expected + assert len(result.findings) == 1 + assert result.findings[0].text_block_path == "#key:/user@example.com" + assert result.replacement_body is None + + +@pytest.mark.parametrize("action_kind", ["observe", "block", "redact"]) +def test_finding_criteria_are_owned_by_every_action(action_kind: str) -> None: + class MixedScanner(Scanner[ScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + return ( + Finding( + entity="email", + scanner_name=self.scanner_name, + start_offset=0, + end_offset=1, + confidence=Confidence.LOW, + ), + Finding( + entity="token", + scanner_name=self.scanner_name, + start_offset=1, + end_offset=2, + confidence=Confidence.HIGH, + ), + ) + + result = RequestProcessor( + [ + MixedScanner( + ScannerConfig(name="mixed", entity_types=frozenset({"email", "token"})) + ) + ] + ).process( + _request( + b'"ab"', + action_kind=action_kind, + entity_types=["token"], + minimum_confidence="medium", + ) + ) + + assert [ + request_body_finding.finding.entity for request_body_finding in result.findings + ] == ["token"] + + +def test_unknown_entity_filter_is_content_safe_and_multi_scanner_union_is_valid() -> ( + None +): + class EmailScanner(Scanner[ScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + return () + + class TokenScanner(Scanner[ScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + return () + + processor = RequestProcessor( + ( + EmailScanner( + ScannerConfig(name="email", entity_types=frozenset({"email"})) + ), + TokenScanner( + ScannerConfig(name="token", entity_types=frozenset({"token"})) + ), + ) + ) + processor.validate_policy_config( + PolicyConfig.from_mapping( + { + "on_finding": { + "action": "redact", + "entity_types": ["email", "token"], + } + } + ) + ) + typo = "sensitive-email-typo-8472" + with pytest.raises(PrivacyGuardError) as exception_info: + processor.validate_policy_config( + PolicyConfig.from_mapping( + { + "on_finding": { + "action": "redact", + "entity_types": [typo], + } + } + ) + ) + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + assert typo not in str(exception_info.value) + with pytest.raises(PrivacyGuardError) as process_error: + processor.process(_request(b'"safe"', entity_types=[typo])) + assert process_error.value.code is ErrorCode.CONFIG_INVALID + assert typo not in str(process_error.value) + + +def test_multi_scanner_overlap_is_retained_for_observe_and_resolved_for_redact() -> ( + None +): + class FirstScanner(Scanner[ScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + return ( + Finding( + entity="short", + scanner_name=self.scanner_name, + start_offset=1, + end_offset=3, + ), + ) + + class SecondScanner(Scanner[ScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + return ( + Finding( + entity="long", + scanner_name=self.scanner_name, + start_offset=0, + end_offset=4, + ), + ) + + processor = RequestProcessor( + ( + FirstScanner( + ScannerConfig(name="first", entity_types=frozenset({"short"})) + ), + SecondScanner( + ScannerConfig(name="second", entity_types=frozenset({"long"})) + ), + ) + ) + observed = processor.process(_request(b'"abcd"', action_kind="observe")) + redacted = processor.process(_request(b'"abcd"', action_kind="redact")) + blocked = processor.process(_request(b'"abcd"', action_kind="block")) + + assert len(observed.findings) == 2 + assert redacted.replacement_body == b'"[long]"' + assert { + request_body_finding.finding.scanner_name + for request_body_finding in redacted.findings + } == { + "first", + "second", + } + assert blocked.decision is ProcessingDecision.DENY + assert len(blocked.findings) == 2 + + +def test_finding_excess_stably_denies_without_returning_partial_findings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import privacy_guard.scanners.base as scanner_module + + class NoisyScanner(Scanner[ScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + return ( + Finding( + entity="a", + scanner_name=self.scanner_name, + start_offset=0, + end_offset=1, + ), + Finding( + entity="b", + scanner_name=self.scanner_name, + start_offset=1, + end_offset=2, + ), + ) + + monkeypatch.setattr(scanner_module, "MAX_FINDINGS_PER_BLOCK", 1) + scanner = NoisyScanner( + ScannerConfig(name="noisy", entity_types=frozenset({"a", "b"})) + ) + result = RequestProcessor([scanner]).process(_request(b'"ab"')) + + assert result.decision is ProcessingDecision.DENY + assert result.reason_code == "privacy_guard_limit_exceeded" + assert result.findings == () + + +def test_scanner_identity_is_non_empty_and_finding_entities_are_bounded() -> None: + with pytest.raises(ValidationError): + ScannerConfig(name="", entity_types=frozenset()) + + with pytest.raises(ValidationError): + Finding( + entity="x" * 1025, + scanner_name="bounded", + start_offset=0, + end_offset=1, + ) + + +@pytest.mark.parametrize(("block_count", "allowed"), [(16, True), (17, False)]) +def test_request_finding_limit_exact_boundary(block_count: int, allowed: bool) -> None: + class DenseScanner(Scanner[ScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + return tuple( + Finding( + entity="unit", + scanner_name=self.scanner_name, + start_offset=index, + end_offset=index + 1, + ) + for index in range(256) + ) + + body = ( + "[" + ",".join('"' + ("x" * 256) + '"' for _ in range(block_count)) + "]" + ).encode() + result = RequestProcessor( + [DenseScanner(ScannerConfig(name="dense", entity_types=frozenset({"unit"})))] + ).process(_request(body, action_kind="observe")) + + assert result.decision is ( + ProcessingDecision.ALLOW if allowed else ProcessingDecision.DENY + ) + assert len(result.findings) == (4096 if allowed else 0) + + +@pytest.mark.asyncio +async def test_slow_scan_does_not_stall_unrelated_rpc() -> None: + started = threading.Event() + release = threading.Event() + + class SlowScanner(Scanner[ScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + started.set() + release.wait(timeout=2) + return () + + servicer = PrivacyGuardMiddleware( + RequestProcessor( + [SlowScanner(ScannerConfig(name="slow", entity_types=frozenset()))] + ) + ) + evaluation = pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + body=b'"safe"', + ) + scan_task = asyncio.create_task(servicer._evaluate_http_request(evaluation)) + assert await asyncio.to_thread(started.wait, 1) + try: + manifest = await asyncio.wait_for( + asyncio.to_thread(servicer._describe), timeout=0.1 + ) + assert manifest.name == "privacy-guard" + finally: + release.set() + await scan_task + await servicer.close() + + +@pytest.mark.asyncio +async def test_cancelled_rpc_holds_scan_slot_until_worker_really_finishes() -> None: + entered = 0 + first_started = threading.Event() + release = threading.Event() + + class BlockingScanner(Scanner[ScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + nonlocal entered + entered += 1 + first_started.set() + release.wait(timeout=2) + return () + + servicer = PrivacyGuardMiddleware( + RequestProcessor( + [BlockingScanner(ScannerConfig(name="blocking", entity_types=frozenset()))] + ) + ) + servicer._scan_slots = asyncio.Semaphore(1) + evaluation = pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, body=b'"safe"' + ) + first = asyncio.create_task(servicer._evaluate_http_request(evaluation)) + assert await asyncio.to_thread(first_started.wait, 1) + first.cancel() + with pytest.raises(asyncio.CancelledError): + await first + + second = asyncio.create_task(servicer._evaluate_http_request(evaluation)) + await asyncio.sleep(0.05) + assert entered == 1 + release.set() + await asyncio.wait_for(second, 1) + assert entered == 2 + await servicer.close() + + +@pytest.mark.asyncio +async def test_operational_log_never_contains_body_or_match( + caplog: pytest.LogCaptureFixture, +) -> None: + sentinel = "sentinel@example.com" + evaluation = pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + context=pb2.RequestContext(request_id="request-42"), + config={"on_finding": {"action": "observe"}}, + body=(f'{{"message":"{sentinel}"}}').encode(), + ) + scanner = DeterministicEmailScanner( + ScannerConfig(name="test_email", entity_types=frozenset({"email"})) + ) + servicer = PrivacyGuardMiddleware(RequestProcessor([scanner])) + with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): + await servicer._evaluate_rpc(evaluation, UnexpectedAbortContext()) + await servicer.close() + + assert "privacy_guard_evaluation" in caplog.text + assert sentinel not in caplog.text + + +def test_operational_log_fields_use_typed_construction() -> None: + extra = servicer_module._evaluation_log_extra( + request_id="request-42", + started=0.0, + action="allow", + finding_count=1, + failure=None, + ) + + request_id: str = extra["request_id"] + duration_ms: float = extra["duration_ms"] + action: str = extra["action"] + finding_count: int = extra["finding_count"] + error_code: str | None = extra["error_code"] + assert request_id == "request-42" + assert duration_ms >= 0 + assert action == "allow" + assert finding_count == 1 + assert error_code is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("size", [MAX_BODY_BYTES, MAX_BODY_BYTES + 1]) +async def test_replacement_body_limit_exact_boundary(size: int) -> None: + class ReplacementProcessor: + def validate_policy_config(self, policy_config: PolicyConfig) -> None: + pass + + def process(self, request: InterceptedRequest) -> ProcessingResult: + return ProcessingResult( + decision=ProcessingDecision.ALLOW, replacement_body=b"x" * size + ) + + response = await PrivacyGuardMiddleware( + ReplacementProcessor() + )._evaluate_http_request( + pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, body=b"{}" + ) + ) + assert response.decision == ( + pb2.DECISION_ALLOW if size == MAX_BODY_BYTES else pb2.DECISION_DENY + ) + assert response.has_body is (size == MAX_BODY_BYTES) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("extra_byte", "allowed"), [(False, True), (True, False)]) +async def test_aggregate_finding_encoded_limit_exact_boundary( + monkeypatch: pytest.MonkeyPatch, extra_byte: bool, allowed: bool +) -> None: + entity = "x" * (900 + int(extra_byte)) + exact_size = pb2.Finding( + type="scanner", label="x" * 900, confidence="high", count=1 + ).ByteSize() + monkeypatch.setattr(servicer_module, "MAX_PROTO_FINDING_BYTES", exact_size) + + class FindingProcessor: + def validate_policy_config(self, policy_config: PolicyConfig) -> None: + pass + + def process(self, request: InterceptedRequest) -> ProcessingResult: + return ProcessingResult( + decision=ProcessingDecision.ALLOW, + findings=( + RequestBodyFinding( + finding=Finding( + entity=entity, + scanner_name="scanner", + start_offset=0, + end_offset=1, + ), + text_block_path="path", + ), + ), + ) + + response = await PrivacyGuardMiddleware(FindingProcessor())._evaluate_http_request( + pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, body=b"{}" + ) + ) + assert response.decision == (pb2.DECISION_ALLOW if allowed else pb2.DECISION_DENY) + assert len(response.findings) == int(allowed) diff --git a/projects/privacy-guard/tests/test_payloads.py b/projects/privacy-guard/tests/test_payloads.py new file mode 100644 index 0000000..cb03734 --- /dev/null +++ b/projects/privacy-guard/tests/test_payloads.py @@ -0,0 +1,118 @@ +import pytest +from pydantic import ValidationError + +from privacy_guard.config import PolicyConfig +from privacy_guard.payloads import ( + InterceptedRequest, + ProcessingDecision, + ProcessingResult, +) +from privacy_guard.scanners import Finding, RequestBodyFinding + + +def test_processing_decision_values() -> None: + assert ProcessingDecision.ALLOW.value == "allow" + assert ProcessingDecision.DENY.value == "deny" + + +def test_payloads_are_frozen_and_hide_body_content_from_repr() -> None: + sensitive_body = b"sensitive-body-8472" + request = InterceptedRequest( + raw_body=sensitive_body, + content_type="application/json", + policy_config=PolicyConfig(), + ) + result = ProcessingResult( + decision=ProcessingDecision.ALLOW, + replacement_body=sensitive_body, + ) + + with pytest.raises(ValidationError): + setattr(request, "raw_body", b"changed") + with pytest.raises(ValidationError): + setattr(result, "replacement_body", b"changed") + assert repr(sensitive_body) not in repr(request) + assert repr(sensitive_body) not in repr(result) + + +def test_processing_result_repr_hides_sensitive_structural_path() -> None: + sensitive_key = "secret-person@example.com" + finding = RequestBodyFinding( + finding=Finding( + entity="email", + scanner_name="scanner", + start_offset=0, + end_offset=1, + ), + text_block_path=f"#key:/{sensitive_key}", + ) + result = ProcessingResult(decision=ProcessingDecision.ALLOW, findings=(finding,)) + + assert sensitive_key not in repr(finding) + assert sensitive_key not in repr(result) + + +def test_empty_replacement_is_distinct_from_no_replacement() -> None: + no_replacement = ProcessingResult(decision=ProcessingDecision.ALLOW) + empty_replacement = ProcessingResult( + decision=ProcessingDecision.ALLOW, replacement_body=b"" + ) + + assert no_replacement.replacement_body is None + assert empty_replacement.replacement_body == b"" + assert no_replacement != empty_replacement + + +def test_processing_result_defaults_to_an_empty_findings_tuple() -> None: + result = ProcessingResult(decision=ProcessingDecision.ALLOW) + + assert result.findings == () + assert type(result.findings) is tuple + assert result.reason_code is None + + +def test_intercepted_request_is_a_passive_record_without_body_method() -> None: + assert not hasattr(InterceptedRequest, "body") + + +@pytest.mark.parametrize( + ("model", "values"), + [ + ( + InterceptedRequest, + { + "raw_body": bytearray(b"body"), + "policy_config": PolicyConfig(), + }, + ), + ( + InterceptedRequest, + { + "raw_body": b"body", + "policy_config": {"on_finding": {"action": "redact"}}, + }, + ), + (ProcessingResult, {"decision": "allow"}), + ( + ProcessingResult, + {"decision": ProcessingDecision.ALLOW, "findings": []}, + ), + ], +) +def test_payloads_reject_dataclass_era_coercions( + model: type[InterceptedRequest] | type[ProcessingResult], + values: dict[str, object], +) -> None: + with pytest.raises(ValidationError): + model.model_validate(values) + + +def test_payloads_forbid_extra_fields() -> None: + with pytest.raises(ValidationError): + InterceptedRequest.model_validate( + { + "raw_body": b"body", + "policy_config": PolicyConfig(), + "legacy_body": b"body", + } + ) diff --git a/projects/privacy-guard/tests/test_processor.py b/projects/privacy-guard/tests/test_processor.py new file mode 100644 index 0000000..c6879c8 --- /dev/null +++ b/projects/privacy-guard/tests/test_processor.py @@ -0,0 +1,684 @@ +from collections.abc import Mapping + +import pytest +from pydantic import ValidationError +from typing_extensions import override + +from privacy_guard.body import FormatHandler, RequestBody, TextBlock +from privacy_guard.config import PolicyAction, PolicyConfig +from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.payloads import ( + InterceptedRequest, + ProcessingDecision, + ProcessingResult, +) +from privacy_guard.processor import RequestProcessor +from privacy_guard.scanners import ( + Finding, + PassthroughScanner, + RequestBodyFinding, + Scanner, + ScannerConfig, + ScannerContractError, + ScannerFindingLimitExceeded, + parse_scanner_output, +) + + +class RecordingScanner(Scanner[ScannerConfig]): + def __init__( + self, findings_by_text: Mapping[str, tuple[Finding, ...]] | None = None + ) -> None: + self.findings_by_text = findings_by_text or {} + entity_types = frozenset( + finding.entity + for findings in self.findings_by_text.values() + for finding in findings + ) + super().__init__(ScannerConfig(name="recording", entity_types=entity_types)) + self.calls: list[str] = [] + + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + self.calls.append(text_block) + return self.findings_by_text.get(text_block, ()) + + +class RecordingHandler(FormatHandler): + def __init__( + self, + text_blocks: tuple[TextBlock, ...] = (), + reconstructed: bytes | None = None, + ) -> None: + super().__init__(format_name="opaque") + self.text_blocks = text_blocks + self.reconstructed = reconstructed + self.normalize_calls = 0 + self.reconstruct_calls = 0 + self.replacements: dict[str, str] | None = None + + @override + def _normalize(self, raw_body: bytes, policy_config: PolicyConfig) -> RequestBody: + self.normalize_calls += 1 + return RequestBody( + text_blocks=self.text_blocks, + parsed_value=None, + original_bytes=raw_body, + ) + + @override + def _reconstruct( + self, + request_body: RequestBody, + replacements_by_path: Mapping[str, str], + ) -> bytes: + self.reconstruct_calls += 1 + self.replacements = dict(replacements_by_path) + return ( + request_body.original_bytes + if self.reconstructed is None + else self.reconstructed + ) + + +def _config( + *, + body_format: str = "opaque", + action_kind: str = "redact", + entity_types: list[str] | None = None, + minimum_confidence: str | None = None, + template: str = "[{entity}]", +) -> PolicyConfig: + on_finding: dict[str, object] = { + "action": action_kind, + "entity_types": entity_types, + "minimum_confidence": minimum_confidence, + } + if action_kind == PolicyAction.REDACT.value: + on_finding["template"] = template + return PolicyConfig.from_mapping( + {"body_format": body_format, "on_finding": on_finding} + ) + + +def _request( + raw_body: bytes = b"body", policy_config: PolicyConfig | None = None +) -> InterceptedRequest: + return InterceptedRequest( + raw_body=raw_body, + content_type="application/test", + policy_config=policy_config or _config(), + ) + + +def _processor( + scanner: Scanner[ScannerConfig], handler: FormatHandler +) -> RequestProcessor: + return RequestProcessor([scanner], {handler.format_name: handler}) + + +def _finding( + start: int, + end: int, + *, + entity: str = "secret", + scanner_name: str = "recording", +) -> Finding: + return Finding( + entity=entity, + scanner_name=scanner_name, + start_offset=start, + end_offset=end, + ) + + +def _request_body_finding( + start: int, + end: int, + *, + entity: str = "secret", + scanner_name: str = "recording", + text_block_path: str, +) -> RequestBodyFinding: + return RequestBodyFinding( + finding=_finding(start, end, entity=entity, scanner_name=scanner_name), + text_block_path=text_block_path, + ) + + +def test_passthrough_scans_each_text_block_and_reconstructs_once() -> None: + scanner = RecordingScanner() + handler = RecordingHandler( + ( + TextBlock(path="text_block::alpha", text="first"), + TextBlock(path="text_block::beta", text="second"), + ) + ) + + result = _processor(scanner, handler).process(_request()) + + assert result == ProcessingResult(decision=ProcessingDecision.ALLOW) + assert scanner.calls == ["first", "second"] + assert handler.normalize_calls == 1 + assert handler.reconstruct_calls == 1 + assert handler.replacements == {} + + +def test_empty_scanner_sequence_is_rejected_at_construction() -> None: + with pytest.raises(PrivacyGuardError) as exception_info: + RequestProcessor([]) + + assert exception_info.value.code is ErrorCode.SCANNER_OUTPUT_INVALID + + +def test_duplicate_scanner_names_are_rejected_at_construction() -> None: + with pytest.raises(PrivacyGuardError) as exception_info: + RequestProcessor([RecordingScanner(), RecordingScanner()]) + + assert exception_info.value.code is ErrorCode.SCANNER_OUTPUT_INVALID + + +def test_scanner_infers_validates_and_uses_concrete_config_type() -> None: + class PrefixScannerConfig(ScannerConfig): + finding_entity: str + + class PrefixScanner(Scanner[PrefixScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + return ( + Finding( + entity=self.config.finding_entity, + scanner_name=self.scanner_name, + start_offset=0, + end_offset=1, + ), + ) + + config = PrefixScannerConfig( + name="prefix", + entity_types=frozenset({"custom"}), + finding_entity="custom", + ) + scanner = PrefixScanner(config) + RequestProcessor([scanner]) + + assert PrefixScanner.get_config_type() is PrefixScannerConfig + assert scanner.config is config + assert scanner.config.finding_entity == "custom" + assert scanner.supported_entity_types is scanner.config.entity_types + assert scanner.scan("x")[0].entity == "custom" + pytest.raises( + ScannerContractError, + PrefixScanner, + ScannerConfig(name="wrong-config-type", entity_types=frozenset()), + ) + + +@pytest.mark.parametrize("scanner", [RecordingScanner(), PassthroughScanner()]) +def test_empty_supported_entity_catalog_rejects_enabled_filter( + scanner: Scanner[ScannerConfig], +) -> None: + processor = RequestProcessor([scanner]) + + with pytest.raises(PrivacyGuardError) as exception_info: + processor.validate_policy_config( + PolicyConfig.from_mapping( + {"on_finding": {"action": "observe", "entity_types": ["email"]}} + ) + ) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + + +def test_handler_registry_key_must_match_immutable_identity() -> None: + with pytest.raises(PrivacyGuardError) as exception_info: + RequestProcessor([PassthroughScanner()], {"wrong": RecordingHandler()}) + + assert exception_info.value.code is ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID + + +def test_malformed_normalize_output_maps_to_handler_contract_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + handler = RecordingHandler() + monkeypatch.setattr(handler, "_normalize", lambda raw_body, policy: object()) + + with pytest.raises(PrivacyGuardError) as exception_info: + _processor(PassthroughScanner(), handler).process(_request()) + + assert exception_info.value.code is ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID + + +def test_non_bytes_reconstruct_output_maps_to_handler_contract_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + handler = RecordingHandler() + monkeypatch.setattr( + handler, + "_reconstruct", + lambda request_body, replacements: bytearray(b"body"), + ) + + with pytest.raises(PrivacyGuardError) as exception_info: + _processor(PassthroughScanner(), handler).process(_request()) + + assert exception_info.value.code is ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID + + +def test_validate_policy_config_selects_and_validates_without_processing() -> None: + handler = RecordingHandler() + processor = _processor(PassthroughScanner(), handler) + + processor.validate_policy_config(_config()) + + assert handler.normalize_calls == 0 + assert handler.reconstruct_calls == 0 + + +def test_bodyless_request_skips_normalization_scanning_and_reconstruction() -> None: + scanner = RecordingScanner() + handler = RecordingHandler((TextBlock(path="text_block::alpha", text="not used"),)) + + result = _processor(scanner, handler).process(_request(b"")) + + assert result == ProcessingResult(decision=ProcessingDecision.ALLOW) + assert handler.normalize_calls == 0 + assert scanner.calls == [] + assert handler.reconstruct_calls == 0 + + +def test_empty_json_object_scans_nothing_and_reconstructs_once() -> None: + scanner = RecordingScanner() + processor = RequestProcessor([scanner]) + request = InterceptedRequest( + raw_body=b"{}", + content_type="application/json", + policy_config=PolicyConfig(), + ) + + result = processor.process(request) + + assert result == ProcessingResult(decision=ProcessingDecision.ALLOW) + assert scanner.calls == [] + + +def test_observe_attaches_paths_without_mutating_body() -> None: + original = _finding(1, 3) + scanner = RecordingScanner({"abcd": (original,)}) + handler = RecordingHandler((TextBlock(path="text_block::alpha", text="abcd"),)) + + result = _processor(scanner, handler).process( + _request(policy_config=_config(action_kind=PolicyAction.OBSERVE.value)) + ) + + assert result.decision is ProcessingDecision.ALLOW + assert result.replacement_body is None + assert result.findings == ( + _request_body_finding(1, 3, text_block_path="text_block::alpha"), + ) + assert not hasattr(original, "text_block_path") + assert handler.replacements == {} + + +@pytest.mark.parametrize( + ("text", "findings", "template", "expected"), + [ + ("abcde", (_finding(1, 3),), "X", "aXde"), + ( + "abcdef", + (_finding(3, 5, entity="b"), _finding(0, 2, entity="a")), + "[{entity}]", + "[a]c[b]f", + ), + ("a🐍éz", (_finding(1, 3),), "X", "aXz"), + ], +) +def test_redact_replaces_sorted_non_overlapping_character_spans( + text: str, findings: tuple[Finding, ...], template: str, expected: str +) -> None: + scanner = RecordingScanner({text: findings}) + handler = RecordingHandler( + (TextBlock(path="text_block::alpha", text=text),), b"changed" + ) + + result = _processor(scanner, handler).process( + _request(policy_config=_config(template=template)) + ) + + assert result.replacement_body == b"changed" + assert handler.replacements == {"text_block::alpha": expected} + + +def test_static_redaction_template_is_supported() -> None: + scanner = RecordingScanner({"secret": (_finding(0, 6),)}) + handler = RecordingHandler( + (TextBlock(path="text_block::alpha", text="secret"),), b"x" + ) + + _processor(scanner, handler).process( + _request(policy_config=_config(template="[redacted]")) + ) + + assert handler.replacements == {"text_block::alpha": "[redacted]"} + + +def test_redaction_expansion_is_denied_before_render_or_reconstruction( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import privacy_guard.processor as processor_module + + scanner = RecordingScanner({"ab": (_finding(0, 1), _finding(1, 2))}) + handler = RecordingHandler((TextBlock(path="text_block::alpha", text="ab"),)) + monkeypatch.setattr(processor_module, "MAX_BODY_BYTES", 8) + + def unexpected_render(*args: object) -> str: + raise AssertionError("redaction must not be rendered after preflight denial") + + monkeypatch.setattr(processor_module, "_redact_text", unexpected_render) + result = _processor(scanner, handler).process( + _request(policy_config=_config(template="x" * 64)) + ) + + assert result.decision is ProcessingDecision.DENY + assert result.reason_code == "privacy_guard_limit_exceeded" + assert handler.reconstruct_calls == 0 + + +def test_serialized_redaction_expansion_is_denied_after_reconstruction( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import privacy_guard.processor as processor_module + + scanner = RecordingScanner({"x": (_finding(0, 1),)}) + monkeypatch.setattr(processor_module, "MAX_BODY_BYTES", 8) + + result = RequestProcessor([scanner]).process( + InterceptedRequest( + raw_body=b'"x"', + policy_config=_config(body_format="json", template='"' * 6), + ) + ) + + assert result.decision is ProcessingDecision.DENY + assert result.reason_code == "privacy_guard_limit_exceeded" + assert result.replacement_body is None + + +def test_block_denies_with_findings_and_suppresses_reconstruction() -> None: + scanner = RecordingScanner({"secret": (_finding(0, 6),)}) + handler = RecordingHandler((TextBlock(path="text_block::alpha", text="secret"),)) + config = _config(action_kind=PolicyAction.BLOCK.value) + + result = _processor(scanner, handler).process(_request(policy_config=config)) + + assert result.decision is ProcessingDecision.DENY + assert result.replacement_body is None + assert result.reason_code == "privacy_guard_blocked" + assert len(result.findings) == 1 + assert handler.reconstruct_calls == 0 + + +def test_block_allows_and_reconstructs_when_no_findings_exist() -> None: + handler = RecordingHandler((TextBlock(path="text_block::alpha", text="safe"),)) + + result = _processor(PassthroughScanner(), handler).process( + _request(policy_config=_config(action_kind=PolicyAction.BLOCK.value)) + ) + + assert result == ProcessingResult(decision=ProcessingDecision.ALLOW) + assert handler.reconstruct_calls == 1 + + +def test_findings_keep_scanner_identity_and_text_block_order_then_span_order() -> None: + scanner = RecordingScanner( + { + "first": ( + _finding(3, 5, entity="late"), + _finding(0, 2, entity="early"), + ), + "second": (_finding(1, 3, entity="second"),), + } + ) + handler = RecordingHandler( + ( + TextBlock(path="text_block::one", text="first"), + TextBlock(path="text_block::two", text="second"), + ) + ) + + result = _processor(scanner, handler).process( + _request(policy_config=_config(action_kind=PolicyAction.OBSERVE.value)) + ) + + assert [ + ( + request_body_finding.finding.entity, + request_body_finding.finding.scanner_name, + request_body_finding.text_block_path, + ) + for request_body_finding in result.findings + ] == [ + ("early", "recording", "text_block::one"), + ("late", "recording", "text_block::one"), + ("second", "recording", "text_block::two"), + ] + + +@pytest.mark.parametrize( + "normalized", + [ + RequestBody( + text_blocks=( + TextBlock(path="same", text="a"), + TextBlock(path="same", text="b"), + ), + parsed_value=None, + original_bytes=b"body", + ), + RequestBody(text_blocks=(), parsed_value=None, original_bytes=b"different"), + ], +) +def test_contextually_invalid_normalized_body_is_rejected_before_scanning( + normalized: RequestBody, +) -> None: + scanner = RecordingScanner() + + class ReturningHandler(RecordingHandler): + @override + def _normalize( + self, raw_body: bytes, policy_config: PolicyConfig + ) -> RequestBody: + self.normalize_calls += 1 + return normalized + + with pytest.raises(PrivacyGuardError) as exception_info: + _processor(scanner, ReturningHandler()).process(_request()) + + assert exception_info.value.code is ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID + assert scanner.calls == [] + + +@pytest.mark.parametrize("scanner_result", [[], (object(),)]) +def test_invalid_scanner_output_shape_is_rejected(scanner_result: object) -> None: + with pytest.raises(ScannerContractError): + parse_scanner_output(scanner_result) + + +def test_request_body_finding_is_not_valid_scanner_output() -> None: + finding = _finding(0, 1) + request_body_finding = RequestBodyFinding(finding=finding, text_block_path="path") + + with pytest.raises(ScannerContractError): + parse_scanner_output((request_body_finding,)) + + +def test_exact_finding_instance_is_reused_by_output_validation() -> None: + finding = _finding(0, 1) + + parsed = parse_scanner_output((finding,)) + + assert parsed[0] is finding + + +def test_scanner_output_limit_is_checked_before_element_validation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import privacy_guard.scanners.base as scanner_module + + monkeypatch.setattr(scanner_module, "MAX_FINDINGS_PER_BLOCK", 1) + + with pytest.raises(ScannerFindingLimitExceeded): + parse_scanner_output((object(), object())) + + +@pytest.mark.parametrize( + "overrides", + [ + {"entity": ""}, + {"scanner_name": ""}, + {"entity": "bad\ud800"}, + {"scanner_name": "recording\ud800"}, + {"start_offset": True}, + {"start_offset": -1}, + {"start_offset": 1, "end_offset": 1}, + {"confidence": "high"}, + ], +) +def test_invalid_finding_fields_fail_at_model_construction( + overrides: dict[str, object], +) -> None: + values: dict[str, object] = { + "entity": "secret", + "scanner_name": "recording", + "start_offset": 0, + "end_offset": 1, + **overrides, + } + with pytest.raises(ValidationError): + Finding.model_validate(values) + + +@pytest.mark.parametrize( + "finding", + [ + _finding(0, 1, scanner_name="other"), + _finding(1, 5), + ], +) +def test_contextually_invalid_finding_is_rejected(finding: Finding) -> None: + scanner = RecordingScanner({"abcd": (finding,)}) + handler = RecordingHandler((TextBlock(path="text_block::alpha", text="abcd"),)) + + with pytest.raises(PrivacyGuardError) as exception_info: + _processor(scanner, handler).process(_request()) + + assert exception_info.value.code is ErrorCode.SCANNER_OUTPUT_INVALID + assert handler.reconstruct_calls == 0 + + +def test_finding_entity_must_be_declared_by_scanner_config() -> None: + class UndeclaredEntityScanner(Scanner[ScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + return ( + _finding(0, 1, entity="undeclared", scanner_name=self.scanner_name), + ) + + scanner = UndeclaredEntityScanner( + ScannerConfig(name="declared", entity_types=frozenset({"declared"})) + ) + handler = RecordingHandler((TextBlock(path="path", text="x"),)) + + with pytest.raises(PrivacyGuardError) as exception_info: + _processor(scanner, handler).process(_request()) + + assert exception_info.value.code is ErrorCode.SCANNER_OUTPUT_INVALID + + +def test_adjacent_scanner_findings_are_accepted() -> None: + scanner = RecordingScanner({"abcd": (_finding(0, 2), _finding(2, 4))}) + handler = RecordingHandler( + (TextBlock(path="text_block::alpha", text="abcd"),), b"x" + ) + + result = _processor(scanner, handler).process(_request()) + + assert result.decision is ProcessingDecision.ALLOW + assert len(result.findings) == 2 + + +@pytest.mark.parametrize( + ("collaborator", "expected_code"), + [ + ("scanner", ErrorCode.SCANNER_EXECUTION_FAILED), + ("handler", ErrorCode.FORMAT_HANDLER_EXECUTION_FAILED), + ], +) +def test_collaborator_exceptions_are_replaced_without_partial_result_or_content( + collaborator: str, expected_code: ErrorCode +) -> None: + sentinel = "sensitive-collaborator-exception-8472" + + class RaisingScanner(Scanner[ScannerConfig]): + @override + def _scan(self, text_block: str) -> tuple[Finding, ...]: + raise RuntimeError(sentinel) + + class RaisingHandler(RecordingHandler): + @override + def _normalize( + self, raw_body: bytes, policy_config: PolicyConfig + ) -> RequestBody: + raise RuntimeError(sentinel) + + scanner: Scanner[ScannerConfig] = ( + RaisingScanner(ScannerConfig(name="raising", entity_types=frozenset())) + if collaborator == "scanner" + else PassthroughScanner() + ) + handler: FormatHandler = ( + RaisingHandler() + if collaborator == "handler" + else RecordingHandler((TextBlock(path="text_block::alpha", text="text"),)) + ) + + with pytest.raises(PrivacyGuardError) as exception_info: + _processor(scanner, handler).process(_request()) + + assert exception_info.value.code is expected_code + assert exception_info.value.__cause__ is None + assert sentinel not in str(exception_info.value) + assert sentinel not in repr(exception_info.value) + + +def test_cataloged_handler_error_propagates_unchanged() -> None: + expected = PrivacyGuardError(ErrorCode.BODY_JSON_INVALID) + + class CatalogRaisingHandler(RecordingHandler): + @override + def _normalize( + self, raw_body: bytes, policy_config: PolicyConfig + ) -> RequestBody: + raise expected + + with pytest.raises(PrivacyGuardError) as exception_info: + _processor(PassthroughScanner(), CatalogRaisingHandler()).process(_request()) + + assert exception_info.value is expected + + +def test_reconstruction_equal_to_original_returns_no_replacement() -> None: + handler = RecordingHandler( + (TextBlock(path="text_block::alpha", text="text"),), reconstructed=b"body" + ) + + result = _processor(PassthroughScanner(), handler).process(_request(b"body")) + + assert result.replacement_body is None + + +def test_processing_result_is_immutable() -> None: + result = ProcessingResult(decision=ProcessingDecision.ALLOW) + + with pytest.raises(ValidationError): + setattr(result, "decision", ProcessingDecision.DENY) diff --git a/projects/privacy-guard/tests/test_single_block_composition.py b/projects/privacy-guard/tests/test_single_block_composition.py new file mode 100644 index 0000000..c4e743c --- /dev/null +++ b/projects/privacy-guard/tests/test_single_block_composition.py @@ -0,0 +1,51 @@ +import json + +import pytest + +from privacy_guard.body import JsonHandler +from privacy_guard.config import PolicyConfig +from privacy_guard.scanners import PassthroughScanner + + +@pytest.mark.parametrize( + ("raw_body", "selected_text_block_path", "expected_value"), + [ + ( + b'{"messages":[{"role":"user","content":"hello"}],"model":"model-a"}', + "/messages/0/content", + { + "messages": [{"role": "user", "content": "hello [test suffix]"}], + "model": "model-a", + }, + ), + ( + b'{"contents":[{"parts":[{"text":"hello"}]}],"model":"model-b"}', + "/contents/0/parts/0/text", + { + "contents": [{"parts": [{"text": "hello [test suffix]"}]}], + "model": "model-b", + }, + ), + ], +) +def test_one_selected_text_block_can_be_scanned_and_explicitly_replaced( + raw_body: bytes, selected_text_block_path: str, expected_value: object +) -> None: + json_handler = JsonHandler() + request_body = json_handler.normalize(raw_body, PolicyConfig()) + selected_text_block = next( + text_block + for text_block in request_body.text_blocks + if text_block.path == selected_text_block_path + ) + + findings = PassthroughScanner().scan(selected_text_block.text) + + assert findings == () + test_suffix = " [test suffix]" + replacement_text = selected_text_block.text + test_suffix + reconstructed_body = json_handler.reconstruct( + request_body, {selected_text_block.path: replacement_text} + ) + + assert json.loads(reconstructed_body) == expected_value diff --git a/projects/privacy-guard/tests/test_typing_policy.py b/projects/privacy-guard/tests/test_typing_policy.py new file mode 100644 index 0000000..b46991d --- /dev/null +++ b/projects/privacy-guard/tests/test_typing_policy.py @@ -0,0 +1,583 @@ +"""AST-enforced type-safety policy for handwritten Python.""" + +from __future__ import annotations + +import ast +from enum import Enum, auto +from pathlib import Path + +from typing_extensions import override + +_HANDWRITTEN_ROOTS = ("src", "tests", "examples") +_GENERATED_BINDINGS = Path("src/privacy_guard/bindings") +_TYPING_MODULES = frozenset({"typing", "typing_extensions"}) + + +class _TypingOrigin(Enum): + MODULE = auto() + CAST = auto() + DYNAMIC = auto() + LITERAL = auto() + ANNOTATED = auto() + + +class _TypingPolicyVisitor(ast.NodeVisitor): + """Resolve prohibited typing symbols without matching unrelated names.""" + + def __init__(self, relative_path: Path) -> None: + self._relative_path = relative_path + self._scopes: list[dict[str, _TypingOrigin | None]] = [{}] + self.violations: list[str] = [] + + def _record(self, node: ast.AST, message: str) -> None: + line_number = ( + node.lineno if isinstance(node, ast.expr | ast.stmt | ast.arg) else 0 + ) + self.violations.append( + f"{self._relative_path}:{line_number}: prohibited {message}" + ) + + def _lookup(self, name: str) -> _TypingOrigin | None: + for scope in reversed(self._scopes): + if name in scope: + return scope[name] + return None + + def _expression_origin(self, node: ast.expr) -> _TypingOrigin | None: + if isinstance(node, ast.Name): + return self._lookup(node.id) + if isinstance(node, ast.Attribute): + owner_origin = self._expression_origin(node.value) + if owner_origin is _TypingOrigin.MODULE: + if node.attr == "cast": + return _TypingOrigin.CAST + if node.attr == "Any": + return _TypingOrigin.DYNAMIC + if node.attr == "Literal": + return _TypingOrigin.LITERAL + if node.attr == "Annotated": + return _TypingOrigin.ANNOTATED + return None + + def _bind(self, name: str, origin: _TypingOrigin | None) -> None: + self._scopes[-1][name] = origin + + def _bind_target( + self, target: ast.expr, origin: _TypingOrigin | None = None + ) -> None: + if isinstance(target, ast.Name): + self._bind(target.id, origin) + elif isinstance(target, ast.List | ast.Tuple): + for element in target.elts: + self._bind_target(element) + elif isinstance(target, ast.Starred): + self._bind_target(target.value) + + def _visit_annotation(self, annotation: ast.expr | None) -> None: + if annotation is None: + return + self._visit_type_expression(annotation) + + @staticmethod + def _subscript_items(node: ast.expr) -> tuple[ast.expr, ...]: + return tuple(node.elts) if isinstance(node, ast.Tuple) else (node,) + + def _visit_type_expression(self, node: ast.expr) -> None: + if self._expression_origin(node) is _TypingOrigin.DYNAMIC: + self._record(node, "explicit Any") + return + if isinstance(node, ast.Constant) and isinstance(node.value, str): + try: + parsed = ast.parse(node.value, mode="eval") + except SyntaxError: + return + if self._type_expression_contains_dynamic(parsed.body): + self._record(node, "explicit Any annotation") + return + if isinstance(node, ast.Subscript): + self.visit(node.value) + items = self._subscript_items(node.slice) + origin = self._expression_origin(node.value) + if origin is _TypingOrigin.LITERAL: + for item in items: + self.visit(item) + return + if origin is _TypingOrigin.ANNOTATED and items: + self._visit_type_expression(items[0]) + for item in items[1:]: + self.visit(item) + return + for item in items: + self._visit_type_expression(item) + return + if isinstance(node, ast.BinOp): + self._visit_type_expression(node.left) + self._visit_type_expression(node.right) + return + if isinstance(node, ast.List | ast.Tuple): + for item in node.elts: + self._visit_type_expression(item) + return + if isinstance(node, ast.Starred): + self._visit_type_expression(node.value) + return + self.visit(node) + + def _inspect_type_comment( + self, + node: ast.stmt | ast.arg, + type_comment: str | None, + *, + function: bool = False, + ) -> None: + if type_comment is None: + return + try: + parsed = ast.parse(type_comment, mode="func_type" if function else "eval") + except SyntaxError: + return + if self._type_expression_contains_dynamic(parsed): + self._record(node, "explicit Any type comment") + + def _value_contains_dynamic(self, node: ast.AST) -> bool: + if ( + isinstance(node, ast.expr) + and self._expression_origin(node) is _TypingOrigin.DYNAMIC + ): + return True + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return False + return any( + self._value_contains_dynamic(child) for child in ast.iter_child_nodes(node) + ) + + def _type_expression_contains_dynamic(self, node: ast.AST, depth: int = 0) -> bool: + if depth > 8: + return False + if ( + isinstance(node, ast.expr) + and self._expression_origin(node) is _TypingOrigin.DYNAMIC + ): + return True + if isinstance(node, ast.Constant) and isinstance(node.value, str): + try: + parsed = ast.parse(node.value, mode="eval") + except SyntaxError: + return False + return self._type_expression_contains_dynamic(parsed.body, depth + 1) + if isinstance(node, ast.Subscript): + items = self._subscript_items(node.slice) + origin = self._expression_origin(node.value) + if origin is _TypingOrigin.LITERAL: + return any(self._value_contains_dynamic(item) for item in items) + if origin is _TypingOrigin.ANNOTATED and items: + return self._type_expression_contains_dynamic(items[0], depth) or any( + self._value_contains_dynamic(item) for item in items[1:] + ) + return any( + self._type_expression_contains_dynamic(child, depth) + for child in ast.iter_child_nodes(node) + ) + + def _visit_arguments(self, arguments: ast.arguments) -> None: + all_arguments = ( + *arguments.posonlyargs, + *arguments.args, + *arguments.kwonlyargs, + ) + for argument in all_arguments: + self._visit_annotation(argument.annotation) + self._inspect_type_comment(argument, argument.type_comment) + if arguments.vararg is not None: + self._visit_annotation(arguments.vararg.annotation) + self._inspect_type_comment(arguments.vararg, arguments.vararg.type_comment) + if arguments.kwarg is not None: + self._visit_annotation(arguments.kwarg.annotation) + self._inspect_type_comment(arguments.kwarg, arguments.kwarg.type_comment) + for default in arguments.defaults: + self.visit(default) + for default in arguments.kw_defaults: + if default is not None: + self.visit(default) + + def _bind_arguments(self, arguments: ast.arguments) -> None: + for argument in ( + *arguments.posonlyargs, + *arguments.args, + *arguments.kwonlyargs, + ): + self._bind(argument.arg, None) + if arguments.vararg is not None: + self._bind(arguments.vararg.arg, None) + if arguments.kwarg is not None: + self._bind(arguments.kwarg.arg, None) + + def _visit_function(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None: + for decorator in node.decorator_list: + self.visit(decorator) + self._visit_arguments(node.args) + self._visit_annotation(node.returns) + self._inspect_type_comment(node, node.type_comment, function=True) + self._bind(node.name, None) + self._scopes.append({}) + try: + self._bind_arguments(node.args) + for statement in node.body: + self.visit(statement) + finally: + self._scopes.pop() + + @override + def visit_Import(self, node: ast.Import) -> None: + for imported in node.names: + bound_name = imported.asname or imported.name.split(".", maxsplit=1)[0] + origin = _TypingOrigin.MODULE if imported.name in _TYPING_MODULES else None + self._bind(bound_name, origin) + + @override + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + is_typing_import = node.level == 0 and node.module in _TYPING_MODULES + for imported in node.names: + if imported.name == "*": + if is_typing_import: + self._record(node, "typing wildcard import") + continue + bound_name = imported.asname or imported.name + origin: _TypingOrigin | None = None + if is_typing_import and imported.name == "cast": + origin = _TypingOrigin.CAST + self._record(node, "typing cast import") + elif is_typing_import and imported.name == "Any": + origin = _TypingOrigin.DYNAMIC + self._record(node, "explicit Any import") + elif is_typing_import and imported.name == "Literal": + origin = _TypingOrigin.LITERAL + elif is_typing_import and imported.name == "Annotated": + origin = _TypingOrigin.ANNOTATED + self._bind(bound_name, origin) + + @override + def visit_Assign(self, node: ast.Assign) -> None: + self._inspect_type_comment(node, node.type_comment) + self.visit(node.value) + origin = self._expression_origin(node.value) + for target in node.targets: + self._bind_target(target, origin) + + @override + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: + self._visit_annotation(node.annotation) + if node.value is not None: + self.visit(node.value) + origin = self._expression_origin(node.value) + else: + origin = None + self._bind_target(node.target, origin) + + @override + def visit_NamedExpr(self, node: ast.NamedExpr) -> None: + self.visit(node.value) + self._bind_target(node.target, self._expression_origin(node.value)) + + def _visit_for(self, node: ast.For | ast.AsyncFor) -> None: + self.visit(node.iter) + self._bind_target(node.target) + self._inspect_type_comment(node, node.type_comment) + for statement in (*node.body, *node.orelse): + self.visit(statement) + + @override + def visit_For(self, node: ast.For) -> None: + self._visit_for(node) + + @override + def visit_AsyncFor(self, node: ast.AsyncFor) -> None: + self._visit_for(node) + + def _visit_with(self, node: ast.With | ast.AsyncWith) -> None: + for item in node.items: + self.visit(item.context_expr) + if item.optional_vars is not None: + self._bind_target(item.optional_vars) + self._inspect_type_comment(node, node.type_comment) + for statement in node.body: + self.visit(statement) + + @override + def visit_With(self, node: ast.With) -> None: + self._visit_with(node) + + @override + def visit_AsyncWith(self, node: ast.AsyncWith) -> None: + self._visit_with(node) + + @override + def visit_Call(self, node: ast.Call) -> None: + if self._expression_origin(node.func) is _TypingOrigin.CAST: + self._record(node, "typing cast call") + else: + self.visit(node.func) + for argument in node.args: + self.visit(argument) + for keyword in node.keywords: + self.visit(keyword.value) + + @override + def visit_Name(self, node: ast.Name) -> None: + origin = self._lookup(node.id) + if origin is _TypingOrigin.CAST: + self._record(node, "typing cast reference") + elif origin is _TypingOrigin.DYNAMIC: + self._record(node, "explicit Any") + + @override + def visit_Attribute(self, node: ast.Attribute) -> None: + origin = self._expression_origin(node) + if origin is _TypingOrigin.CAST: + self._record(node, "typing cast reference") + elif origin is _TypingOrigin.DYNAMIC: + self._record(node, "explicit Any") + else: + self.visit(node.value) + + @override + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + self._visit_function(node) + + @override + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + self._visit_function(node) + + @override + def visit_Lambda(self, node: ast.Lambda) -> None: + self._visit_arguments(node.args) + self._scopes.append({}) + try: + self._bind_arguments(node.args) + self.visit(node.body) + finally: + self._scopes.pop() + + @override + def visit_ClassDef(self, node: ast.ClassDef) -> None: + for decorator in node.decorator_list: + self.visit(decorator) + for base in node.bases: + self.visit(base) + for keyword in node.keywords: + self.visit(keyword.value) + self._bind(node.name, None) + self._scopes.append({}) + try: + for statement in node.body: + self.visit(statement) + finally: + self._scopes.pop() + + +def _handwritten_python_files(project_root: Path) -> tuple[Path, ...]: + files: list[Path] = [] + for source_root_name in _HANDWRITTEN_ROOTS: + source_root = project_root / source_root_name + if not source_root.is_dir(): + continue + for path in source_root.rglob("*"): + if path.suffix not in {".py", ".pyi"}: + continue + relative_path = path.relative_to(project_root) + if relative_path.is_relative_to(_GENERATED_BINDINGS): + continue + files.append(path) + return tuple(sorted(files)) + + +def _typing_policy_violations(project_root: Path) -> tuple[str, ...]: + violations: list[str] = [] + for path in _handwritten_python_files(project_root): + tree = ast.parse( + path.read_text(encoding="utf-8"), filename=str(path), type_comments=True + ) + visitor = _TypingPolicyVisitor(path.relative_to(project_root)) + visitor.visit(tree) + violations.extend(visitor.violations) + return tuple(violations) + + +def test_handwritten_python_is_cast_free_and_has_no_explicit_any() -> None: + project_root = Path(__file__).resolve().parents[1] + + assert _typing_policy_violations(project_root) == () + + +def test_typing_policy_rejects_all_supported_typing_origins(tmp_path: Path) -> None: + source_root = tmp_path / "src" + source_root.mkdir() + (source_root / "bad.py").write_text( + """ +import typing +import typing as t +import typing_extensions as extensions +from typing import Annotated as TypeAnnotated +from typing import Any as DynamicType +from typing_extensions import Annotated as ExtensionAnnotated +from typing_extensions import cast as narrow + +direct_dynamic: DynamicType +qualified_dynamic: t.Any +quoted_dynamic: "typing.Any" +extension_quoted: "extensions.Any" +nested_quoted: list["typing.Any"] +annotated_nested: t.Annotated["typing.Any", "metadata"] +annotated_alias_nested: TypeAnnotated["typing.Any", "metadata"] +annotated_extension_nested: extensions.Annotated["extensions.Any", "metadata"] +quoted_annotated_nested: "typing.Annotated['typing.Any', 'metadata']" +quoted_annotated_alias_nested: "TypeAnnotated['typing.Any', 'metadata']" +quoted_extension_annotated_alias_nested: ( + "ExtensionAnnotated['extensions.Any', 'metadata']" +) +literal_actual_dynamic: typing.Literal[typing.Any] +annotated_metadata_actual_dynamic: typing.Annotated[str, typing.Any] +direct_cast = narrow(str, object()) +qualified_cast = extensions.cast(str, object()) +rebound = t.cast +rebound_cast = rebound(str, object()) +""", + encoding="utf-8", + ) + (source_root / "bad_argument_comments.py").write_text( + """ +import typing + +def parameters( + positional_only, # type: typing.Any + /, + positional, # type: typing.Any + *variadic, # type: typing.Any + keyword_only, # type: typing.Any + **keywords, # type: typing.Any +): + return positional_only + +async def async_parameter( + value, # type: typing.Any +): + return value +""", + encoding="utf-8", + ) + (source_root / "bad_type_comments.py").write_text( + """ +import typing + +assigned = None # type: typing.Any + +for item in (): # type: typing.Any + pass + +with open(__file__) as stream: # type: typing.Any + pass + +def commented(value): + # type: (typing.Any) -> typing.Any + return value +""", + encoding="utf-8", + ) + + violations = _typing_policy_violations(tmp_path) + regular_violations = tuple( + violation for violation in violations if "src/bad.py:" in violation + ) + type_comment_violations = tuple( + violation + for violation in violations + if "src/bad_type_comments.py:" in violation + ) + argument_comment_violations = tuple( + violation + for violation in violations + if "src/bad_argument_comments.py:" in violation + ) + + assert any("explicit Any import" in violation for violation in regular_violations) + assert sum("explicit Any" in violation for violation in regular_violations) == 14 + assert any("typing cast import" in violation for violation in regular_violations) + assert sum("typing cast call" in violation for violation in regular_violations) == 3 + assert any("typing cast reference" in violation for violation in regular_violations) + assert len(type_comment_violations) == 4 + assert all( + "explicit Any type comment" in violation + for violation in type_comment_violations + ) + assert len(argument_comment_violations) == 6 + assert all( + "explicit Any type comment" in violation + for violation in argument_comment_violations + ) + + +def test_typing_policy_allows_unrelated_cast_and_any_names(tmp_path: Path) -> None: + source_root = tmp_path / "src" + source_root.mkdir() + (source_root / "allowed.py").write_text( + """ +import typing +import typing_extensions as extensions +from typing import Annotated as TypeAnnotated +from typing import Literal as TypeLiteral +from typing_extensions import Annotated as ExtensionAnnotated +from typing_extensions import Literal as ExtensionLiteral + + +class Domain: + Any = "ordinary domain value" + + +class Converter: + def cast(self, value: object) -> object: + return value + + +def cast(value: object) -> object: + return value + + +method_result = Converter().cast(Domain.Any) +function_result = cast(object()) +ordinary_string = "typing.Any" +ordinary_comment = None # type: object +literal_module: typing.Literal["typing.Any"] +literal_alias: TypeLiteral["typing.Any"] +literal_extension: extensions.Literal["extensions.Any"] +literal_extension_alias: ExtensionLiteral["extensions.Any"] +annotated_module: typing.Annotated[str, "typing.Any"] +annotated_alias: TypeAnnotated[str, "typing.Any"] +annotated_extension: extensions.Annotated[str, "extensions.Any"] +annotated_extension_alias: ExtensionAnnotated[str, "extensions.Any"] +quoted_literal: "typing.Literal['typing.Any']" +quoted_annotated: "extensions.Annotated[str, 'extensions.Any']" +quoted_literal_alias: "TypeLiteral['typing.Any']" +quoted_extension_literal_alias: "ExtensionLiteral['extensions.Any']" +quoted_annotated_alias: "TypeAnnotated[str, 'typing.Any']" +quoted_extension_annotated_alias: "ExtensionAnnotated[str, 'extensions.Any']" +""", + encoding="utf-8", + ) + + assert _typing_policy_violations(tmp_path) == () + + +def test_typing_policy_excludes_only_generated_bindings(tmp_path: Path) -> None: + generated = tmp_path / _GENERATED_BINDINGS + generated.mkdir(parents=True) + (generated / "generated.py").write_text( + "from typing import Any, cast\nvalue: Any = cast(Any, None)\n", + encoding="utf-8", + ) + handwritten = tmp_path / "src/privacy_guard/generated_elsewhere.py" + handwritten.write_text("from typing import Any\nvalue: Any\n", encoding="utf-8") + + violations = _typing_policy_violations(tmp_path) + + assert violations + assert all("generated_elsewhere.py" in violation for violation in violations) diff --git a/projects/privacy-guard/uv.lock b/projects/privacy-guard/uv.lock new file mode 100644 index 0000000..49f6485 --- /dev/null +++ b/projects/privacy-guard/uv.lock @@ -0,0 +1,376 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "grpcio" +version = "1.82.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/bc/656b89387d6f4ed7e0686c7b64c2ae7e554a759aa58122c8e5fb99392c32/grpcio-1.82.1.tar.gz", hash = "sha256:707b24abd90fcb1e45bcc080577da1dbf9971d107490589b9539af8e1e77b4b5", size = 13187300, upload-time = "2026-07-08T12:36:16.588Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/5b/e5092af97fa671ca279b3e373251af4bf87d5fbda7dc85f6a616899562a7/grpcio-1.82.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:0ddb18a9a9e1f46692b3567ae4abb3f8d117ce6afea48650f8eca06d8ab5d06f", size = 6181472, upload-time = "2026-07-08T12:34:31.009Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/18053a3a2ca03d0c2a1b8cc7271e705007a16aa5dae84bac00935c5b1a7f/grpcio-1.82.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:cf855b1af246720f567b0ce5d0724d45dfa4188eecc3296a2a69257b11b9e94b", size = 11970995, upload-time = "2026-07-08T12:34:33.603Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7e/21b1acb052876ad00959ec4d1b05fe08607d650bcfa282073bb164c2703c/grpcio-1.82.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddb30cb13e25bc13cea70ffc69d6d90c49d36ea6c1d4549e6912f70177834cac", size = 6760127, upload-time = "2026-07-08T12:34:36.122Z" }, + { url = "https://files.pythonhosted.org/packages/3e/12/25eef9c245c54f0061317d13a302357fe8ea03bac240b2b02ececcf54da4/grpcio-1.82.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1e822b2774f719c017cbe700b6e47173b6ae290fb84906f52a5a3c2c60b62e1e", size = 7484377, upload-time = "2026-07-08T12:34:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/a0/41/1a348767eb9d9bd7765dc4fa8a01723d3bb386d67f981ee5c6f9c02b8b1c/grpcio-1.82.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5dafb1ece8ed45dee7c738f166ec82e19673221ed5ab8967f72858a4685345b2", size = 6924269, upload-time = "2026-07-08T12:34:40.583Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b9/3aae7a03d34c86ea27988db859a6087c186f6c3f53f9b551e07afd989bfa/grpcio-1.82.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e06503106e7271e0a49fd5a1ac04747f1e47e87d900476db6fe45bc87ee411f4", size = 7531848, upload-time = "2026-07-08T12:34:43.277Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/3c4afa625d0dac9090707966916284c035fc5b2fb3e2c51e156accee6735/grpcio-1.82.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ff99bc8cafb6a952201c37b995f425e641c93ffa6e072258525feab57290141d", size = 8568217, upload-time = "2026-07-08T12:34:45.502Z" }, + { url = "https://files.pythonhosted.org/packages/20/9c/d8489c628e73e20a3d034e7f66912de7b1acb405f01d388f056a88e47924/grpcio-1.82.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:644ae1b94266ac785330f4590a69e52b6a7eb73029043a02209db81c81397d69", size = 7938771, upload-time = "2026-07-08T12:34:48.323Z" }, + { url = "https://files.pythonhosted.org/packages/4b/b7/0a92cfd1658f3a896d4aa12d4efeb7dd4ddfc723725ae22741a5241ea710/grpcio-1.82.1-cp311-cp311-win32.whl", hash = "sha256:e203d2e19d471630084a16c815616f8211dff21c268ab3c5f5bf38417832e074", size = 4256432, upload-time = "2026-07-08T12:34:50.432Z" }, + { url = "https://files.pythonhosted.org/packages/c7/6a/2872c761b025d9ec74386f22a4a7d59c5a5b00ebf718761b33739ffc45de/grpcio-1.82.1-cp311-cp311-win_amd64.whl", hash = "sha256:0d8299c285fe6cc6a1f56badf8d3bc5078c8d20273ee64bafa3783b4bc29a769", size = 5009633, upload-time = "2026-07-08T12:34:52.67Z" }, + { url = "https://files.pythonhosted.org/packages/dc/88/d1350bf3343a2ed87d801584e40609f6c6bd3087926eeca03de50348cf4a/grpcio-1.82.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:c09bd5fa0d5b1fbd773ec349fe61441c3e4ebf168c229aa7538a820bdfad6a58", size = 6144689, upload-time = "2026-07-08T12:34:55.567Z" }, + { url = "https://files.pythonhosted.org/packages/e6/33/71875cdecd27c24ac1385d4783a09853f01b84a825a36aec2a2bc7d0d080/grpcio-1.82.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1eae24810720734598e3e6a1a528d5de0f265fe3fc86575e9ecce424b9ec7379", size = 11952034, upload-time = "2026-07-08T12:34:58.128Z" }, + { url = "https://files.pythonhosted.org/packages/82/b2/d9125df3d8a140dec12cc82c05b7deafedeababcff6496f28b2fd5634d10/grpcio-1.82.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a6bd5daf5bde7b24d7ad2cbaf8bf9eac620d96222016bb5e7ddde930dec0673f", size = 6710772, upload-time = "2026-07-08T12:35:01.33Z" }, + { url = "https://files.pythonhosted.org/packages/88/9b/69e2d1627398b964f34437dc476a5aff5a2cc8e7f247d26272b5674b5faf/grpcio-1.82.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1ecfde669cb687ac020d31ff76debe5dc7a62213335f02262eb6625628da1c03", size = 7450677, upload-time = "2026-07-08T12:35:03.926Z" }, + { url = "https://files.pythonhosted.org/packages/e4/e7/8f855ca29c294956122a2a73023655b9b02602d5111dad2b9b00e7631c68/grpcio-1.82.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:011c8badee95734dee8bf05ce3464756a0ac3ebb8d443afd20c0e2b5e4640ad9", size = 6886855, upload-time = "2026-07-08T12:35:06.174Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f0/fa87e85f49925f44c479d07e58b051e69bcfef6b6d5fbc6749d140f6730a/grpcio-1.82.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b85f4564926fb23114d239392bdcae200db1e6179629edd7d7ab0ab89c96a197", size = 7501323, upload-time = "2026-07-08T12:35:08.49Z" }, + { url = "https://files.pythonhosted.org/packages/67/55/2e0b10ae1d3ef9dcc480b91dc2158f4931fc4675d3af0a2836e39b2a744f/grpcio-1.82.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2c0c8270833395644c3fe6b6a806397955a2bc0538000a19a78b90c05a6c16e0", size = 8536899, upload-time = "2026-07-08T12:35:10.975Z" }, + { url = "https://files.pythonhosted.org/packages/bd/95/a3d8b0431fa221efc51ee39d73595ede74ba82a43b7c4313192e580face2/grpcio-1.82.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2ba199205ff46c7778290fe1673c91ac8e7e45678dd5c86e9e56fa33ec8788f6", size = 7913892, upload-time = "2026-07-08T12:35:13.944Z" }, + { url = "https://files.pythonhosted.org/packages/b8/92/f2651ec704d9852a56faef394775038afba435b50ce82ab2404d119c3355/grpcio-1.82.1-cp312-cp312-win32.whl", hash = "sha256:06127691866e295c14e84a1fb86356dd962254f6abd0da4ca4b001eea9e89438", size = 4240985, upload-time = "2026-07-08T12:35:16.048Z" }, + { url = "https://files.pythonhosted.org/packages/96/4f/a5fe8bf0d0a1b24855f370293075c931f27de4eb55f0f158786095bf3c11/grpcio-1.82.1-cp312-cp312-win_amd64.whl", hash = "sha256:1fa3223a3a2e1db74f4c2b255189eb7ea875dfba56e221d252ee3fc7b204778e", size = 5001580, upload-time = "2026-07-08T12:35:18.689Z" }, + { url = "https://files.pythonhosted.org/packages/1b/3e/496992d08c0aaa11272eb6228dc8ab947da01fe835de243cd00521bce4c4/grpcio-1.82.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:b454a2d97bfab7565683a02345f86bd182ab69fd7c2bdb7414171e7538f266b1", size = 6146068, upload-time = "2026-07-08T12:35:21.365Z" }, + { url = "https://files.pythonhosted.org/packages/e7/8f/f263d6f14fdba6b56cfadd91fd3e158a52682b72c6016d1f8723d435659f/grpcio-1.82.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:3dde70abfc80b3be11de53ba0d601c439e7fb2afd3583ad1788d1146bec92fdc", size = 11948600, upload-time = "2026-07-08T12:35:24.312Z" }, + { url = "https://files.pythonhosted.org/packages/8c/14/3a02e6ee49c2d85bc15eaae321e0e11ab3542cad3c5b2de121ecce0c4296/grpcio-1.82.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5523099c98c292ea1ae08e617249db760c56a78f8deae879027fe7d1ffbcbf6", size = 6714591, upload-time = "2026-07-08T12:35:27.027Z" }, + { url = "https://files.pythonhosted.org/packages/69/80/58e3738696f48ab7645347b98d8a7f93d10e00e6218388fbfcd6c9310e3d/grpcio-1.82.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5e5c4dc0a59b0f8490a6bdfd6fc8395b9d8ad8a8407c7d67ca7b5bba15c0877f", size = 7454995, upload-time = "2026-07-08T12:35:29.599Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6c/2557c1a889363072fbf2285ecd0e8c44860d4dbd60f017a32537c5b863e2/grpcio-1.82.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c40d94ba820329cc191981bc22fa6f6eed0799c6d921f3c6709521d59d4a2fd7", size = 6888621, upload-time = "2026-07-08T12:35:32.38Z" }, + { url = "https://files.pythonhosted.org/packages/d2/66/907706ccaff1223f1e10fd5b37fc16faead43392fccb4e786e7e390ac141/grpcio-1.82.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c816180e31e273caaec6f8bd86a8392499d5bbb26f41da44e3dce48bde69095", size = 7505069, upload-time = "2026-07-08T12:35:35.072Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7c/ff97b0d0f635987ee5ec80dfedafa1aad629303745d48e8637d10eec5b80/grpcio-1.82.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e31fd780b261830720cb70b0fd8f0aa51d49e75a66d7464ad2e31d4b765f2580", size = 8535384, upload-time = "2026-07-08T12:35:37.954Z" }, + { url = "https://files.pythonhosted.org/packages/62/9e/a97fddd970a8d1588cade06eca20443761c1858b0ad6590a5c835aa18062/grpcio-1.82.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d76152d7c31d7210d4a106e5d8b64da5bba5d6abf11be30e2f7b0a0c59bbcbf", size = 7910707, upload-time = "2026-07-08T12:35:40.797Z" }, + { url = "https://files.pythonhosted.org/packages/20/e4/eaba1517888af483a88d449eb7566f0f7f63446d46f339c5891798435875/grpcio-1.82.1-cp313-cp313-win32.whl", hash = "sha256:38e9dcb5258226fb3282630b31b16a968df52c8c6ad514af540646e0a4578f8a", size = 4240363, upload-time = "2026-07-08T12:35:43.298Z" }, + { url = "https://files.pythonhosted.org/packages/b0/42/66a98d47732e35290bef722f6149fed3709cd4cf61166f6f53a12f417302/grpcio-1.82.1-cp313-cp313-win_amd64.whl", hash = "sha256:3dbfb52c36d9511ac2b8e6c94fdde837b393ae520cc321f52a333a2deedf5a90", size = 5000980, upload-time = "2026-07-08T12:35:46.262Z" }, + { url = "https://files.pythonhosted.org/packages/b4/cb/cf9ae9e164c6e6dc8a494faa9771763df9da150eefe19671009624d1559f/grpcio-1.82.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:35f990f7784c8fd2872644f07f96ebb4d9e48e145a190ab80d0280af91a1bfb2", size = 6146901, upload-time = "2026-07-08T12:35:49.261Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2a/eccf26dbcfb7f7cab8027c5490a16c8937c5aa7a2ec20a3eab2cf7a43165/grpcio-1.82.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:46536a4a1f4434df3c851b9254ff6fc7df5705b273681a15ca277d5921c178a0", size = 11954756, upload-time = "2026-07-08T12:35:52.196Z" }, + { url = "https://files.pythonhosted.org/packages/ad/75/3b3b4a3cc9f084b026af96e1d3e539b1af29ec7f41ed0dfff3cb99cc8626/grpcio-1.82.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d6650a7c1ebb7921c70e12a385439a8118efb99e669fa9ed31cf25db1843937c", size = 6723087, upload-time = "2026-07-08T12:35:54.973Z" }, + { url = "https://files.pythonhosted.org/packages/9c/8b/b0f0c9b1400a99a4da4c09b114f101b192f8f11192e76f620b8962f5d90b/grpcio-1.82.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b8e110c66df5204c0506d6c8787b35d48b8b699ef5aa366d6c4d67325c67fe9a", size = 7454542, upload-time = "2026-07-08T12:35:57.586Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bd/428e38868382aa193697a5aa53973f29c58e58ba4268aa0c86a2715ee58b/grpcio-1.82.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f853eae07235a51a27bb5d6a9a175a59ca55dc9b99edc6ce2f76f07332d333ae", size = 6889588, upload-time = "2026-07-08T12:36:00.012Z" }, + { url = "https://files.pythonhosted.org/packages/49/ce/03e01d5e10259bf5c08ee50570cc94724e79c956f61fd2f09b341af0956c/grpcio-1.82.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:60b0f2c95337694fc094b77d9f60f50566c84b5677393e342eb98daeee242d98", size = 7514166, upload-time = "2026-07-08T12:36:02.693Z" }, + { url = "https://files.pythonhosted.org/packages/ff/59/278b4b600329e2ba3849f3c1ea3c820b3a01b38a7ad184ba09595e8d2733/grpcio-1.82.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:b064fc444812bdaa9825d33c26f8d732d63ee6a5d78557c1faf92c98687fed27", size = 8536166, upload-time = "2026-07-08T12:36:05.349Z" }, + { url = "https://files.pythonhosted.org/packages/44/27/7ccf2ef00f27a8e47a79d641c8ceaf7d3028c7a03d9a97b4c8a9a783c086/grpcio-1.82.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7d7ede11d747b4e1bd05e3bc0260e155b65a88735a895a10f6521f19b889511e", size = 7912572, upload-time = "2026-07-08T12:36:08.393Z" }, + { url = "https://files.pythonhosted.org/packages/0d/be/33742482d2753f2d3a1b7641664b6622262d44f2f3b609f13425dd86d36f/grpcio-1.82.1-cp314-cp314-win32.whl", hash = "sha256:3d21f19838dc255ecbb79321b15ae9b98fbddff4c3d4aedb0a81bdd7f4ab572a", size = 4321856, upload-time = "2026-07-08T12:36:10.899Z" }, + { url = "https://files.pythonhosted.org/packages/cc/67/03329c847172c78ddeb1eb9be6b444fdbc12775a84c958b27e427e7b926d/grpcio-1.82.1-cp314-cp314-win_amd64.whl", hash = "sha256:e20f1edbb15f99e3128ec86433f9785fd5a451d8f115e74fe0056134f092a9d5", size = 5141114, upload-time = "2026-07-08T12:36:13.595Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "privacy-guard" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "typing-extensions" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "ruff" }, + { name = "ty" }, +] + +[package.metadata] +requires-dist = [ + { name = "grpcio", specifier = ">=1.81.1,<2" }, + { name = "protobuf", specifier = ">=6.33.5,<7" }, + { name = "pydantic", specifier = ">=2.11,<3" }, + { name = "typing-extensions", specifier = ">=4.12,<5" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.3,<9" }, + { name = "pytest-asyncio", specifier = ">=0.25,<2" }, + { name = "ruff", specifier = ">=0.12,<0.13" }, + { name = "ty", specifier = ">=0.0.1a16,<0.1" }, +] + +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + +[[package]] +name = "ruff" +version = "0.12.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/f0/e0965dd709b8cabe6356811c0ee8c096806bb57d20b5019eb4e48a117410/ruff-0.12.12.tar.gz", hash = "sha256:b86cd3415dbe31b3b46a71c598f4c4b2f550346d1ccf6326b347cc0c8fd063d6", size = 5359915, upload-time = "2025-09-04T16:50:18.273Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/79/8d3d687224d88367b51c7974cec1040c4b015772bfbeffac95face14c04a/ruff-0.12.12-py3-none-linux_armv6l.whl", hash = "sha256:de1c4b916d98ab289818e55ce481e2cacfaad7710b01d1f990c497edf217dafc", size = 12116602, upload-time = "2025-09-04T16:49:18.892Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c3/6e599657fe192462f94861a09aae935b869aea8a1da07f47d6eae471397c/ruff-0.12.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7acd6045e87fac75a0b0cdedacf9ab3e1ad9d929d149785903cff9bb69ad9727", size = 12868393, upload-time = "2025-09-04T16:49:23.043Z" }, + { url = "https://files.pythonhosted.org/packages/e8/d2/9e3e40d399abc95336b1843f52fc0daaceb672d0e3c9290a28ff1a96f79d/ruff-0.12.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:abf4073688d7d6da16611f2f126be86523a8ec4343d15d276c614bda8ec44edb", size = 12036967, upload-time = "2025-09-04T16:49:26.04Z" }, + { url = "https://files.pythonhosted.org/packages/e9/03/6816b2ed08836be272e87107d905f0908be5b4a40c14bfc91043e76631b8/ruff-0.12.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:968e77094b1d7a576992ac078557d1439df678a34c6fe02fd979f973af167577", size = 12276038, upload-time = "2025-09-04T16:49:29.056Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d5/707b92a61310edf358a389477eabd8af68f375c0ef858194be97ca5b6069/ruff-0.12.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42a67d16e5b1ffc6d21c5f67851e0e769517fb57a8ebad1d0781b30888aa704e", size = 11901110, upload-time = "2025-09-04T16:49:32.07Z" }, + { url = "https://files.pythonhosted.org/packages/9d/3d/f8b1038f4b9822e26ec3d5b49cf2bc313e3c1564cceb4c1a42820bf74853/ruff-0.12.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b216ec0a0674e4b1214dcc998a5088e54eaf39417327b19ffefba1c4a1e4971e", size = 13668352, upload-time = "2025-09-04T16:49:35.148Z" }, + { url = "https://files.pythonhosted.org/packages/98/0e/91421368ae6c4f3765dd41a150f760c5f725516028a6be30e58255e3c668/ruff-0.12.12-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:59f909c0fdd8f1dcdbfed0b9569b8bf428cf144bec87d9de298dcd4723f5bee8", size = 14638365, upload-time = "2025-09-04T16:49:38.892Z" }, + { url = "https://files.pythonhosted.org/packages/74/5d/88f3f06a142f58ecc8ecb0c2fe0b82343e2a2b04dcd098809f717cf74b6c/ruff-0.12.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ac93d87047e765336f0c18eacad51dad0c1c33c9df7484c40f98e1d773876f5", size = 14060812, upload-time = "2025-09-04T16:49:42.732Z" }, + { url = "https://files.pythonhosted.org/packages/13/fc/8962e7ddd2e81863d5c92400820f650b86f97ff919c59836fbc4c1a6d84c/ruff-0.12.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:01543c137fd3650d322922e8b14cc133b8ea734617c4891c5a9fccf4bfc9aa92", size = 13050208, upload-time = "2025-09-04T16:49:46.434Z" }, + { url = "https://files.pythonhosted.org/packages/53/06/8deb52d48a9a624fd37390555d9589e719eac568c020b27e96eed671f25f/ruff-0.12.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2afc2fa864197634e549d87fb1e7b6feb01df0a80fd510d6489e1ce8c0b1cc45", size = 13311444, upload-time = "2025-09-04T16:49:49.931Z" }, + { url = "https://files.pythonhosted.org/packages/2a/81/de5a29af7eb8f341f8140867ffb93f82e4fde7256dadee79016ac87c2716/ruff-0.12.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0c0945246f5ad776cb8925e36af2438e66188d2b57d9cf2eed2c382c58b371e5", size = 13279474, upload-time = "2025-09-04T16:49:53.465Z" }, + { url = "https://files.pythonhosted.org/packages/7f/14/d9577fdeaf791737ada1b4f5c6b59c21c3326f3f683229096cccd7674e0c/ruff-0.12.12-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a0fbafe8c58e37aae28b84a80ba1817f2ea552e9450156018a478bf1fa80f4e4", size = 12070204, upload-time = "2025-09-04T16:49:56.882Z" }, + { url = "https://files.pythonhosted.org/packages/77/04/a910078284b47fad54506dc0af13839c418ff704e341c176f64e1127e461/ruff-0.12.12-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b9c456fb2fc8e1282affa932c9e40f5ec31ec9cbb66751a316bd131273b57c23", size = 11880347, upload-time = "2025-09-04T16:49:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/df/58/30185fcb0e89f05e7ea82e5817b47798f7fa7179863f9d9ba6fd4fe1b098/ruff-0.12.12-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5f12856123b0ad0147d90b3961f5c90e7427f9acd4b40050705499c98983f489", size = 12891844, upload-time = "2025-09-04T16:50:02.591Z" }, + { url = "https://files.pythonhosted.org/packages/21/9c/28a8dacce4855e6703dcb8cdf6c1705d0b23dd01d60150786cd55aa93b16/ruff-0.12.12-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:26a1b5a2bf7dd2c47e3b46d077cd9c0fc3b93e6c6cc9ed750bd312ae9dc302ee", size = 13360687, upload-time = "2025-09-04T16:50:05.8Z" }, + { url = "https://files.pythonhosted.org/packages/c8/fa/05b6428a008e60f79546c943e54068316f32ec8ab5c4f73e4563934fbdc7/ruff-0.12.12-py3-none-win32.whl", hash = "sha256:173be2bfc142af07a01e3a759aba6f7791aa47acf3604f610b1c36db888df7b1", size = 12052870, upload-time = "2025-09-04T16:50:09.121Z" }, + { url = "https://files.pythonhosted.org/packages/85/60/d1e335417804df452589271818749d061b22772b87efda88354cf35cdb7a/ruff-0.12.12-py3-none-win_amd64.whl", hash = "sha256:e99620bf01884e5f38611934c09dd194eb665b0109104acae3ba6102b600fd0d", size = 13178016, upload-time = "2025-09-04T16:50:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/28/7e/61c42657f6e4614a4258f1c3b0c5b93adc4d1f8575f5229d1906b483099b/ruff-0.12.12-py3-none-win_arm64.whl", hash = "sha256:2a8199cab4ce4d72d158319b63370abf60991495fb733db96cd923a34c52d093", size = 12256762, upload-time = "2025-09-04T16:50:15.737Z" }, +] + +[[package]] +name = "ty" +version = "0.0.61" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/63/6944925d0fe9a4bb9cc744e6c045a42bbd2ee4654c103190674577a36c3f/ty-0.0.61.tar.gz", hash = "sha256:acbf0d914cc7e2e57ccc440036af36114819e2a604a5ffb554e72e4ca7dd65a2", size = 6234957, upload-time = "2026-07-18T01:39:54.696Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/cf/044f31523e2768e3e64b0ca2ec32f70b3a731d4a2caa6ea110baf26e251c/ty-0.0.61-py3-none-linux_armv6l.whl", hash = "sha256:148779b8675eac93f40ec58bd70037fe67537117f20a23272264f8f136d41336", size = 11891448, upload-time = "2026-07-18T01:39:18.449Z" }, + { url = "https://files.pythonhosted.org/packages/d2/55/558cfe76b65d91d1854bbfac336020bd42fd887caa632d845d13c0c539eb/ty-0.0.61-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:08217382b3385808ee7288501ea3214b32631b08d1fd091ece6799b0c95264c5", size = 11602442, upload-time = "2026-07-18T01:39:20.914Z" }, + { url = "https://files.pythonhosted.org/packages/27/be/78c0ae6634cd606a68e5b46b338db427a48a1800c96a749b2d2f7a702e03/ty-0.0.61-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6d99c729011b47dec20e78a32ac9c8f6defd4cf62f7bb851bbccf70dde6cee50", size = 11125286, upload-time = "2026-07-18T01:39:22.893Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/a40793962f1b6337938ddb0bca7496b54e70879e23b4d2cc8dfd7e5d1af3/ty-0.0.61-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cda607978ae271b77e51c947663218bce635c3507e256865444b10c37cdb60d", size = 11663403, upload-time = "2026-07-18T01:39:25.017Z" }, + { url = "https://files.pythonhosted.org/packages/98/c1/7879244da5b30407dc368946d36be5024380073408b079f144ffe034030e/ty-0.0.61-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0d78f160a0f9434d570cdcdbc4dafba1f6aac3c47a32f9f63995b3cb55ffe4b6", size = 11715250, upload-time = "2026-07-18T01:39:27.045Z" }, + { url = "https://files.pythonhosted.org/packages/35/c4/8a4637cd58abd37f315dd515e24c582986cb1bfdf2edc4786882f5a4f69a/ty-0.0.61-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09aeab4800b36e93e4ce918699004da642d74988cac920b7592a6a2b9be6611c", size = 12393876, upload-time = "2026-07-18T01:39:29.197Z" }, + { url = "https://files.pythonhosted.org/packages/27/4b/27e7c640b1272743503229aa17ae2167a538040c4716a2fa1777c2b34fea/ty-0.0.61-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dccc8136df44142a109953a168be17b4915c99876b047d0b6672c31dae939bdf", size = 12958187, upload-time = "2026-07-18T01:39:31.308Z" }, + { url = "https://files.pythonhosted.org/packages/3a/f5/70eaaefb6081fb0a8115cff66fbfaa20dafac8c646df2477adad95a59de2/ty-0.0.61-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:220760c2d13a887d027ee1093172c24ac35b6e634805329c93a30908ae4d3f5c", size = 12560101, upload-time = "2026-07-18T01:39:33.35Z" }, + { url = "https://files.pythonhosted.org/packages/e3/5a/17bae3b6429b5c479dc6c1e344d34e1f79efbc27531f15f3ee5b5da63745/ty-0.0.61-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:effefbb89da7128d18059529d1c2ea390fe7f1f3882690d257ca2143d49a0c34", size = 12225389, upload-time = "2026-07-18T01:39:35.436Z" }, + { url = "https://files.pythonhosted.org/packages/d2/0e/2ac380ba20d6395542c8df1d6fa4f00e2aead784c2e6aaefa1e02ed0610c/ty-0.0.61-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ba8b28a5ef811d5bb6461e37d76110c06fd20487474865c323d3d18b08b972b2", size = 12548403, upload-time = "2026-07-18T01:39:37.556Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e5/7da4b73e825e1a9808c26d68b0156e9a37aede1846191210dfffb8c64042/ty-0.0.61-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:88ecd6d9b05e8174b1860dac9bd3e188d6cef5702b0d3239fd9f94f6ac73a29d", size = 11621813, upload-time = "2026-07-18T01:39:39.919Z" }, + { url = "https://files.pythonhosted.org/packages/9b/3c/5b58015e998cd0d89b17a463b6321421457d86d987574e8dac65ddfceba3/ty-0.0.61-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb0cdfe4c48542ffb9a1139825dfa3d4aae49e96e966682ef7da762ab97831ff", size = 11734101, upload-time = "2026-07-18T01:39:42.097Z" }, + { url = "https://files.pythonhosted.org/packages/a6/21/294f4cc819b7b12ed659fd860e5cdfbd592d4c768c8f23596685dbc43e6b/ty-0.0.61-py3-none-musllinux_1_2_i686.whl", hash = "sha256:dff03873c0c3d0b44738f8b6d403b0756a31cf54c65136397df7624c6159b1f0", size = 11988401, upload-time = "2026-07-18T01:39:44.183Z" }, + { url = "https://files.pythonhosted.org/packages/2e/26/0f96f79fdac118521a9771e9eef3f9b3f447d647b2c77953e80a1715c7e8/ty-0.0.61-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a9210e80e3d41c1dfc751e9e8e0980272f475031fafd0fb0f48aee233c78da03", size = 12330624, upload-time = "2026-07-18T01:39:46.662Z" }, + { url = "https://files.pythonhosted.org/packages/e3/08/1e62d1bca5c0cebdc7a34db1f4b61557aab85961cedd56953dd2c32d3e66/ty-0.0.61-py3-none-win32.whl", hash = "sha256:e3e1fe06f49a5492a922a5df2739834aa5ee978c7dd10414119dc8755cc40c9c", size = 11313991, upload-time = "2026-07-18T01:39:48.761Z" }, + { url = "https://files.pythonhosted.org/packages/26/f1/d8e33b3aeb36b73d81ae34d10e46ec4abf506d68f4e0a1491a76a593dd42/ty-0.0.61-py3-none-win_amd64.whl", hash = "sha256:25f2291169e0298fcdbba1b1fea64f8207a6c1908dddef32346fd5e3e6ac9221", size = 12311717, upload-time = "2026-07-18T01:39:50.881Z" }, + { url = "https://files.pythonhosted.org/packages/e1/14/7caec26d93a943c0e7d15eb7374644508d08cbd387d112b722b12d14e044/ty-0.0.61-py3-none-win_arm64.whl", hash = "sha256:3e496f7698bc4b5bbb1eb66d8b5799ba87596d88d36604ca359083893fa2fc49", size = 11693485, upload-time = "2026-07-18T01:39:52.73Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] From 3b36f682228e59fa3479dcfbff4c8aa2458c631f Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Wed, 22 Jul 2026 21:03:59 +0000 Subject: [PATCH 02/82] Refine Privacy Guard structure and examples --- AGENTS.md | 2 + projects/privacy-guard/BENCHMARKS.md | 15 +- projects/privacy-guard/CONTRIBUTING.md | 9 +- projects/privacy-guard/README.md | 34 +-- .../README.md | 23 +- .../gateway.toml | 4 +- .../middleware_server.py | 18 +- .../policy.yaml | 2 +- .../examples/openshell-e2e/README.md | 70 ------- .../examples/openshell-e2e/capture_server.py | 60 ------ .../examples/openshell-e2e/policy.yaml | 38 ---- .../examples/openshell-e2e/run.sh | 197 ------------------ .../src/privacy_guard/processor.py | 14 +- .../{body => request_body}/__init__.py | 10 +- .../{body => request_body}/base.py | 0 .../{body => request_body}/json.py | 2 +- .../tests/examples/test_email_scanner.py | 36 ++++ .../tests/examples/test_openshell_e2e.py | 80 ------- .../tests/{body => request_body}/test_base.py | 4 +- .../tests/{body => request_body}/test_json.py | 21 +- projects/privacy-guard/tests/test_config.py | 6 +- .../privacy-guard/tests/test_hardening.py | 6 +- .../privacy-guard/tests/test_processor.py | 2 +- .../tests/test_single_block_composition.py | 2 +- 24 files changed, 138 insertions(+), 517 deletions(-) rename projects/privacy-guard/examples/{openshell-manual => email-scanner}/README.md (77%) rename projects/privacy-guard/examples/{openshell-manual => email-scanner}/gateway.toml (73%) rename projects/privacy-guard/examples/{openshell-e2e => email-scanner}/middleware_server.py (68%) rename projects/privacy-guard/examples/{openshell-manual => email-scanner}/policy.yaml (96%) delete mode 100644 projects/privacy-guard/examples/openshell-e2e/README.md delete mode 100644 projects/privacy-guard/examples/openshell-e2e/capture_server.py delete mode 100644 projects/privacy-guard/examples/openshell-e2e/policy.yaml delete mode 100755 projects/privacy-guard/examples/openshell-e2e/run.sh rename projects/privacy-guard/src/privacy_guard/{body => request_body}/__init__.py (85%) rename projects/privacy-guard/src/privacy_guard/{body => request_body}/base.py (100%) rename projects/privacy-guard/src/privacy_guard/{body => request_body}/json.py (99%) create mode 100644 projects/privacy-guard/tests/examples/test_email_scanner.py delete mode 100644 projects/privacy-guard/tests/examples/test_openshell_e2e.py rename projects/privacy-guard/tests/{body => request_body}/test_base.py (98%) rename projects/privacy-guard/tests/{body => request_body}/test_json.py (97%) diff --git a/AGENTS.md b/AGENTS.md index 503b4a4..03de54f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,6 +20,8 @@ read `docs/development/index.md`. ## Repository rules - Make the smallest change that satisfies the task and preserve unrelated work. +- Prefer explicit, clear names and language over concise but ambiguous + alternatives. Value concision when it does not reduce clarity. - Use `uv` for Python dependency management, environments, locking, builds, and command execution unless a project explicitly documents an exception. Treat `pyproject.toml` and the committed `uv.lock` as the dependency sources of truth. diff --git a/projects/privacy-guard/BENCHMARKS.md b/projects/privacy-guard/BENCHMARKS.md index a4a1b3f..651c8db 100644 --- a/projects/privacy-guard/BENCHMARKS.md +++ b/projects/privacy-guard/BENCHMARKS.md @@ -1,4 +1,9 @@ -# Full-path benchmark evidence +# Diagnostic benchmark + +This is a manual development tool, not a contributor or release gate. It has no +pass/fail thresholds, and results should be compared only across equivalent, +controlled environments. The recorded table below is a one-time snapshot, not +an evergreen performance baseline. The benchmark invokes `RequestProcessor.process` and therefore covers JSON normalization, scanner calls, scanner-output validation, contextual finding @@ -63,3 +68,11 @@ domain. The maximum-width peak includes the unavoidable overlap while the cached Pydantic adapter constructs its validated output from the stdlib parse tree; the raw parse tree becomes unreachable when the parse helper returns, before text-block materialization begins. + +The harness deliberately isolates processor overhead with a synthetic scanner. +It does not represent the latency of a future production PII scanner and does +not include protobuf conversion, gRPC transport, executor or semaphore queuing, +concurrent throughput, or whole-process memory. Add realistic request +distributions and service-level concurrency measurements when a production +scanner is introduced; establish regression thresholds only on pinned +infrastructure. diff --git a/projects/privacy-guard/CONTRIBUTING.md b/projects/privacy-guard/CONTRIBUTING.md index 2873e64..4b4e1f3 100644 --- a/projects/privacy-guard/CONTRIBUTING.md +++ b/projects/privacy-guard/CONTRIBUTING.md @@ -39,7 +39,8 @@ suppressions narrow and rule-specific. Production and test overrides are explicit; example scripts omit `@override` to keep their public API surface minimal and copyable. -Run the committed full-path performance evidence with: +When changing performance-sensitive processing code, run the optional +diagnostic benchmark with: ```bash uv run --frozen python scripts/benchmark_privacy_guard.py --suite full @@ -47,4 +48,8 @@ uv run --frozen python scripts/benchmark_privacy_guard.py --suite full Use the default quick suite during iteration. Both suites verify findings and reconstruction outcomes before reporting median wall time and peak traced -allocation. +allocation. The benchmark is not part of the contributor gate and has no +pass/fail thresholds; compare results only on a controlled environment. It uses +a synthetic scanner and the synchronous processor path, so it does not measure +real scanner cost, service concurrency, queuing, gRPC adaptation, or process +RSS. diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 99bb120..9d64dc2 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -12,9 +12,9 @@ original. > a loopback server are implemented. The production scanner is still the > passthrough scanner; no real PII scanner exists yet. -The [OpenShell E2E example](examples/openshell-e2e/README.md) launches the -middleware, attaches it to a disposable sandbox, and verifies that OpenShell -forwards a reconstructed provider-shaped request to a local capture endpoint. +The self-contained [email scanner example](examples/email-scanner/README.md) +provides a deterministic scanner, middleware entry point, gateway registration, +sandbox policy, and manual Claude Code workflow. ## Request flow @@ -23,10 +23,12 @@ forwards a reconstructed provider-shaped request to a local capture endpoint. ``` proto HttpRequestEvaluation -> payloads.InterceptedRequest proto-free capture of the request - -> body.FormatHandler.normalize() -> body.RequestBody (TextBlocks) + -> request_body.FormatHandler.normalize() + -> request_body.RequestBody (TextBlocks) -> scanners.Scanner.scan(text_block) -> scanners.Finding[] -> policy via config.PolicyConfig -> per-block replacements - -> body.FormatHandler.reconstruct() -> rewritten body (bytes) + -> request_body.FormatHandler.reconstruct() + -> rewritten body (bytes) -> payloads.ProcessingResult decision + replacement + findings -> proto HttpRequestResult ``` @@ -86,7 +88,7 @@ and entity. A scanner sequence is passed to `RequestProcessor`; scanner names must be unique and remain visible in aggregated findings. The default service uses `PassthroughScanner`. The deterministic email regex and -its server entry point live entirely under `examples/openshell-e2e`; production +its server entry point live entirely under `examples/email-scanner`; production package modules neither export nor select it. A scanner is a nominal extension: declare its strict configuration type and @@ -152,7 +154,7 @@ release its scanner slot until its synchronous worker really exits. | `constants` | Package-wide limits, service metadata, and stable protocol values | | `errors` | Closed, content-safe error catalog shared by all components | | `payloads` | Frozen `InterceptedRequest` and `ProcessingResult` domain records | -| `body` | Nominal `FormatHandler` ABC + `JsonHandler`; strict `RequestBody`, `TextBlock` models | +| `request_body` | Nominal `FormatHandler` ABC + `JsonHandler`; strict `RequestBody`, `TextBlock` models | | `scanners` | `Scanner` ABC + strict `ScannerConfig`, `Finding`, and `RequestBodyFinding` models | | `processor` | Proto-free request orchestration and policy application | | `service` | High-level `MiddlewareServer`, gRPC lifecycle, and servicer adapter | @@ -184,7 +186,7 @@ release its scanner slot until its synchronous worker really exits. from typing_extensions import override - from privacy_guard.body import FormatHandler, RequestBody + from privacy_guard.request_body import FormatHandler, RequestBody from privacy_guard.config import PolicyConfig @@ -227,16 +229,20 @@ The AST policy test rejects cast operations and explicit dynamic typing in handwritten `src`, `tests`, and `examples`; only generated protobuf/gRPC bindings are excluded. -The deterministic benchmark gate uses three samples per median and covers every -required size, finding load, scanner shape, and reconstruction mode: +The optional diagnostic benchmark uses three samples per median and covers a +focused set of body sizes, finding loads, scanner shapes, and reconstruction +modes: ```bash uv run --frozen python scripts/benchmark_privacy_guard.py ``` -For the broader scenario set and seven samples per median, pass `--suite full`. +Use it manually when changing performance-sensitive processing code; it is not +part of `scripts/validate.sh` and does not enforce regression thresholds. For +the broader scenario set and seven samples per median, pass `--suite full`. Pass `--profile profile.out` to either suite to record a `cProfile` artifact. The harness reports median wall time and median peak traced allocation for the -complete normalize, scan, output validation, policy, and reconstruction path. -Committed reference measurements and methodology are in -[BENCHMARKS.md](BENCHMARKS.md). +synchronous normalize, synthetic scan, output validation, policy, and +reconstruction path. It does not measure a real PII scanner, gRPC adaptation, +executor queuing, concurrent throughput, or process RSS. Methodology and one +platform-specific development snapshot are in [BENCHMARKS.md](BENCHMARKS.md). diff --git a/projects/privacy-guard/examples/openshell-manual/README.md b/projects/privacy-guard/examples/email-scanner/README.md similarity index 77% rename from projects/privacy-guard/examples/openshell-manual/README.md rename to projects/privacy-guard/examples/email-scanner/README.md index 063475a..bbe56f3 100644 --- a/projects/privacy-guard/examples/openshell-manual/README.md +++ b/projects/privacy-guard/examples/email-scanner/README.md @@ -1,11 +1,16 @@ -# Manually try Privacy Guard with Claude Code +# Email scanner example -This example temporarily runs the installed OpenShell gateway with a config from -this directory. It does not create or modify `~/.config/openshell/gateway.toml`, -and it does not create a project-local state directory. +This self-contained example supplies a deterministic email scanner, its Privacy +Guard server entry point, and the OpenShell gateway and sandbox policy needed to +try redaction with Claude Code. It temporarily runs the installed OpenShell +gateway with the config from this directory. It does not create or modify +`~/.config/openshell/gateway.toml`, and it does not create a project-local state +directory. -The example scanner detects email addresses in Claude Code request bodies and -replaces them with `[email]` before Anthropic receives the request. +The illustrative regex scanner in `middleware_server.py` detects email-shaped +text in Claude Code request bodies. The policy replaces matches with `[email]` +before Anthropic receives the request. The scanner is intentionally small and +deterministic; it is not intended for production PII detection. ## Prerequisites @@ -20,7 +25,7 @@ replaces them with `[email]` before Anthropic receives the request. Run the example commands from this directory: ```bash -cd projects/privacy-guard/examples/openshell-manual +cd projects/privacy-guard/examples/email-scanner ``` ## 1. Edit the example config @@ -46,7 +51,7 @@ Only the checked-out `gateway.toml` is edited. Do not copy it into In terminal 1: ```bash -uv run --project ../.. python ../openshell-e2e/middleware_server.py \ +uv run --project ../.. python middleware_server.py \ --listen 0.0.0.0:50051 ``` @@ -144,6 +149,6 @@ brew services start openshell Stop Privacy Guard with `Ctrl-C` in terminal 1. No default OpenShell config was changed. -This test uses Claude Code because subscription prompts are sent in inspectable +This example uses Claude Code because subscription prompts are sent in inspectable HTTP request bodies. ChatGPT-subscription Codex currently sends prompts in WebSocket frames, which this HTTP middleware cannot inspect. diff --git a/projects/privacy-guard/examples/openshell-manual/gateway.toml b/projects/privacy-guard/examples/email-scanner/gateway.toml similarity index 73% rename from projects/privacy-guard/examples/openshell-manual/gateway.toml rename to projects/privacy-guard/examples/email-scanner/gateway.toml index fb9189f..00ca9e7 100644 --- a/projects/privacy-guard/examples/openshell-manual/gateway.toml +++ b/projects/privacy-guard/examples/email-scanner/gateway.toml @@ -1,4 +1,4 @@ -# OpenShell gateway configuration for the manual Privacy Guard example. +# OpenShell gateway configuration for the email-scanner example. # Replace REPLACE_WITH_HOST_IP, then pass this file to openshell-gateway with # --config. Do not copy it into ~/.config/openshell. @@ -6,7 +6,7 @@ version = 1 [[openshell.supervisor.middleware]] -name = "privacy-guard-manual" +name = "privacy-guard-email-scanner" grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051" max_body_bytes = 4194304 timeout = "5s" diff --git a/projects/privacy-guard/examples/openshell-e2e/middleware_server.py b/projects/privacy-guard/examples/email-scanner/middleware_server.py similarity index 68% rename from projects/privacy-guard/examples/openshell-e2e/middleware_server.py rename to projects/privacy-guard/examples/email-scanner/middleware_server.py index 3750c3f..ec99bd0 100644 --- a/projects/privacy-guard/examples/openshell-e2e/middleware_server.py +++ b/projects/privacy-guard/examples/email-scanner/middleware_server.py @@ -1,22 +1,17 @@ #!/usr/bin/env python3 -"""Development-only deterministic Privacy Guard server for the manual E2E.""" +"""Run Privacy Guard with the example deterministic email scanner.""" from __future__ import annotations import argparse import re -from privacy_guard.scanners import ( - Confidence, - Finding, - Scanner, - ScannerConfig, -) +from privacy_guard.scanners import Confidence, Finding, Scanner, ScannerConfig from privacy_guard.service import MiddlewareServer -class ExampleEmailScanner(Scanner[ScannerConfig]): - """Detect example email addresses deterministically; not a production scanner.""" +class EmailScanner(Scanner[ScannerConfig]): + """Detect common email-shaped text for this example, not for production.""" _EMAIL = re.compile( r"(? None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--listen", default="127.0.0.1:50051") arguments = parser.parse_args() - scanner = ExampleEmailScanner( + scanner = EmailScanner( ScannerConfig(name="example_regex", entity_types=frozenset({"email"})) ) - server = MiddlewareServer(scanner=scanner) - server.serve(arguments.listen) + MiddlewareServer(scanner=scanner).serve(arguments.listen) if __name__ == "__main__": diff --git a/projects/privacy-guard/examples/openshell-manual/policy.yaml b/projects/privacy-guard/examples/email-scanner/policy.yaml similarity index 96% rename from projects/privacy-guard/examples/openshell-manual/policy.yaml rename to projects/privacy-guard/examples/email-scanner/policy.yaml index 8457472..c117f9e 100644 --- a/projects/privacy-guard/examples/openshell-manual/policy.yaml +++ b/projects/privacy-guard/examples/email-scanner/policy.yaml @@ -33,7 +33,7 @@ network_policies: network_middlewares: privacy_guard_redaction: name: Redact deterministic example emails - middleware: privacy-guard-manual + middleware: privacy-guard-email-scanner order: 0 config: body_format: json diff --git a/projects/privacy-guard/examples/openshell-e2e/README.md b/projects/privacy-guard/examples/openshell-e2e/README.md deleted file mode 100644 index 76596ee..0000000 --- a/projects/privacy-guard/examples/openshell-e2e/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Privacy Guard OpenShell E2E example - -This example proves the complete local path: - -```text -sandbox curl - -> OpenShell network policy and HTTP middleware - -> Privacy Guard gRPC service - -> reconstructed request - -> host capture endpoint -``` - -It uses no provider credentials or paid API. The sandbox sends a provider-shaped -JSON request to a local capture endpoint. Privacy Guard uses its explicit -development-only deterministic scanner to redact `user@example.com`, and verifies the -captured request body after OpenShell forwards it. - -## Prerequisites - -- macOS with the Homebrew OpenShell `0.0.86` gateway running -- Docker Desktop running with - pinned `ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:aeef1c63f00e2913ea002ccb3aaf925f338b5c5d70e63576f0d95c16a138044e` available locally -- `brew`, `docker`, `openshell`, `python3`, and `uv` -- ports `50051` and `18080` available on the host - -Run: - -```bash -./projects/privacy-guard/examples/openshell-e2e/run.sh -``` - -The pytest hook is skipped by default. From the repository root, run the same -state-mutating harness through the integration suite only with: - -```bash -PRIVACY_GUARD_RUN_OPEN_SHELL_E2E=1 \ - uv run --project projects/privacy-guard pytest -q \ - projects/privacy-guard/tests/examples/test_openshell_e2e.py -``` - -The harness temporarily appends a `privacy-guard-e2e` registration to -`~/.config/openshell/gateway.toml`, restarts the gateway, creates a disposable -sandbox, and restores the exact prior gateway configuration on exit. Existing -gateway configuration is preserved. - -If the default `en0` address is not reachable from Docker, provide the host IPv4 -address explicitly: - -```bash -PRIVACY_GUARD_E2E_HOST_ADDRESS=192.168.1.10 \ - ./projects/privacy-guard/examples/openshell-e2e/run.sh -``` - -The default reference is the locally verified immutable digest used during this -spike. Override the sandbox image when necessary with -`PRIVACY_GUARD_E2E_SANDBOX_IMAGE`. The image must already exist in the local -Docker daemon and provide `/usr/bin/curl`. - -## Interrupted-run recovery - -Normal failures and signals trigger automatic cleanup. If the process is killed -without running its trap, inspect these files before restarting OpenShell: - -- `~/.config/openshell/gateway.toml.privacy-guard-e2e.bak` means the original - configuration should replace `gateway.toml`. -- `~/.config/openshell/gateway.toml.privacy-guard-e2e.absent` means no user - configuration existed before the run; remove both the marker and the temporary - `gateway.toml`. - -Then run `brew services restart openshell` and verify `openshell status`. diff --git a/projects/privacy-guard/examples/openshell-e2e/capture_server.py b/projects/privacy-guard/examples/openshell-e2e/capture_server.py deleted file mode 100644 index 68a885e..0000000 --- a/projects/privacy-guard/examples/openshell-e2e/capture_server.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Capture one HTTP POST body for the Privacy Guard OpenShell E2E example.""" - -from __future__ import annotations - -import argparse -import os -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path - -MAX_CAPTURE_BYTES = 1024 * 1024 - - -class _CaptureHandler(BaseHTTPRequestHandler): - output_path: Path - - def do_POST(self) -> None: - """Persist one bounded request body and return a deterministic response.""" - content_length = int(self.headers.get("content-length", "0")) - if content_length < 0 or content_length > MAX_CAPTURE_BYTES: - self.send_error(413) - return - - request_body = self.rfile.read(content_length) - temporary_path = self.output_path.with_suffix(".tmp") - temporary_path.write_bytes(request_body) - os.replace(temporary_path, self.output_path) - - response_body = b'{"captured":true}' - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(response_body))) - self.end_headers() - self.wfile.write(response_body) - - # The example expects exactly one request. - self.server.shutdown() - - def log_message(self, format: str, *args: object) -> None: - """Suppress request content and default access-log output.""" - - -def main() -> int: - """Serve until one request is captured.""" - parser = argparse.ArgumentParser() - parser.add_argument("--host", default="0.0.0.0") - parser.add_argument("--port", type=int, default=18080) - parser.add_argument("--output", type=Path, required=True) - arguments = parser.parse_args() - - _CaptureHandler.output_path = arguments.output - server = ThreadingHTTPServer((arguments.host, arguments.port), _CaptureHandler) - try: - server.serve_forever() - finally: - server.server_close() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/projects/privacy-guard/examples/openshell-e2e/policy.yaml b/projects/privacy-guard/examples/openshell-e2e/policy.yaml deleted file mode 100644 index 732ee29..0000000 --- a/projects/privacy-guard/examples/openshell-e2e/policy.yaml +++ /dev/null @@ -1,38 +0,0 @@ -version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - privacy_guard_capture: - name: Privacy Guard E2E capture endpoint - endpoints: - - host: host.openshell.internal - port: 18080 - protocol: rest - enforcement: enforce - access: full - binaries: - - { path: /usr/bin/curl } - -network_middlewares: - privacy_guard_redaction: - name: Redact deterministic example email - middleware: privacy-guard-e2e - order: 0 - config: - body_format: json - on_finding: - action: redact - entity_types: [email] - minimum_confidence: high - on_error: fail_closed - endpoints: - include: [host.openshell.internal] diff --git a/projects/privacy-guard/examples/openshell-e2e/run.sh b/projects/privacy-guard/examples/openshell-e2e/run.sh deleted file mode 100755 index ab6d2f7..0000000 --- a/projects/privacy-guard/examples/openshell-e2e/run.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PYTHON_PROJECT="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -MIDDLEWARE_PORT=50051 -CAPTURE_PORT=18080 -GATEWAY_CONFIG="${HOME}/.config/openshell/gateway.toml" -GATEWAY_BACKUP="${GATEWAY_CONFIG}.privacy-guard-e2e.bak" -GATEWAY_ABSENT_MARKER="${GATEWAY_CONFIG}.privacy-guard-e2e.absent" -SANDBOX_NAME="privacy-guard-e2e-$$" -SANDBOX_IMAGE="${PRIVACY_GUARD_E2E_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:aeef1c63f00e2913ea002ccb3aaf925f338b5c5d70e63576f0d95c16a138044e}" -EXPECTED_TEXT="hello [email]" - -temporary_directory="" -middleware_pid="" -capture_pid="" -gateway_config_changed=false - -log() { - printf '[privacy-guard-e2e] %s\n' "$*" -} - -fail() { - printf '[privacy-guard-e2e] ERROR: %s\n' "$*" >&2 - exit 1 -} - -require_command() { - command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1" -} - -wait_for_tcp_port() { - local port="$1" - local attempt - for attempt in $(seq 1 60); do - if python3 -c "import socket; s=socket.create_connection(('127.0.0.1', ${port}), 0.2); s.close()" 2>/dev/null; then - return 0 - fi - sleep 0.25 - done - return 1 -} - -wait_for_gateway() { - local attempt - for attempt in $(seq 1 60); do - if openshell status >/dev/null 2>&1; then - return 0 - fi - sleep 1 - done - return 1 -} - -restore_gateway_config() { - if [[ "${gateway_config_changed}" != true ]]; then - return - fi - - if [[ -f "${GATEWAY_BACKUP}" ]]; then - mv "${GATEWAY_BACKUP}" "${GATEWAY_CONFIG}" - elif [[ -f "${GATEWAY_ABSENT_MARKER}" ]]; then - rm -f "${GATEWAY_CONFIG}" "${GATEWAY_ABSENT_MARKER}" - else - printf '[privacy-guard-e2e] ERROR: gateway backup state is missing; not restarting gateway\n' >&2 - return - fi - gateway_config_changed=false - brew services restart openshell >/dev/null - wait_for_gateway || printf '[privacy-guard-e2e] WARNING: restored gateway did not become ready\n' >&2 -} - -cleanup() { - set +e - openshell sandbox delete "${SANDBOX_NAME}" >/dev/null 2>&1 - restore_gateway_config - if [[ -n "${middleware_pid}" ]]; then - kill "${middleware_pid}" >/dev/null 2>&1 - wait "${middleware_pid}" >/dev/null 2>&1 - fi - if [[ -n "${capture_pid}" ]]; then - kill "${capture_pid}" >/dev/null 2>&1 - wait "${capture_pid}" >/dev/null 2>&1 - fi - if [[ -n "${temporary_directory}" ]]; then - rm -rf "${temporary_directory}" - fi -} - -trap cleanup EXIT INT TERM - -if [[ "$(uname -s)" != Darwin ]]; then - fail "this automated example currently supports the Homebrew macOS gateway" -fi - -for command_name in brew docker openshell python3 uv; do - require_command "${command_name}" -done - -if [[ -e "${GATEWAY_BACKUP}" || -e "${GATEWAY_ABSENT_MARKER}" ]]; then - fail "stale gateway recovery files exist; follow README.md recovery instructions" -fi - -host_address="${PRIVACY_GUARD_E2E_HOST_ADDRESS:-$(ipconfig getifaddr en0 || true)}" -[[ "${host_address}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || \ - fail "set PRIVACY_GUARD_E2E_HOST_ADDRESS to a host IPv4 address reachable from Docker" -docker image inspect "${SANDBOX_IMAGE}" >/dev/null 2>&1 || \ - fail "sandbox image is not local; pull ${SANDBOX_IMAGE} or set PRIVACY_GUARD_E2E_SANDBOX_IMAGE" - -temporary_directory="$(mktemp -d)" -capture_file="${temporary_directory}/captured-request.json" -middleware_log="${temporary_directory}/middleware.log" -capture_log="${temporary_directory}/capture.log" - -log "starting Privacy Guard and capture endpoint on ${host_address}" -uv run --project "${PYTHON_PROJECT}" python "${SCRIPT_DIR}/middleware_server.py" \ - --listen "0.0.0.0:${MIDDLEWARE_PORT}" >"${middleware_log}" 2>&1 & -middleware_pid=$! -python3 "${SCRIPT_DIR}/capture_server.py" --port "${CAPTURE_PORT}" \ - --output "${capture_file}" >"${capture_log}" 2>&1 & -capture_pid=$! - -wait_for_tcp_port "${MIDDLEWARE_PORT}" || { - tail -n 40 "${middleware_log}" >&2 - fail "Privacy Guard did not start" -} -wait_for_tcp_port "${CAPTURE_PORT}" || { - tail -n 40 "${capture_log}" >&2 - fail "capture server did not start" -} - -mkdir -p "$(dirname "${GATEWAY_CONFIG}")" -if [[ -f "${GATEWAY_CONFIG}" ]]; then - grep -q 'name = "privacy-guard-e2e"' "${GATEWAY_CONFIG}" && \ - fail "gateway already contains a privacy-guard-e2e registration" - cp -p "${GATEWAY_CONFIG}" "${GATEWAY_BACKUP}" -else - touch "${GATEWAY_ABSENT_MARKER}" - : >"${GATEWAY_CONFIG}" -fi -gateway_config_changed=true - -cat >>"${GATEWAY_CONFIG}" </dev/null -wait_for_gateway || { - tail -n 80 /opt/homebrew/var/log/openshell/openshell-gateway.err.log >&2 || true - fail "OpenShell gateway did not accept the middleware registration" -} - -request_body='{"messages":[{"role":"user","content":"hello user@example.com"}],"model":"capture-model"}' -log "creating disposable sandbox and sending provider-shaped request" -openshell sandbox create \ - --name "${SANDBOX_NAME}" \ - --from "${SANDBOX_IMAGE}" \ - --no-auto-providers \ - --policy "${SCRIPT_DIR}/policy.yaml" \ - --no-keep \ - --no-tty \ - -- \ - curl --fail-with-body --silent --show-error \ - --request POST \ - --header 'content-type: application/json' \ - --data "${request_body}" \ - "http://host.openshell.internal:${CAPTURE_PORT}/v1/chat/completions" - -for attempt in $(seq 1 40); do - [[ -f "${capture_file}" ]] && break - sleep 0.25 -done -[[ -f "${capture_file}" ]] || fail "capture endpoint did not receive the request" - -python3 - "${capture_file}" "${EXPECTED_TEXT}" <<'PY' -import json -import pathlib -import sys - -captured = json.loads(pathlib.Path(sys.argv[1]).read_bytes()) -actual = captured["messages"][0]["content"] -if actual != sys.argv[2]: - raise SystemExit(f"unexpected reconstructed content: {actual!r}") -if captured["model"] != "capture-model": - raise SystemExit("untouched provider-shaped field changed") -PY - -log "SUCCESS: OpenShell forwarded reconstructed content: ${EXPECTED_TEXT}" diff --git a/projects/privacy-guard/src/privacy_guard/processor.py b/projects/privacy-guard/src/privacy_guard/processor.py index a3b6310..8d75c5c 100644 --- a/projects/privacy-guard/src/privacy_guard/processor.py +++ b/projects/privacy-guard/src/privacy_guard/processor.py @@ -6,13 +6,6 @@ from dataclasses import dataclass from string import Formatter -from privacy_guard.body import ( - DEFAULT_FORMAT_HANDLERS, - FormatHandler, - FormatHandlerContractError, - RequestBody, - TextBlock, -) from privacy_guard.config import ( ActionConfig, BlockActionConfig, @@ -35,6 +28,13 @@ ProcessingDecision, ProcessingResult, ) +from privacy_guard.request_body import ( + DEFAULT_FORMAT_HANDLERS, + FormatHandler, + FormatHandlerContractError, + RequestBody, + TextBlock, +) from privacy_guard.scanners import ( Finding, RequestBodyFinding, diff --git a/projects/privacy-guard/src/privacy_guard/body/__init__.py b/projects/privacy-guard/src/privacy_guard/request_body/__init__.py similarity index 85% rename from projects/privacy-guard/src/privacy_guard/body/__init__.py rename to projects/privacy-guard/src/privacy_guard/request_body/__init__.py index 6611f95..c4b5486 100644 --- a/projects/privacy-guard/src/privacy_guard/body/__init__.py +++ b/projects/privacy-guard/src/privacy_guard/request_body/__init__.py @@ -5,15 +5,15 @@ from collections.abc import Mapping from types import MappingProxyType -from privacy_guard.body.base import ( +from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.request_body.base import ( FormatHandler, FormatHandlerContractError, RequestBody, TextBlock, parse_normalized_body, ) -from privacy_guard.body.json import JsonHandler -from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.request_body.json import JsonHandler DEFAULT_FORMAT_HANDLERS: Mapping[str, FormatHandler] = MappingProxyType( {handler.format_name: handler for handler in (JsonHandler(),)} @@ -21,7 +21,7 @@ """Built-in handlers keyed by name; processors may receive a custom mapping.""" -def select_handler(body_format: str) -> FormatHandler: +def select_format_handler(body_format: str) -> FormatHandler: """Return the registered FormatHandler for ``body_format`` (e.g. "json"). ``body_format`` comes from ``PolicyConfig.body_format``. Raise for an @@ -41,5 +41,5 @@ def select_handler(body_format: str) -> FormatHandler: "RequestBody", "TextBlock", "parse_normalized_body", - "select_handler", + "select_format_handler", ] diff --git a/projects/privacy-guard/src/privacy_guard/body/base.py b/projects/privacy-guard/src/privacy_guard/request_body/base.py similarity index 100% rename from projects/privacy-guard/src/privacy_guard/body/base.py rename to projects/privacy-guard/src/privacy_guard/request_body/base.py diff --git a/projects/privacy-guard/src/privacy_guard/body/json.py b/projects/privacy-guard/src/privacy_guard/request_body/json.py similarity index 99% rename from projects/privacy-guard/src/privacy_guard/body/json.py rename to projects/privacy-guard/src/privacy_guard/request_body/json.py index 1399c88..3906f9b 100644 --- a/projects/privacy-guard/src/privacy_guard/body/json.py +++ b/projects/privacy-guard/src/privacy_guard/request_body/json.py @@ -12,13 +12,13 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError from typing_extensions import TypeAliasType, override -from privacy_guard.body.base import FormatHandler, RequestBody, TextBlock from privacy_guard.constants import ( MAX_JSON_NESTING, MAX_SCANNED_CHARACTERS, MAX_TEXT_BLOCKS, ) from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.request_body.base import FormatHandler, RequestBody, TextBlock from privacy_guard.validation import ScalarString, parse_scalar_string if TYPE_CHECKING: diff --git a/projects/privacy-guard/tests/examples/test_email_scanner.py b/projects/privacy-guard/tests/examples/test_email_scanner.py new file mode 100644 index 0000000..c5694a4 --- /dev/null +++ b/projects/privacy-guard/tests/examples/test_email_scanner.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +EXAMPLE_DIRECTORY = Path(__file__).parents[2] / "examples" / "email-scanner" + + +def test_example_scanner_detects_email_and_server_entry_point_imports() -> None: + probe = """ +from middleware_server import EmailScanner +from privacy_guard.scanners import ScannerConfig + +scanner = EmailScanner( + ScannerConfig(name="example_regex", entity_types=frozenset({"email"})) +) +findings = scanner.scan("safe user@example.com text") +assert len(findings) == 1 +assert findings[0].entity == "email" +assert findings[0].start_offset == 5 +assert findings[0].end_offset == 21 +""" + subprocess.run([sys.executable, "-c", probe], cwd=EXAMPLE_DIRECTORY, check=True) + + +def test_example_configuration_targets_its_local_middleware() -> None: + policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() + gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() + readme = (EXAMPLE_DIRECTORY / "README.md").read_text() + + assert "middleware: privacy-guard-email-scanner" in policy + assert 'name = "privacy-guard-email-scanner"' in gateway + assert "action: redact" in policy + assert "entity_types: [email]" in policy + assert "python middleware_server.py" in readme diff --git a/projects/privacy-guard/tests/examples/test_openshell_e2e.py b/projects/privacy-guard/tests/examples/test_openshell_e2e.py deleted file mode 100644 index 16991e9..0000000 --- a/projects/privacy-guard/tests/examples/test_openshell_e2e.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -import os -import socket -import subprocess -import sys -import time -import urllib.request -from pathlib import Path - -import pytest - -EXAMPLE_DIRECTORY = Path(__file__).parents[2] / "examples" / "openshell-e2e" - - -def test_capture_server_persists_exact_post_body(tmp_path: Path) -> None: - output_path = tmp_path / "captured.json" - with socket.socket() as reservation: - reservation.bind(("127.0.0.1", 0)) - port = reservation.getsockname()[1] - process = subprocess.Popen( - [ - sys.executable, - str(EXAMPLE_DIRECTORY / "capture_server.py"), - "--host", - "127.0.0.1", - "--port", - str(port), - "--output", - str(output_path), - ] - ) - request_body = b'{"message":"exact bytes"}' - - try: - for _ in range(40): - try: - request = urllib.request.Request( - f"http://127.0.0.1:{port}/capture", - data=request_body, - method="POST", - ) - with urllib.request.urlopen(request, timeout=1) as response: - assert response.read() == b'{"captured":true}' - break - except OSError: - time.sleep(0.05) - else: - raise AssertionError("capture server did not become ready") - - assert output_path.read_bytes() == request_body - assert process.wait(timeout=2) == 0 - finally: - process.terminate() - process.wait(timeout=2) - - -def test_harness_shell_is_valid_and_policy_targets_privacy_guard() -> None: - subprocess.run( - ["bash", "-n", str(EXAMPLE_DIRECTORY / "run.sh")], - check=True, - ) - policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() - - assert "middleware: privacy-guard-e2e" in policy - assert "on_finding:" in policy - assert "action: redact" in policy - assert "entity_types: [email]" in policy - harness = (EXAMPLE_DIRECTORY / "run.sh").read_text() - assert 'python "${SCRIPT_DIR}/middleware_server.py"' in harness - assert "base@sha256:" in harness - assert "base:latest" not in harness - - -@pytest.mark.skipif( - os.environ.get("PRIVACY_GUARD_RUN_OPEN_SHELL_E2E") != "1", - reason="state-mutating OpenShell harness is explicitly opt-in", -) -def test_opt_in_full_openshell_harness() -> None: - subprocess.run([str(EXAMPLE_DIRECTORY / "run.sh")], check=True) diff --git a/projects/privacy-guard/tests/body/test_base.py b/projects/privacy-guard/tests/request_body/test_base.py similarity index 98% rename from projects/privacy-guard/tests/body/test_base.py rename to projects/privacy-guard/tests/request_body/test_base.py index 337c81e..c1697c5 100644 --- a/projects/privacy-guard/tests/body/test_base.py +++ b/projects/privacy-guard/tests/request_body/test_base.py @@ -5,14 +5,14 @@ from pydantic import ValidationError from typing_extensions import override -from privacy_guard.body import ( +from privacy_guard.config import PolicyConfig +from privacy_guard.request_body import ( FormatHandler, FormatHandlerContractError, RequestBody, TextBlock, parse_normalized_body, ) -from privacy_guard.config import PolicyConfig class OpaqueHandler(FormatHandler): diff --git a/projects/privacy-guard/tests/body/test_json.py b/projects/privacy-guard/tests/request_body/test_json.py similarity index 97% rename from projects/privacy-guard/tests/body/test_json.py rename to projects/privacy-guard/tests/request_body/test_json.py index 12fd3a0..493c3d9 100644 --- a/projects/privacy-guard/tests/body/test_json.py +++ b/projects/privacy-guard/tests/request_body/test_json.py @@ -6,10 +6,15 @@ import pytest from pydantic import ValidationError -import privacy_guard.body.json as json_module -from privacy_guard.body import JsonHandler, RequestBody, TextBlock, select_handler +import privacy_guard.request_body.json as json_module from privacy_guard.config import PolicyConfig from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.request_body import ( + JsonHandler, + RequestBody, + TextBlock, + select_format_handler, +) def _assert_safe_error( @@ -244,7 +249,7 @@ def test_normalize_rejects_invalid_json() -> None: def test_normalize_shape_failure_drops_sensitive_exception_context( monkeypatch: pytest.MonkeyPatch, ) -> None: - import privacy_guard.body.json as json_module + import privacy_guard.request_body.json as json_module sensitive_text = "distinctive-sensitive-shape-8472" monkeypatch.setattr(json_module, "MAX_SCANNED_CHARACTERS", len(sensitive_text) - 1) @@ -557,18 +562,18 @@ def test_reconstruct_does_not_mutate_original_parsed_value() -> None: assert _json_state_value(request_body) == original_value -def test_select_handler_returns_registered_json_singleton() -> None: - json_handler = select_handler("json") +def test_select_format_handler_returns_registered_json_singleton() -> None: + json_handler = select_format_handler("json") assert isinstance(json_handler, JsonHandler) - assert select_handler("json") is json_handler + assert select_format_handler("json") is json_handler -def test_select_handler_rejects_unknown_kind_without_fallback() -> None: +def test_select_format_handler_rejects_unknown_kind_without_fallback() -> None: sentinel = "unknown-sensitive-format-8472" with pytest.raises(PrivacyGuardError) as exception_info: - select_handler(sentinel) + select_format_handler(sentinel) assert exception_info.value.code is ErrorCode.BODY_FORMAT_UNSUPPORTED assert sentinel not in str(exception_info.value) diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py index 676e34f..ffd21df 100644 --- a/projects/privacy-guard/tests/test_config.py +++ b/projects/privacy-guard/tests/test_config.py @@ -1,7 +1,6 @@ import pytest from pydantic import ValidationError -from privacy_guard.body import select_handler from privacy_guard.config import ( ActionConfig, BlockActionConfig, @@ -13,6 +12,7 @@ ) from privacy_guard.constants import MAX_SCANNER_METADATA_BYTES from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.request_body import select_format_handler from privacy_guard.scanners import Confidence from privacy_guard.validation import ( BoundedMetadataString, @@ -358,11 +358,11 @@ def test_validation_failure_does_not_leak_input_or_pydantic_error() -> None: assert error.__cause__ is None -def test_select_handler_error_does_not_leak_unknown_format() -> None: +def test_select_format_handler_error_does_not_leak_unknown_format() -> None: sentinel = "sensitive-unknown-format-8472" with pytest.raises(PrivacyGuardError) as exception_info: - select_handler(sentinel) + select_format_handler(sentinel) assert exception_info.value.code is ErrorCode.BODY_FORMAT_UNSUPPORTED assert sentinel not in str(exception_info.value) diff --git a/projects/privacy-guard/tests/test_hardening.py b/projects/privacy-guard/tests/test_hardening.py index 5875770..1f0bdbc 100644 --- a/projects/privacy-guard/tests/test_hardening.py +++ b/projects/privacy-guard/tests/test_hardening.py @@ -13,7 +13,6 @@ import privacy_guard.service.servicer as servicer_module from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 -from privacy_guard.body import JsonHandler from privacy_guard.config import PolicyConfig from privacy_guard.constants import MAX_BODY_BYTES from privacy_guard.errors import ErrorCode, PrivacyGuardError @@ -23,6 +22,7 @@ ProcessingResult, ) from privacy_guard.processor import RequestProcessor +from privacy_guard.request_body import JsonHandler from privacy_guard.scanners import ( Confidence, Finding, @@ -92,7 +92,7 @@ def test_json_nesting_beyond_adapter_recursion_limit_is_shape_error(depth: int) def test_json_walker_bounds_allocation_at_exact_aggregate_boundaries( monkeypatch: pytest.MonkeyPatch, ) -> None: - import privacy_guard.body.json as json_module + import privacy_guard.request_body.json as json_module monkeypatch.setattr(json_module, "MAX_TEXT_BLOCKS", 2) monkeypatch.setattr(json_module, "MAX_SCANNED_CHARACTERS", 2) @@ -111,7 +111,7 @@ def test_json_walker_bounds_allocation_at_exact_aggregate_boundaries( def test_json_walker_keeps_wide_container_traversal_allocation_bounded() -> None: - import privacy_guard.body.json as json_module + import privacy_guard.request_body.json as json_module raw_body = b"[" + (b"0," * 99_999) + b"0]" handler = JsonHandler() diff --git a/projects/privacy-guard/tests/test_processor.py b/projects/privacy-guard/tests/test_processor.py index c6879c8..f02cc0d 100644 --- a/projects/privacy-guard/tests/test_processor.py +++ b/projects/privacy-guard/tests/test_processor.py @@ -4,7 +4,6 @@ from pydantic import ValidationError from typing_extensions import override -from privacy_guard.body import FormatHandler, RequestBody, TextBlock from privacy_guard.config import PolicyAction, PolicyConfig from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.payloads import ( @@ -13,6 +12,7 @@ ProcessingResult, ) from privacy_guard.processor import RequestProcessor +from privacy_guard.request_body import FormatHandler, RequestBody, TextBlock from privacy_guard.scanners import ( Finding, PassthroughScanner, diff --git a/projects/privacy-guard/tests/test_single_block_composition.py b/projects/privacy-guard/tests/test_single_block_composition.py index c4e743c..fdb5063 100644 --- a/projects/privacy-guard/tests/test_single_block_composition.py +++ b/projects/privacy-guard/tests/test_single_block_composition.py @@ -2,8 +2,8 @@ import pytest -from privacy_guard.body import JsonHandler from privacy_guard.config import PolicyConfig +from privacy_guard.request_body import JsonHandler from privacy_guard.scanners import PassthroughScanner From cef0e9b0cbbd9bc1a6d8622da53d25f932318198 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Wed, 22 Jul 2026 22:18:02 +0000 Subject: [PATCH 03/82] Add configurable Privacy Guard regex scanner --- projects/privacy-guard/BENCHMARKS.md | 23 +- projects/privacy-guard/README.md | 49 +- projects/privacy-guard/REGEX_SCANNER_PLAN.md | 280 ++++++++++++ .../examples/email-scanner/README.md | 17 +- .../email-scanner/middleware_server.py | 12 +- .../examples/regex-configs/README.md | 11 + .../examples/regex-configs/customer.yaml | 10 + .../examples/regex-configs/hipaa.yaml | 8 + .../examples/regex-configs/profiles.yaml | 19 + projects/privacy-guard/pyproject.toml | 1 + .../scripts/benchmark_privacy_guard.py | 4 +- .../src/privacy_guard/constants.py | 19 + .../privacy-guard/src/privacy_guard/errors.py | 18 + .../src/privacy_guard/processor.py | 73 ++- .../src/privacy_guard/scanners/__init__.py | 16 +- .../src/privacy_guard/scanners/base.py | 81 +++- .../src/privacy_guard/scanners/passthrough.py | 23 - .../src/privacy_guard/scanners/regex.py | 430 ++++++++++++++++++ .../src/privacy_guard/service/server.py | 30 +- .../src/privacy_guard/service/servicer.py | 30 +- .../privacy-guard/tests/scanner_helpers.py | 3 +- .../tests/scanners/test_regex.py | 230 ++++++++++ ..._passthrough.py => test_scanner_models.py} | 78 ++-- .../tests/service/test_server.py | 50 +- .../tests/service/test_servicer.py | 60 ++- .../privacy-guard/tests/test_hardening.py | 28 +- .../privacy-guard/tests/test_processor.py | 89 +++- .../tests/test_single_block_composition.py | 6 +- projects/privacy-guard/uv.lock | 57 +++ 29 files changed, 1546 insertions(+), 209 deletions(-) create mode 100644 projects/privacy-guard/REGEX_SCANNER_PLAN.md create mode 100644 projects/privacy-guard/examples/regex-configs/README.md create mode 100644 projects/privacy-guard/examples/regex-configs/customer.yaml create mode 100644 projects/privacy-guard/examples/regex-configs/hipaa.yaml create mode 100644 projects/privacy-guard/examples/regex-configs/profiles.yaml delete mode 100644 projects/privacy-guard/src/privacy_guard/scanners/passthrough.py create mode 100644 projects/privacy-guard/src/privacy_guard/scanners/regex.py create mode 100644 projects/privacy-guard/tests/scanners/test_regex.py rename projects/privacy-guard/tests/scanners/{test_passthrough.py => test_scanner_models.py} (68%) diff --git a/projects/privacy-guard/BENCHMARKS.md b/projects/privacy-guard/BENCHMARKS.md index 651c8db..85b2946 100644 --- a/projects/privacy-guard/BENCHMARKS.md +++ b/projects/privacy-guard/BENCHMARKS.md @@ -70,9 +70,20 @@ tree; the raw parse tree becomes unreachable when the parse helper returns, before text-block materialization begins. The harness deliberately isolates processor overhead with a synthetic scanner. -It does not represent the latency of a future production PII scanner and does -not include protobuf conversion, gRPC transport, executor or semaphore queuing, -concurrent throughput, or whole-process memory. Add realistic request -distributions and service-level concurrency measurements when a production -scanner is introduced; establish regression thresholds only on pinned -infrastructure. +It does not include protobuf conversion, gRPC transport, executor or semaphore +queuing, concurrent throughput, or whole-process memory. + +## Regex catalog scalability gate + +A separate one-shot gate measured the required 1,000 active entities and 10,000 +active patterns. The generated 749,790-byte single-profile YAML gave every +entity ten unique literal patterns. The representative sparse input was 46 +characters and produced zero matches. This deliberately measures catalog +evaluation rather than dense-result aggregation, which has independent limits. + +Reference environment: CPython 3.14.4 on Linux x86_64 in the Codex workspace. +Recorded 2026-07-22 using the stdlib `re` engine. Parsing, strict validation, +and compiling took 14.399 seconds with 132.1 MiB peak traced allocation. +Scanning took 9 ms, within the one-second default request budget. These are +reproducibility snapshots, not release thresholds; the shared deadline remains +the cooperative runtime CPU budget. diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 9d64dc2..224adf9 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -7,10 +7,10 @@ body, scans its text for sensitive values, and returns an allow/deny decision plus an optionally rewritten body that the supervisor forwards instead of the original. -> Status: **hardened research project with a passthrough production default.** Strict policy parsing, -> request-level processing, the generic JSON handler, safe gRPC adaptation, and -> a loopback server are implemented. The production scanner is still the -> passthrough scanner; no real PII scanner exists yet. +> Status: **hardened research project with a configurable regex scanner.** Strict +> policy and scanner configuration, request-level processing, bounded scanning, +> the generic JSON handler, safe gRPC adaptation, and a loopback server are +> implemented. The self-contained [email scanner example](examples/email-scanner/README.md) provides a deterministic scanner, middleware entry point, gateway registration, @@ -87,20 +87,36 @@ winners deterministically by confidence, span length, offsets, scanner identity, and entity. A scanner sequence is passed to `RequestProcessor`; scanner names must be unique and remain visible in aggregated findings. -The default service uses `PassthroughScanner`. The deterministic email regex and -its server entry point live entirely under `examples/email-scanner`; production -package modules neither export nor select it. +`RegexScanner` is the packaged command's default implementation. Its entity +catalog remains explicit: the command refuses to start without +`--scanner-config`. Single-profile files contain a non-empty entity list; +multi-profile files contain only a non-empty `profiles` mapping and require +`--profile`. See [examples/regex-configs](examples/regex-configs) for both forms. + +```bash +uv run privacy-guard \ + --scanner-config examples/regex-configs/customer.yaml \ + --listen 127.0.0.1:50051 +``` + +Each entity has a unique name and a non-empty `patterns` list. Patterns declare +`name`, `regex`, `confidence`, and optional `ignore_case`, `multiline`, +`dot_all`, and `ascii` booleans. Every match carries its configured pattern name +in general finding metadata and is reported as `entity/pattern-name` by the +service. Policy filtering remains at entity level. A scanner is a nominal extension: declare its strict configuration type and -implement only `_scan`. The base constructor validates and retains the config, -and the public `scan` wrapper validates the returned tuple and each `Finding`. +implement `_scan`. Scanners that need derived, reusable state may also override +`_initialize`; the base constructor calls it after validating and retaining the +configuration. The public `scan` wrapper validates the returned tuple and each +`Finding`. ```python -from privacy_guard.scanners import Finding, Scanner, ScannerConfig +from privacy_guard.scanners import Finding, ScanBudget, Scanner, ScannerConfig class ExampleScanner(Scanner[ScannerConfig]): - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: return () ``` @@ -134,6 +150,9 @@ manifest or add the lower limit to service configuration. JSON parsing is additionally bounded to 64 nesting levels, 4,096 text blocks, and 4 MiB of scanned characters. Scanning is capped at 256 findings per block, 4,096 per request, four active scanner workers, and 16 concurrent gRPC calls. +One default one-second monotonic scan budget is shared across every block and +scanner in a request. `RegexScanner` checks it before and after each expression +evaluation and while consuming overlapping matches. Shape excess is invalid input. Finding or outbound representation excess returns a stable `privacy_guard_limit_exceeded` deny with no body or partial findings, avoiding a failure-mode-dependent fail open. @@ -169,6 +188,14 @@ release its scanner slot until its synchronous worker really exits. - **Finding types.** A scanner returns strict block-relative `scanners.Finding` values. The processor creates `RequestBodyFinding` values with the owning text-block path, and the servicer aggregates those into the protocol's count-based `Finding`. + A finding may include an immutable, bounded string metadata mapping for + scanner-specific attribution. Regex findings use its `pattern_name` key. +- **Scanner initialization.** Override `_initialize` only when validated config + must be compiled or transformed into reusable immutable scanner state. The + default hook does nothing. +- **Scan budgets.** The protected scanner hook receives a request-scoped + `ScanBudget`. Standalone `scan` calls create a safe default when no budget is + supplied. Potentially unbounded scanners must cooperate with the deadline. - **Text-block paths.** `Finding` has no path state. A `Scanner` sees only one text block; the processor attaches `TextBlock.path` by creating `RequestBodyFinding`. - **Format selection.** `PolicyConfig.body_format` (default `json`) picks a handler diff --git a/projects/privacy-guard/REGEX_SCANNER_PLAN.md b/projects/privacy-guard/REGEX_SCANNER_PLAN.md new file mode 100644 index 0000000..186988a --- /dev/null +++ b/projects/privacy-guard/REGEX_SCANNER_PLAN.md @@ -0,0 +1,280 @@ +# Regex scanner implementation plan + +## Outcome + +`RegexScanner` will become Privacy Guard's default scanner implementation. It +will compile a strict YAML configuration at startup and emit findings for every +configured entity and pattern match. Operators will be able to maintain focused +configuration files for different environments, such as customer-specific data +or HIPAA-regulated identifiers, without changing Python code. + +The scanner type will be the default; the rule set will remain explicit. The +service must never silently start with an empty or missing rule set. + +“Supports” a catalog size means that the complete configuration can be parsed, +validated, compiled, and used to scan representative requests within the +default request budget. Merely accepting a large file that predictably times +out does not satisfy the scalability goal. Catalog capacity is independent of +match cardinality: a profile may define thousands of entities even though one +unusually dense request can still hit the existing per-block, per-request, or +protocol-result limits and be denied safely. + +## Proposed YAML contract + +A single-profile configuration is its complete entity catalog. Every listed +entity is active, so an additional top-level `entities` wrapper would add no +information: + +```yaml +- name: email + patterns: + - name: common-email + regex: '(?)`. The trailing empty named marker + makes the pattern identity observable without inserting a capture before the + user's expression, so unnamed group numbering and numeric backreferences are + preserved. Retain the original pattern name alongside the compiled rule. + Compile patterns separately and use overlapping iteration so matches can + overlap both within one pattern and across different patterns, while + normalized group names remain rule-local. Read the automatic marker to + identify the matched rule, then report the original, non-normalized pattern + name from immutable rule metadata. Scanner calls will remain stateless and + thread-safe. +3. Add an immutable request-scoped `ScanBudget` containing a monotonic deadline + and export it from `privacy_guard.scanners` for advanced scanner authors. + Begin with a conservative provisional package default, then document and + finalize it using the benchmark gate below. `RequestProcessor` will accept an + optional finite, positive, upper-bounded `scan_timeout_seconds` construction + argument with that safe default; callers do not need to configure it. Reject + booleans as well as non-finite, non-positive, and over-maximum numeric values. + `RequestProcessor.process` will create one budget immediately before scanning + and pass it through every `Scanner.scan` call for every block and scanner. + `Scanner.scan` will accept an optional keyword-only budget and create a fresh + default budget for standalone calls when none is supplied. The protected + scanner hook receives the effective non-optional budget; simple scanners may + ignore it, while scanners performing potentially unbounded work must + cooperate with it. The budget is therefore an automatic hard bound for + `RegexScanner`, but only a cooperative contract for custom scanners. Before + each pattern evaluation, + `RegexScanner` will pass the smaller of the remaining request budget and the + per-pattern ceiling to a timeout-capable regex engine. Check the budget while + consuming the engine's iterator, not only when constructing it. Budget + exhaustion or an engine timeout will raise one exported, content-safe + `ScanBudgetExceeded` exception from `Scanner.scan`, with no partial findings. + `RequestProcessor` will catch only that typed condition, discard findings + accumulated from earlier scanners or blocks, and produce the existing stable + limit deny. Standalone scanner callers receive the typed exception instead of + a service-specific decision. This deliberately evolves the protected scanner + extension signature; all built-in examples, tests, and extension + documentation must migrate in the same change. +4. Define independent limits for configuration bytes, profiles, entities, + patterns, pattern bytes, matches per pattern, and request evaluation time. + Configuration capacity must not be derived from response aggregation limits: + profiles should support thousands of entities and patterns. Establish the + final hard ceilings with startup-time, startup-memory, and scan-time + benchmarks. The scalability gate is 1,000 active entities and 10,000 active + patterns on documented representative inputs and reference hardware. Record + input size and match density with each result so the claim is reproducible. + The request-scoped budget remains the runtime CPU bound regardless of + catalog size. If separately evaluating 10,000 compiled patterns cannot meet + the default budget, introduce bounded batching, indexing, or a multi-pattern + engine before claiming that scale; preserve overlapping results and original + pattern identity in any optimization. If batching combines patterns, assign + collision-free internal group identifiers even when configured names + normalize to the same value. Implement conservative provisional file and + count ceilings first, then finalize and document them only after this gate + passes. +5. Add optional bounded string metadata to the domain `Finding` so scanners can + attach scanner-specific attribution without adding fields to the shared model. + `RegexScanner` stores its configured pattern name under `pattern_name`. + Include that metadata value, or `""` when absent, after entity in + scanner-result, block-result, overlap-winner, and final-winner ordering keys + so otherwise identical findings remain deterministic without changing higher-priority + span or confidence semantics. +6. Include the pattern in audit-safe service reporting. Findings without a + pattern retain the existing `type=scanner_name` and `label=entity` shape. + Findings with a pattern retain `type=scanner_name` and use the unambiguous + `label=entity/pattern_name` shape. Group by scanner, entity, optional pattern, + and confidence; validate the combined UTF-8 label and encoded protobuf size + before returning it. Keep the current limit of 32 aggregated findings per + middleware stage: the checked-in OpenShell protocol explicitly documents + that receiver limit. It limits distinct groups matched in one response, not + entities or patterns configured in a profile. The existing 4,096 domain + findings per request and 4 KiB per-group encoded limits also remain. If more + than 32 distinct groups must be reported in one stage, coordinate that as a + separate OpenShell protocol change before increasing Privacy Guard's limit; + the scanner implementation must not emit a response its consumer rejects. +7. Add a loader such as `RegexScanner.from_yaml(path, profile=None)`. Update the + packaged command to accept one required `--scanner-config PATH`, a + conditionally required `--profile NAME`, and an optional `--scanner-name`. + Load and compile the configuration before binding a listening socket. Restore + the `privacy-guard` script only in the same change, so the command cannot run + with an implicit allow-all scanner between implementations. +8. Add example rule sets under `examples/regex-configs/`: single-profile + `customer.yaml` and `hipaa.yaml` files plus a `profiles.yaml` file showing + both catalogs in the multi-profile shape. The HIPAA example must be + described as a starting rule set, not a claim of compliance; operational + controls and validation remain the deployer's responsibility. Migrate the + self-contained `examples/email-scanner/` example from its handwritten scanner + to `RegexScanner` and a colocated YAML file, so the primary manual example + exercises the new default without depending on another example directory. + +## Verification + +- Unit-test both YAML shapes, profile selection, missing and unexpected + selections, ordering, limits, duplicate keys, aliases, unsafe tags, excessive + nesting and scalar sizes, unsupported flags, malformed regexes, and + content-safe failures. Verify that every profile is validated even when it is + not selected. +- Test Unicode offsets, multiple entities and patterns, overlaps, confidence, + case handling, automatic hyphen normalization, normalized-name coexistence, + trailing named markers, numeric backreferences, reserved named-group and + inline-flag rejection, load-time and contextual zero-width rejection, + deterministic ordering, and finding limits. +- Test per-pattern and request-wide timeout behavior with adversarial + expressions, many patterns, many text blocks, and long inputs. Verify timeout + paths raise `ScanBudgetExceeded` for standalone calls and return a stable deny + through `RequestProcessor`, with no partial findings in either case. +- Test that `RequestProcessor` and standalone `Scanner.scan` calls use safe + default budgets when callers provide no timeout or budget, and that an + explicit processor timeout applies one shared deadline across the request. +- Test concurrent calls against one scanner instance. +- Exercise observe, redact, and block through `RequestProcessor` and the gRPC + boundary with both customer and HIPAA example configurations. Verify that + domain and aggregated service findings report the configured pattern name, + legacy findings without a pattern retain their current representation, and + profiles containing more than 32 entities load and scan sparse inputs + normally. Verify the exact 32-group protocol boundary, the 4,096 + domain-finding boundary, and the per-group encoded-size boundary; the + thirty-third distinct response group must produce the stable limit deny with + no partial findings. +- Extend the diagnostic benchmark with representative rule counts and matching + densities, including 100, 1,000, and 10,000 active patterns. Record parse and + compilation time, retained memory, scan latency, timeout rate, input size, + and match density before establishing the final configuration ceilings, + default request budget, or performance thresholds. + +## Delivery sequence + +Each delivery step includes its focused unit tests; the final step adds +cross-component and adversarial coverage rather than deferring all testing. + +1. Configuration models, bounded YAML loader, dependencies, and conservative + provisional safety limits. +2. Request-scoped scan budget, regex compilation, and scanning engine. +3. Scalability benchmark, engine optimization if required, and final limits and + default budget. +4. Backward-compatible finding metadata and pattern-aware service reporting. +5. Service CLI wiring and restoration of the package script. +6. Customer and HIPAA example configurations and user documentation. +7. Security-focused tests and full integration tests. diff --git a/projects/privacy-guard/examples/email-scanner/README.md b/projects/privacy-guard/examples/email-scanner/README.md index bbe56f3..92deb9b 100644 --- a/projects/privacy-guard/examples/email-scanner/README.md +++ b/projects/privacy-guard/examples/email-scanner/README.md @@ -1,16 +1,17 @@ # Email scanner example -This self-contained example supplies a deterministic email scanner, its Privacy -Guard server entry point, and the OpenShell gateway and sandbox policy needed to -try redaction with Claude Code. It temporarily runs the installed OpenShell -gateway with the config from this directory. It does not create or modify +This self-contained example supplies a deterministic custom email scanner, its +Privacy Guard server entry point, and the OpenShell gateway and sandbox policy +needed to try redaction with Claude Code. It temporarily runs the installed +OpenShell gateway with the config from this directory. It does not create or modify `~/.config/openshell/gateway.toml`, and it does not create a project-local state directory. -The illustrative regex scanner in `middleware_server.py` detects email-shaped -text in Claude Code request bodies. The policy replaces matches with `[email]` -before Anthropic receives the request. The scanner is intentionally small and -deterministic; it is not intended for production PII detection. +The `EmailScanner` class in `middleware_server.py` demonstrates how to implement +the scanner extension contract directly. It detects email-shaped text in Claude +Code request bodies. The policy replaces matches with `[email]` before Anthropic +receives the request. The implementation is intentionally small and is not +intended as comprehensive production PII detection. ## Prerequisites diff --git a/projects/privacy-guard/examples/email-scanner/middleware_server.py b/projects/privacy-guard/examples/email-scanner/middleware_server.py index ec99bd0..b964733 100644 --- a/projects/privacy-guard/examples/email-scanner/middleware_server.py +++ b/projects/privacy-guard/examples/email-scanner/middleware_server.py @@ -6,18 +6,24 @@ import argparse import re -from privacy_guard.scanners import Confidence, Finding, Scanner, ScannerConfig +from privacy_guard.scanners import ( + Confidence, + Finding, + ScanBudget, + Scanner, + ScannerConfig, +) from privacy_guard.service import MiddlewareServer class EmailScanner(Scanner[ScannerConfig]): - """Detect common email-shaped text for this example, not for production.""" + """Detect common email-shaped text as a minimal custom scanner example.""" _EMAIL = re.compile( r"(? tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: return tuple( Finding( entity="email", diff --git a/projects/privacy-guard/examples/regex-configs/README.md b/projects/privacy-guard/examples/regex-configs/README.md new file mode 100644 index 0000000..03deabf --- /dev/null +++ b/projects/privacy-guard/examples/regex-configs/README.md @@ -0,0 +1,11 @@ +# Regex scanner configurations + +`customer.yaml` and `hipaa.yaml` demonstrate the single-profile entity-list +shape. `profiles.yaml` combines both catalogs under explicit profile names; run +Privacy Guard with `--profile customer-support` or `--profile hipaa` when using +that file. + +The HIPAA catalog is only a starting rule set. It is not a claim of compliance. +Deployers remain responsible for validating detection behavior and implementing +the operational, administrative, and technical controls their environment +requires. diff --git a/projects/privacy-guard/examples/regex-configs/customer.yaml b/projects/privacy-guard/examples/regex-configs/customer.yaml new file mode 100644 index 0000000..3508588 --- /dev/null +++ b/projects/privacy-guard/examples/regex-configs/customer.yaml @@ -0,0 +1,10 @@ +- name: email + patterns: + - name: common-email + regex: '(?=1.81.1,<2", "protobuf>=6.33.5,<7", "pydantic>=2.11,<3", + "pyyaml>=6.0.2,<7", "typing-extensions>=4.12,<5", ] diff --git a/projects/privacy-guard/scripts/benchmark_privacy_guard.py b/projects/privacy-guard/scripts/benchmark_privacy_guard.py index b9772ef..d922553 100755 --- a/projects/privacy-guard/scripts/benchmark_privacy_guard.py +++ b/projects/privacy-guard/scripts/benchmark_privacy_guard.py @@ -28,7 +28,7 @@ ) from privacy_guard.payloads import InterceptedRequest, ProcessingDecision from privacy_guard.processor import RequestProcessor -from privacy_guard.scanners import Finding, Scanner, ScannerConfig +from privacy_guard.scanners import Finding, ScanBudget, Scanner, ScannerConfig _KIB = 1024 _MIB = 1024 * 1024 @@ -44,7 +44,7 @@ class _BenchmarkScannerConfig(ScannerConfig): class _BenchmarkScanner(Scanner[_BenchmarkScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: if not text_block.startswith( _MARKER_CHARACTER * self.config.findings_per_block ): diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index 1bccc98..8dee5ca 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -16,6 +16,7 @@ BLOCK_REASON_CODE = "privacy_guard_blocked" LIMIT_REASON = "Privacy Guard denied a result that exceeded a safety limit" LIMIT_REASON_CODE = "privacy_guard_limit_exceeded" +PATTERN_NAME_METADATA_KEY = "pattern_name" # Request-body and JSON shape limits. MAX_BODY_BYTES = 4 * 1024 * 1024 @@ -27,9 +28,27 @@ MAX_FINDINGS_PER_BLOCK = 256 MAX_FINDINGS_PER_REQUEST = 4096 MAX_SCANNER_METADATA_BYTES = 1024 +MAX_FINDING_METADATA_ENTRIES = 32 MAX_PROTO_FINDING_GROUPS = 32 MAX_PROTO_FINDING_BYTES = 4 * 1024 +# Regex scanner configuration and execution limits. Catalog limits are independent +# of request result limits: sparse requests can use large, focused rule sets. +MAX_SCANNER_CONFIG_BYTES = 16 * 1024 * 1024 +MAX_SCANNER_CONFIG_NESTING = 16 +MAX_SCANNER_CONFIG_NODES = 250_000 +MAX_SCANNER_CONFIG_SCALAR_BYTES = 16 * 1024 +MAX_REGEX_NAME_BYTES = 128 +MAX_REGEX_PROFILES = 32 +MAX_REGEX_ENTITIES_PER_PROFILE = 2_000 +MAX_REGEX_PATTERNS_PER_PROFILE = 10_000 +MAX_REGEX_ENTITIES_TOTAL = 10_000 +MAX_REGEX_PATTERNS_TOTAL = 50_000 +MAX_REGEX_PATTERN_BYTES = 16 * 1024 +MAX_MATCHES_PER_PATTERN = 256 +DEFAULT_SCAN_TIMEOUT_SECONDS = 1.0 +MAX_SCAN_TIMEOUT_SECONDS = 30.0 + # Service concurrency and transport limits. MAX_CONCURRENT_SCANS = 4 MAX_CONCURRENT_RPCS = 16 diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py index 561641c..1bd0643 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -41,6 +41,7 @@ class ErrorCode(StrEnum): FORMAT_HANDLER_EXECUTION_FAILED = "format_handler_execution_failed" SCANNER_OUTPUT_INVALID = "scanner_output_invalid" SCANNER_EXECUTION_FAILED = "scanner_execution_failed" + SCANNER_CONFIG_INVALID = "scanner_config_invalid" FINDING_LIMIT_EXCEEDED = "finding_limit_exceeded" RESULT_LIMIT_EXCEEDED = "result_limit_exceeded" SERVER_BIND_FAILED = "server_bind_failed" @@ -94,6 +95,15 @@ def __str__(self) -> str: ) +class InternalPrivacyGuardError(PrivacyGuardError): + """A cataloged failure reclassified as internal for its runtime context.""" + + @property + @override + def kind(self) -> ErrorKind: + return ErrorKind.INTERNAL + + _ERROR_SPECS: dict[ErrorCode, ErrorSpec] = { ErrorCode.CONFIG_INVALID: ErrorSpec( ErrorKind.INVALID_INPUT, @@ -103,6 +113,14 @@ def __str__(self) -> str: "Check allowed fields, strict string types, finding action, confidence, " "entity filters, and redact template syntax.", ), + ErrorCode.SCANNER_CONFIG_INVALID: ErrorSpec( + ErrorKind.INVALID_INPUT, + ErrorComponent.SCANNER, + "configure", + "Scanner configuration is invalid.", + "Check the YAML shape, selected profile, catalog limits, names, and " + "regular expressions.", + ), ErrorCode.REQUEST_PHASE_INVALID: ErrorSpec( ErrorKind.INVALID_INPUT, ErrorComponent.SERVICE, diff --git a/projects/privacy-guard/src/privacy_guard/processor.py b/projects/privacy-guard/src/privacy_guard/processor.py index 8d75c5c..dffc3ae 100644 --- a/projects/privacy-guard/src/privacy_guard/processor.py +++ b/projects/privacy-guard/src/privacy_guard/processor.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math from collections.abc import Mapping, Sequence from dataclasses import dataclass from string import Formatter @@ -15,12 +16,15 @@ from privacy_guard.constants import ( BLOCK_REASON_CODE, CONFIDENCE_RANK, + DEFAULT_SCAN_TIMEOUT_SECONDS, LIMIT_REASON_CODE, MAX_BODY_BYTES, MAX_FINDINGS_PER_BLOCK, MAX_FINDINGS_PER_REQUEST, + MAX_SCAN_TIMEOUT_SECONDS, MAX_SCANNED_CHARACTERS, MAX_TEXT_BLOCKS, + PATTERN_NAME_METADATA_KEY, ) from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.payloads import ( @@ -38,6 +42,8 @@ from privacy_guard.scanners import ( Finding, RequestBodyFinding, + ScanBudget, + ScanBudgetExceeded, Scanner, ScannerConfig, ScannerContractError, @@ -56,7 +62,17 @@ def __init__( self, scanners: Sequence[Scanner[ScannerConfig]], format_handlers: Mapping[str, FormatHandler] = DEFAULT_FORMAT_HANDLERS, + *, + scan_timeout_seconds: float = DEFAULT_SCAN_TIMEOUT_SECONDS, ) -> None: + if ( + isinstance(scan_timeout_seconds, bool) + or not isinstance(scan_timeout_seconds, int | float) + or not math.isfinite(scan_timeout_seconds) + or scan_timeout_seconds <= 0 + or scan_timeout_seconds > MAX_SCAN_TIMEOUT_SECONDS + ): + raise ValueError("scan timeout must be finite, positive, and bounded") scanner_tuple = tuple(scanners) if not scanner_tuple: raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) @@ -72,6 +88,7 @@ def __init__( supported_entities.update(item.supported_entity_types) self._scanners: tuple[Scanner[ScannerConfig], ...] = scanner_tuple self._supported_entities = frozenset(supported_entities) + self._scan_timeout_seconds = float(scan_timeout_seconds) self._format_handlers: dict[str, FormatHandler] = {} for format_name, handler in format_handlers.items(): if not isinstance(handler, FormatHandler): @@ -94,7 +111,8 @@ def process(self, request: InterceptedRequest) -> ProcessingResult: request_body = _normalize_request_body(handler, request) verified_body = _validate_normalized_body(request_body, request.raw_body) - scan_result = self._scan_text_blocks(verified_body.text_blocks, action) + budget = ScanBudget.from_timeout(self._scan_timeout_seconds) + scan_result = self._scan_text_blocks(verified_body.text_blocks, action, budget) if scan_result.limit_exceeded: return ProcessingResult( decision=ProcessingDecision.DENY, reason_code=LIMIT_REASON_CODE @@ -164,14 +182,23 @@ def _validate_entity_filter(self, action: ActionConfig) -> None: raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) def _scan_and_validate( - self, scanner: Scanner[ScannerConfig], text_block: TextBlock + self, + scanner: Scanner[ScannerConfig], + text_block: TextBlock, + budget: ScanBudget, ) -> tuple[Finding, ...]: try: - result = scanner.scan(text_block.text) + result = scanner.scan(text_block.text, budget=budget) + except ScanBudgetExceeded: + raise except ScannerFindingLimitExceeded: raise PrivacyGuardError(ErrorCode.FINDING_LIMIT_EXCEEDED) from None except ScannerContractError: raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) from None + except PrivacyGuardError as error: + if error.code is ErrorCode.SCANNER_CONFIG_INVALID: + raise + raise PrivacyGuardError(ErrorCode.SCANNER_EXECUTION_FAILED) from None except Exception: raise PrivacyGuardError(ErrorCode.SCANNER_EXECUTION_FAILED) from None for finding in result: @@ -182,11 +209,22 @@ def _scan_and_validate( ): raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) return tuple( - sorted(result, key=lambda item: (item.start_offset, item.end_offset)) + sorted( + result, + key=lambda item: ( + item.start_offset, + item.end_offset, + item.entity, + _finding_pattern_name(item), + ), + ) ) def _scan_text_blocks( - self, text_blocks: tuple[TextBlock, ...], action: ActionConfig + self, + text_blocks: tuple[TextBlock, ...], + action: ActionConfig, + budget: ScanBudget, ) -> _ScanResult: by_path: dict[str, tuple[Finding, ...]] = {} all_findings: list[RequestBodyFinding] = [] @@ -200,7 +238,7 @@ def _scan_text_blocks( block_findings: list[Finding] = [] try: for scanner in self._scanners: - for finding in self._scan_and_validate(scanner, block): + for finding in self._scan_and_validate(scanner, block, budget): if (enabled is None or finding.entity in enabled) and ( threshold is None or CONFIDENCE_RANK[finding.confidence] >= threshold @@ -212,6 +250,8 @@ def _scan_text_blocks( > MAX_FINDINGS_PER_REQUEST ): return _ScanResult((), {}, True) + except ScanBudgetExceeded: + return _ScanResult((), {}, True) except PrivacyGuardError as error: if error.code is ErrorCode.FINDING_LIMIT_EXCEEDED: return _ScanResult((), {}, True) @@ -224,6 +264,7 @@ def _scan_text_blocks( item.end_offset, item.scanner_name, item.entity, + _finding_pattern_name(item), ), ) ) @@ -296,6 +337,7 @@ def _resolve_overlaps(findings: tuple[Finding, ...]) -> tuple[Finding, ...]: item.end_offset, item.scanner_name, item.entity, + _finding_pattern_name(item), ), ) for candidate in ranked: @@ -305,7 +347,18 @@ def _resolve_overlaps(findings: tuple[Finding, ...]) -> tuple[Finding, ...]: for winner in winners ): winners.append(candidate) - return tuple(sorted(winners, key=lambda item: (item.start_offset, item.end_offset))) + return tuple( + sorted( + winners, + key=lambda item: ( + item.start_offset, + item.end_offset, + item.scanner_name, + item.entity, + _finding_pattern_name(item), + ), + ) + ) def _redact_text( @@ -378,3 +431,9 @@ def _reconstruct_body( raise except Exception: raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_EXECUTION_FAILED) from None + + +def _finding_pattern_name(finding: Finding) -> str: + if finding.metadata is None: + return "" + return finding.metadata.get(PATTERN_NAME_METADATA_KEY, "") diff --git a/projects/privacy-guard/src/privacy_guard/scanners/__init__.py b/projects/privacy-guard/src/privacy_guard/scanners/__init__.py index 2334b6b..67212d6 100644 --- a/projects/privacy-guard/src/privacy_guard/scanners/__init__.py +++ b/projects/privacy-guard/src/privacy_guard/scanners/__init__.py @@ -6,19 +6,31 @@ Confidence, Finding, RequestBodyFinding, + ScanBudget, + ScanBudgetExceeded, Scanner, ScannerConfig, ScannerContractError, ScannerFindingLimitExceeded, parse_scanner_output, ) -from privacy_guard.scanners.passthrough import PassthroughScanner +from privacy_guard.scanners.regex import ( + RegexEntity, + RegexPattern, + RegexScanner, + RegexScannerConfig, +) __all__ = [ "Confidence", "Finding", - "PassthroughScanner", "RequestBodyFinding", + "RegexEntity", + "RegexPattern", + "RegexScanner", + "RegexScannerConfig", + "ScanBudget", + "ScanBudgetExceeded", "Scanner", "ScannerConfig", "ScannerContractError", diff --git a/projects/privacy-guard/src/privacy_guard/scanners/base.py b/projects/privacy-guard/src/privacy_guard/scanners/base.py index 85533a0..f600c85 100644 --- a/projects/privacy-guard/src/privacy_guard/scanners/base.py +++ b/projects/privacy-guard/src/privacy_guard/scanners/base.py @@ -3,7 +3,10 @@ from __future__ import annotations from abc import ABC, abstractmethod +from collections.abc import Mapping from enum import StrEnum +from time import monotonic +from types import MappingProxyType from typing import Generic, Self, TypeVar, get_args, get_origin from pydantic import ( @@ -14,7 +17,12 @@ ) from typing_extensions import TypeIs -from privacy_guard.constants import MAX_FINDINGS_PER_BLOCK, MAX_SCANNER_METADATA_BYTES +from privacy_guard.constants import ( + DEFAULT_SCAN_TIMEOUT_SECONDS, + MAX_FINDING_METADATA_ENTRIES, + MAX_FINDINGS_PER_BLOCK, + MAX_SCANNER_METADATA_BYTES, +) from privacy_guard.validation import ScalarString, StrictDomainModel @@ -34,6 +42,7 @@ class Finding(StrictDomainModel): start_offset: int = Field(..., ge=0) end_offset: int confidence: Confidence = Confidence.HIGH + metadata: Mapping[str, str] | None = Field(default=None, repr=False) @field_validator("entity", "scanner_name") @classmethod @@ -45,6 +54,21 @@ def validate_metadata_boundary(cls, value: str) -> str: raise ValueError("finding metadata exceeds the UTF-8 byte limit") return value + @field_validator("metadata") + @classmethod + def validate_finding_metadata( + cls, value: Mapping[str, str] | None + ) -> Mapping[str, str] | None: + """Copy valid, bounded string metadata into an immutable mapping.""" + if value is None: + return None + if len(value) > MAX_FINDING_METADATA_ENTRIES: + raise ValueError("finding metadata has too many entries") + for key, item in value.items(): + cls.validate_metadata_boundary(key) + cls.validate_metadata_boundary(item) + return MappingProxyType(dict(value)) + @model_validator(mode="after") def validate_non_empty_span(self) -> Self: """Require the finding to cover at least one character.""" @@ -75,6 +99,32 @@ class ScannerFindingLimitExceeded(Exception): """The scanner exceeded the bounded per-block finding count.""" +class ScanBudgetExceeded(Exception): + """A content-safe signal that scanning exhausted its shared request budget.""" + + +class ScanBudget(StrictDomainModel): + """Immutable request-scoped monotonic deadline shared by all scanner calls.""" + + deadline: float = Field(allow_inf_nan=False) + + @classmethod + def from_timeout(cls, timeout_seconds: float) -> Self: + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, int | float) + or timeout_seconds <= 0 + ): + raise ValueError("scan timeout must be finite and positive") + return cls(deadline=monotonic() + timeout_seconds) + + def remaining_seconds(self) -> float: + remaining = self.deadline - monotonic() + if remaining <= 0: + raise ScanBudgetExceeded + return remaining + + def parse_scanner_output(result: object) -> tuple[Finding, ...]: """Validate a scanner's tuple output against the Finding contract.""" if not isinstance(result, tuple): @@ -107,6 +157,7 @@ def __init__(self, config: _ScannerConfigT) -> None: raise ScannerContractError("scanner configuration is invalid") from None self.__config = validated_config self.__scanner_name = validated_config.name + self._initialize() @classmethod def get_config_type(cls) -> type[_ScannerConfigT]: @@ -133,14 +184,30 @@ def supported_entity_types(self) -> frozenset[str]: """Return the complete entity catalog declared by the scanner config.""" return self.config.entity_types - def scan(self, text_block: ScalarString) -> tuple[Finding, ...]: - """Run the implementation and validate its observable output shape.""" - result: object = self._scan(text_block) + def _initialize(self) -> None: + """Optionally initialize scanner state after validated config is retained.""" + + def scan( + self, text_block: ScalarString, *, budget: ScanBudget | None = None + ) -> tuple[Finding, ...]: + """Scan one block using a shared or standalone request deadline. + + RequestProcessor supplies one budget to every scanner and text block in a + request. Standalone callers may omit it and receive a fresh default budget. + """ + effective_budget = budget or ScanBudget.from_timeout( + DEFAULT_SCAN_TIMEOUT_SECONDS + ) + result: object = self._scan(text_block, effective_budget) return parse_scanner_output(result) @abstractmethod - def _scan(self, text_block: str) -> tuple[Finding, ...]: - """Return block-relative findings for one block of text.""" + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: + """Return block-relative findings for one block of text. + + Implementations with potentially expensive work should call + ``budget.remaining_seconds()`` at practical interruption points. + """ raise NotImplementedError @@ -148,6 +215,8 @@ def _scan(self, text_block: str) -> tuple[Finding, ...]: "Confidence", "Finding", "RequestBodyFinding", + "ScanBudget", + "ScanBudgetExceeded", "Scanner", "ScannerConfig", "ScannerContractError", diff --git a/projects/privacy-guard/src/privacy_guard/scanners/passthrough.py b/projects/privacy-guard/src/privacy_guard/scanners/passthrough.py deleted file mode 100644 index 1b48714..0000000 --- a/projects/privacy-guard/src/privacy_guard/scanners/passthrough.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Scanner that deliberately reports no findings for a text block.""" - -from __future__ import annotations - -from typing_extensions import override - -from privacy_guard.scanners.base import ( - Finding, - Scanner, - ScannerConfig, -) - - -class PassthroughScanner(Scanner[ScannerConfig]): - """Scan one text block without reporting sensitive values.""" - - def __init__(self) -> None: - super().__init__(ScannerConfig(name="passthrough", entity_types=frozenset())) - - @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: - """Return no findings for the supplied text block.""" - return () diff --git a/projects/privacy-guard/src/privacy_guard/scanners/regex.py b/projects/privacy-guard/src/privacy_guard/scanners/regex.py new file mode 100644 index 0000000..0fd34c4 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/scanners/regex.py @@ -0,0 +1,430 @@ +"""Strict YAML-configured regular-expression scanner.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Self + +import yaml +from pydantic import Field, ValidationError, field_validator, model_validator +from typing_extensions import TypeIs, override +from yaml.events import ( + AliasEvent, + CollectionEndEvent, + CollectionStartEvent, + NodeEvent, + ScalarEvent, +) +from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode + +from privacy_guard.constants import ( + MAX_FINDINGS_PER_BLOCK, + MAX_MATCHES_PER_PATTERN, + MAX_REGEX_ENTITIES_PER_PROFILE, + MAX_REGEX_ENTITIES_TOTAL, + MAX_REGEX_NAME_BYTES, + MAX_REGEX_PATTERN_BYTES, + MAX_REGEX_PATTERNS_PER_PROFILE, + MAX_REGEX_PATTERNS_TOTAL, + MAX_REGEX_PROFILES, + MAX_SCANNER_CONFIG_BYTES, + MAX_SCANNER_CONFIG_NESTING, + MAX_SCANNER_CONFIG_NODES, + MAX_SCANNER_CONFIG_SCALAR_BYTES, + PATTERN_NAME_METADATA_KEY, +) +from privacy_guard.errors import ( + ErrorCode, + InternalPrivacyGuardError, + PrivacyGuardError, +) +from privacy_guard.scanners.base import ( + Confidence, + Finding, + ScanBudget, + Scanner, + ScannerConfig, + ScannerFindingLimitExceeded, +) +from privacy_guard.validation import StrictSensitiveModel + + +class RegexPattern(StrictSensitiveModel): + """One named expression and its explicit engine flags.""" + + name: str + regex: str = Field(repr=False) + confidence: Confidence + ignore_case: bool = False + multiline: bool = False + dot_all: bool = False + ascii: bool = False + + @field_validator("name") + @classmethod + def _name_is_safe(cls, value: str) -> str: + return _validate_name(value) + + @field_validator("regex") + @classmethod + def _regex_is_bounded(cls, value: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError("regex must be a non-empty string") + if any("\ud800" <= character <= "\udfff" for character in value): + raise ValueError("regex must contain valid Unicode") + if len(value.encode("utf-8")) > MAX_REGEX_PATTERN_BYTES: + raise ValueError("regex is too long") + return value + + @field_validator("confidence", mode="before") + @classmethod + def _parse_confidence(cls, value: object) -> Confidence: + if not isinstance(value, str): + raise ValueError("confidence must be a string") + return Confidence(value) + + +class RegexEntity(StrictSensitiveModel): + """A named entity and its non-empty pattern catalog.""" + + name: str + patterns: tuple[RegexPattern, ...] = Field(repr=False) + + @field_validator("name") + @classmethod + def _name_is_safe(cls, value: str) -> str: + return _validate_name(value) + + @field_validator("patterns", mode="before") + @classmethod + def _patterns_are_a_list(cls, value: object) -> object: + if not isinstance(value, list | tuple) or not value: + raise ValueError("patterns must be a non-empty list") + return tuple(value) + + @model_validator(mode="after") + def _pattern_names_are_unique(self) -> Self: + names = [pattern.name for pattern in self.patterns] + if len(names) != len(set(names)): + raise ValueError("pattern names must be unique within an entity") + return self + + +class RegexScannerConfig(ScannerConfig): + """Selected, immutable regular-expression entity catalog.""" + + entities: tuple[RegexEntity, ...] = Field(repr=False) + + @model_validator(mode="after") + def _catalog_is_consistent(self) -> Self: + if not self.entities: + raise ValueError("entity catalog must not be empty") + names = [entity.name for entity in self.entities] + if len(names) != len(set(names)) or self.entity_types != frozenset(names): + raise ValueError("entity catalog is inconsistent") + if len(self.entities) > MAX_REGEX_ENTITIES_PER_PROFILE: + raise ValueError("entity catalog is too large") + if sum(len(entity.patterns) for entity in self.entities) > ( + MAX_REGEX_PATTERNS_PER_PROFILE + ): + raise ValueError("pattern catalog is too large") + return self + + +class RegexScanner(Scanner[RegexScannerConfig]): + """Find every configured, possibly overlapping regular-expression match.""" + + @override + def _initialize(self) -> None: + """Compile the validated catalog once for reuse across scanner calls.""" + try: + self._rules = tuple( + _compile_rule(entity.name, pattern) + for entity in self.config.entities + for pattern in entity.patterns + ) + except (ValueError, re.error): + raise PrivacyGuardError(ErrorCode.SCANNER_CONFIG_INVALID) from None + + @classmethod + def from_yaml( + cls, + path: str | Path, + profile: str | None = None, + *, + scanner_name: str = "regex", + ) -> Self: + """Load, validate, select, and compile a complete YAML catalog.""" + try: + raw = Path(path).read_bytes() + value = _load_yaml(raw) + catalogs = _parse_catalogs(value) + for entities in catalogs.values(): + for entity in entities: + for pattern in entity.patterns: + _compile_rule(entity.name, pattern) + entities = _select_catalog(catalogs, profile) + config = RegexScannerConfig( + name=_validate_name(scanner_name), + entity_types=frozenset(entity.name for entity in entities), + entities=entities, + ) + return cls(config) + except PrivacyGuardError: + raise + except ( + OSError, + UnicodeError, + ValueError, + TypeError, + ValidationError, + yaml.YAMLError, + re.error, + ): + raise PrivacyGuardError(ErrorCode.SCANNER_CONFIG_INVALID) from None + + @override + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: + findings: list[Finding] = [] + for rule in self._rules: + match_count = 0 + next_position = 0 + while next_position <= len(text_block): + budget.remaining_seconds() + match = rule.expression.search(text_block, next_position) + budget.remaining_seconds() + if match is None: + break + start, end = match.span() + if start == end: + raise InternalPrivacyGuardError(ErrorCode.SCANNER_CONFIG_INVALID) + if match.span(rule.marker_name) == (-1, -1): + raise InternalPrivacyGuardError(ErrorCode.SCANNER_CONFIG_INVALID) + match_count += 1 + if match_count > MAX_MATCHES_PER_PATTERN: + raise ScannerFindingLimitExceeded + findings.append( + Finding( + entity=rule.entity_name, + metadata={PATTERN_NAME_METADATA_KEY: rule.pattern_name}, + scanner_name=self.scanner_name, + start_offset=start, + end_offset=end, + confidence=rule.confidence, + ) + ) + if len(findings) > MAX_FINDINGS_PER_BLOCK: + raise ScannerFindingLimitExceeded + next_position = start + 1 + return tuple(findings) + + +__all__ = ["RegexEntity", "RegexPattern", "RegexScanner", "RegexScannerConfig"] + + +@dataclass(frozen=True) +class _CompiledRule: + entity_name: str + pattern_name: str + confidence: Confidence + marker_name: str + expression: re.Pattern[str] + + +def _validate_name(value: str) -> str: + if not isinstance(value, str) or _NAME_PATTERN.fullmatch(value) is None: + raise ValueError("name is invalid") + if len(value.encode("ascii")) > MAX_REGEX_NAME_BYTES: + raise ValueError("name is too long") + return value + + +def _compile_rule(entity_name: str, pattern: RegexPattern) -> _CompiledRule: + flags = 0 + if pattern.ignore_case: + flags |= re.IGNORECASE + if pattern.multiline: + flags |= re.MULTILINE + if pattern.dot_all: + flags |= re.DOTALL + if pattern.ascii: + flags |= re.ASCII + if _contains_inline_flags(pattern.regex): + raise ValueError("inline flags are unsupported") + unmarked = re.compile(pattern.regex, flags) + if unmarked.groupindex: + raise ValueError("named groups are reserved") + if unmarked.search("") is not None: + raise ValueError("regex must not match empty input") + marker_name = pattern.name.replace("-", "_") + expression = re.compile(f"(?:{pattern.regex})(?P<{marker_name}>)", flags) + return _CompiledRule( + entity_name=entity_name, + pattern_name=pattern.name, + confidence=pattern.confidence, + marker_name=marker_name, + expression=expression, + ) + + +def _contains_inline_flags(expression: str) -> bool: + escaped = False + in_character_class = False + index = 0 + while index < len(expression): + character = expression[index] + if escaped: + escaped = False + elif character == "\\": + escaped = True + elif character == "[": + in_character_class = True + elif character == "]" and in_character_class: + in_character_class = False + elif not in_character_class and expression.startswith("(?", index): + suffix = expression[index + 2 :] + flag_match = re.match(r"[A-Za-z0-9-]+(?=[:)])", suffix) + if flag_match is not None: + return True + index += 1 + return False + + +def _load_yaml(raw: bytes) -> object: + if not raw or len(raw) > MAX_SCANNER_CONFIG_BYTES: + raise ValueError("configuration size is invalid") + text = raw.decode("utf-8", errors="strict") + _validate_yaml_events(text) + root = yaml.compose(text, Loader=yaml.SafeLoader) + if root is None: + raise ValueError("configuration is empty") + _validate_yaml_node(root, depth=1, count=[0]) + return yaml.safe_load(text) + + +def _validate_yaml_events(text: str) -> None: + depth = 0 + node_count = 0 + for event in yaml.parse(text, Loader=yaml.SafeLoader): + if isinstance(event, AliasEvent) or ( + isinstance(event, NodeEvent) and event.anchor is not None + ): + raise ValueError("anchors and aliases are unsupported") + if isinstance(event, NodeEvent): + node_count += 1 + if node_count > MAX_SCANNER_CONFIG_NODES: + raise ValueError("configuration has too many nodes") + if event.tag is not None and not event.tag.startswith("tag:yaml.org,2002:"): + raise ValueError("YAML tag is unsupported") + if isinstance(event, ScalarEvent) and ( + len(event.value.encode("utf-8")) > MAX_SCANNER_CONFIG_SCALAR_BYTES + ): + raise ValueError("configuration scalar is too large") + if isinstance(event, CollectionStartEvent): + depth += 1 + if depth > MAX_SCANNER_CONFIG_NESTING: + raise ValueError("configuration nesting is too deep") + elif isinstance(event, CollectionEndEvent): + depth -= 1 + + +def _validate_yaml_node(node: Node, *, depth: int, count: list[int]) -> None: + count[0] += 1 + if depth > MAX_SCANNER_CONFIG_NESTING or count[0] > MAX_SCANNER_CONFIG_NODES: + raise ValueError("configuration structure is too large") + if isinstance(node, ScalarNode): + if len(node.value.encode("utf-8")) > MAX_SCANNER_CONFIG_SCALAR_BYTES: + raise ValueError("configuration scalar is too large") + return + if isinstance(node, SequenceNode): + for child in node.value: + _validate_yaml_node(child, depth=depth + 1, count=count) + return + if isinstance(node, MappingNode): + keys: set[tuple[str, str]] = set() + for key, value in node.value: + if not isinstance(key, ScalarNode): + raise ValueError("mapping keys must be scalars") + identity = (key.tag, key.value) + if identity in keys: + raise ValueError("mapping keys must be unique") + keys.add(identity) + _validate_yaml_node(key, depth=depth + 1, count=count) + _validate_yaml_node(value, depth=depth + 1, count=count) + return + raise ValueError("YAML node is unsupported") + + +def _parse_catalogs(value: object) -> dict[str | None, tuple[RegexEntity, ...]]: + if isinstance(value, list): + return {None: _parse_entities(value)} + if not _is_object_mapping(value) or set(value) != {"profiles"}: + raise ValueError("configuration shape is invalid") + profiles = value["profiles"] + if not isinstance(profiles, Mapping) or not profiles: + raise ValueError("profiles must be a non-empty mapping") + if len(profiles) > MAX_REGEX_PROFILES: + raise ValueError("too many profiles") + catalogs: dict[str | None, tuple[RegexEntity, ...]] = {} + for name, entities in profiles.items(): + if not isinstance(name, str): + raise ValueError("profile name must be a string") + catalogs[_validate_name(name)] = _parse_entities(entities) + _validate_document_totals(catalogs) + return catalogs + + +def _is_object_mapping(value: object) -> TypeIs[Mapping[object, object]]: + return isinstance(value, Mapping) + + +def _parse_entities(value: object) -> tuple[RegexEntity, ...]: + if not isinstance(value, list) or not value: + raise ValueError("entity catalog must be a non-empty list") + entities = tuple(RegexEntity.model_validate(item) for item in value) + names = [entity.name for entity in entities] + if len(names) != len(set(names)): + raise ValueError("entity names must be unique") + if len(entities) > MAX_REGEX_ENTITIES_PER_PROFILE: + raise ValueError("too many entities") + pattern_count = sum(len(entity.patterns) for entity in entities) + if pattern_count > MAX_REGEX_PATTERNS_PER_PROFILE: + raise ValueError("too many patterns") + return entities + + +def _validate_document_totals( + catalogs: Mapping[str | None, tuple[RegexEntity, ...]], +) -> None: + if sum(len(entities) for entities in catalogs.values()) > MAX_REGEX_ENTITIES_TOTAL: + raise ValueError("too many entities in document") + if ( + sum( + len(entity.patterns) + for entities in catalogs.values() + for entity in entities + ) + > MAX_REGEX_PATTERNS_TOTAL + ): + raise ValueError("too many patterns in document") + + +def _select_catalog( + catalogs: Mapping[str | None, tuple[RegexEntity, ...]], profile: str | None +) -> tuple[RegexEntity, ...]: + if None in catalogs: + if profile is not None: + raise ValueError("profile is invalid for single-profile configuration") + return catalogs[None] + if profile is None: + raise ValueError("profile is required") + try: + return catalogs[_validate_name(profile)] + except KeyError: + raise ValueError("profile does not exist") from None + + +_NAME_PATTERN = re.compile(r"[A-Za-z_][A-Za-z0-9_-]*\Z") diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 0aa4b0c..2dca880 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -17,7 +17,7 @@ from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.processor import RequestProcessor -from privacy_guard.scanners import PassthroughScanner, Scanner, ScannerConfig +from privacy_guard.scanners import RegexScanner, Scanner, ScannerConfig from privacy_guard.service.servicer import PrivacyGuardMiddleware @@ -29,13 +29,12 @@ def __init__(self, *, scanner: Scanner[ScannerConfig]) -> None: def serve(self, listen: str = "127.0.0.1:50051") -> None: """Serve until termination using a managed synchronous entry point.""" - asyncio.run(serve(listen, self._servicer)) + asyncio.run(serve(self._servicer, listen)) -def create_server(servicer: PrivacyGuardMiddleware | None = None) -> grpc.aio.Server: +def create_server(servicer: PrivacyGuardMiddleware) -> grpc.aio.Server: """Build an unstarted gRPC server with the servicer mounted (no port bound). - Accepts a servicer for tests; defaults to a fresh ``PrivacyGuardMiddleware``. The receive limit reserves bounded space around the advertised body maximum for the protobuf envelope; the servicer enforces the body limit itself. """ @@ -43,19 +42,16 @@ def create_server(servicer: PrivacyGuardMiddleware | None = None) -> grpc.aio.Se maximum_concurrent_rpcs=MAX_CONCURRENT_RPCS, options=(("grpc.max_receive_message_length", MAX_RECEIVE_MESSAGE_BYTES),), ) - pb2_grpc.add_SupervisorMiddlewareServicer_to_server( - servicer if servicer is not None else PrivacyGuardMiddleware(), server - ) + pb2_grpc.add_SupervisorMiddlewareServicer_to_server(servicer, server) return server async def serve( + servicer: PrivacyGuardMiddleware, listen: str = "127.0.0.1:50051", - servicer: PrivacyGuardMiddleware | None = None, ) -> None: """Bind ``listen``, start the server, and serve until terminated.""" - effective_servicer = servicer if servicer is not None else PrivacyGuardMiddleware() - server = create_server(effective_servicer) + server = create_server(servicer) try: bound_port = server.add_insecure_port(listen) if bound_port == 0: @@ -64,15 +60,23 @@ async def serve( await server.wait_for_termination() finally: await server.stop(grace=0) - await effective_servicer.close() + await servicer.close() def main(argv: Sequence[str] | None = None) -> int: - """CLI entry point: parse args and run the high-level middleware server.""" + """Load the required regex catalog, then run the middleware server.""" parser = argparse.ArgumentParser(description="Run the Privacy Guard middleware") + parser.add_argument("--scanner-config", required=True) + parser.add_argument("--profile") + parser.add_argument("--scanner-name", default="regex") parser.add_argument("--listen", default="127.0.0.1:50051") arguments = parser.parse_args(argv) - MiddlewareServer(scanner=PassthroughScanner()).serve(arguments.listen) + scanner = RegexScanner.from_yaml( + arguments.scanner_config, + arguments.profile, + scanner_name=arguments.scanner_name, + ) + MiddlewareServer(scanner=scanner).serve(arguments.listen) return 0 diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index dff52bb..4d2aeab 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -27,6 +27,7 @@ MAX_CONCURRENT_SCANS, MAX_PROTO_FINDING_BYTES, MAX_PROTO_FINDING_GROUPS, + PATTERN_NAME_METADATA_KEY, REASON_CODE_PATTERN, SERVICE_NAME, SERVICE_VERSION, @@ -38,8 +39,7 @@ ProcessingDecision, ProcessingResult, ) -from privacy_guard.processor import RequestProcessor -from privacy_guard.scanners import PassthroughScanner, RequestBodyFinding +from privacy_guard.scanners import RequestBodyFinding class RequestProcessorLike(Protocol): @@ -53,12 +53,8 @@ def process(self, request: InterceptedRequest) -> ProcessingResult: ... class PrivacyGuardMiddleware(pb2_grpc.SupervisorMiddlewareServicer): """Translate protobuf requests and responses at the transport boundary.""" - def __init__(self, processor: RequestProcessorLike | None = None) -> None: - self._processor = ( - processor - if processor is not None - else RequestProcessor([PassthroughScanner()]) - ) + def __init__(self, processor: RequestProcessorLike) -> None: + self._processor = processor self._scan_slots = asyncio.Semaphore(MAX_CONCURRENT_SCANS) self._scan_executor = ThreadPoolExecutor( max_workers=MAX_CONCURRENT_SCANS, @@ -299,21 +295,31 @@ def _limit_deny() -> pb2.HttpRequestResult: def _aggregate_findings(findings: tuple[RequestBodyFinding, ...]) -> list[pb2.Finding]: - groups: OrderedDict[tuple[str, str, str], int] = OrderedDict() + groups: OrderedDict[tuple[str, str, str | None, str], int] = OrderedDict() for request_body_finding in findings: finding = request_body_finding.finding - key = (finding.scanner_name, finding.entity, finding.confidence.value) + pattern_name = ( + None + if finding.metadata is None + else finding.metadata.get(PATTERN_NAME_METADATA_KEY) + ) + key = ( + finding.scanner_name, + finding.entity, + pattern_name, + finding.confidence.value, + ) groups[key] = groups.get(key, 0) + 1 if len(groups) > MAX_PROTO_FINDING_GROUPS: raise PrivacyGuardError(ErrorCode.RESULT_LIMIT_EXCEEDED) aggregated = [ pb2.Finding( type=scanner_name, - label=entity, + label=entity if pattern_name is None else f"{entity}/{pattern_name}", confidence=confidence, count=min(count, UINT32_MAX), ) - for (scanner_name, entity, confidence), count in groups.items() + for (scanner_name, entity, pattern_name, confidence), count in groups.items() ] if any(finding.ByteSize() > MAX_PROTO_FINDING_BYTES for finding in aggregated): raise PrivacyGuardError(ErrorCode.RESULT_LIMIT_EXCEEDED) diff --git a/projects/privacy-guard/tests/scanner_helpers.py b/projects/privacy-guard/tests/scanner_helpers.py index 845ae6e..81188ea 100644 --- a/projects/privacy-guard/tests/scanner_helpers.py +++ b/projects/privacy-guard/tests/scanner_helpers.py @@ -9,6 +9,7 @@ from privacy_guard.scanners import ( Confidence, Finding, + ScanBudget, Scanner, ScannerConfig, ) @@ -20,7 +21,7 @@ class DeterministicEmailScanner(Scanner[ScannerConfig]): ) @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: return tuple( Finding( entity="email", diff --git a/projects/privacy-guard/tests/scanners/test_regex.py b/projects/privacy-guard/tests/scanners/test_regex.py new file mode 100644 index 0000000..b83783f --- /dev/null +++ b/projects/privacy-guard/tests/scanners/test_regex.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from time import monotonic + +import pytest + +from privacy_guard.errors import ErrorCode, ErrorKind, PrivacyGuardError +from privacy_guard.scanners import RegexScanner, ScanBudget, ScanBudgetExceeded + + +def _write(path: Path, text: str) -> Path: + path.write_text(text, encoding="utf-8") + return path + + +def _single(patterns: str) -> str: + return f""" +- name: token + patterns: +{patterns} +""" + + +def test_single_profile_reports_overlaps_names_confidence_and_unicode( + tmp_path: Path, +) -> None: + path = _write( + tmp_path / "entities.yaml", + _single( + """ - name: whole-token + regex: 'aba' + confidence: high + - name: suffix-token + regex: 'ba' + confidence: medium +""" + ), + ) + + findings = RegexScanner.from_yaml(path).scan("🐍aba") + + assert [ + ( + item.metadata["pattern_name"] if item.metadata is not None else None, + item.start_offset, + item.end_offset, + item.confidence.value, + ) + for item in findings + ] == [("whole-token", 1, 4, "high"), ("suffix-token", 2, 4, "medium")] + + +def test_one_pattern_uses_overlapping_iteration(tmp_path: Path) -> None: + path = _write( + tmp_path / "overlap.yaml", + _single(" - name: pair\n regex: 'aa'\n confidence: high\n"), + ) + + findings = RegexScanner.from_yaml(path).scan("aaa") + + assert [(item.start_offset, item.end_offset) for item in findings] == [ + (0, 2), + (1, 3), + ] + + +def test_hyphen_normalization_coexists_with_underscore_name(tmp_path: Path) -> None: + path = _write( + tmp_path / "names.yaml", + _single( + """ - name: same-name + regex: 'x' + confidence: high + - name: same_name + regex: 'y' + confidence: high +""" + ), + ) + + findings = RegexScanner.from_yaml(path).scan("xy") + + assert [ + item.metadata["pattern_name"] if item.metadata is not None else None + for item in findings + ] == ["same-name", "same_name"] + + +def test_numeric_backreferences_keep_their_group_numbers(tmp_path: Path) -> None: + path = _write( + tmp_path / "backref.yaml", + _single( + " - name: repeated\n regex: '(a)\\1'\n confidence: high\n" + ), + ) + + finding = RegexScanner.from_yaml(path).scan("aa")[0] + assert finding.metadata is not None + assert finding.metadata["pattern_name"] == "repeated" + + +def test_profiles_require_selection_and_do_not_merge(tmp_path: Path) -> None: + path = _write( + tmp_path / "profiles.yaml", + """ +profiles: + first: + - name: alpha + patterns: + - name: alpha + regex: 'a' + confidence: high + second: + - name: beta + patterns: + - name: beta + regex: 'b' + confidence: high +""", + ) + + with pytest.raises(PrivacyGuardError) as missing: + RegexScanner.from_yaml(path) + assert missing.value.code is ErrorCode.SCANNER_CONFIG_INVALID + + scanner = RegexScanner.from_yaml(path, "second") + assert scanner.supported_entity_types == frozenset({"beta"}) + assert [item.entity for item in scanner.scan("ab")] == ["beta"] + + +@pytest.mark.parametrize( + "document", + [ + "- name: x\n name: y\n patterns: []\n", + "- &entity\n name: x\n patterns: []\n", + "- !unsafe {name: x, patterns: []}\n", + "- name: x\n patterns:\n" + " [{name: p, regex: 'x', confidence: high, extra: true}]\n", + "- name: x\n patterns: [{name: p, regex: '(?Px)', confidence: high}]\n", + "- name: x\n patterns: [{name: p, regex: '(?i:x)', confidence: high}]\n", + "- name: x\n patterns: [{name: p, regex: 'x*', confidence: high}]\n", + ], +) +def test_invalid_yaml_and_patterns_use_one_content_safe_error( + tmp_path: Path, document: str +) -> None: + path = _write(tmp_path / "sensitive-name-8472.yaml", document) + + with pytest.raises(PrivacyGuardError) as exception_info: + RegexScanner.from_yaml(path) + + assert exception_info.value.code is ErrorCode.SCANNER_CONFIG_INVALID + assert exception_info.value.kind is ErrorKind.INVALID_INPUT + assert "8472" not in str(exception_info.value) + + +def test_inactive_profiles_are_fully_validated(tmp_path: Path) -> None: + path = _write( + tmp_path / "profiles.yaml", + """ +profiles: + selected: + - name: alpha + patterns: [{name: alpha, regex: 'a', confidence: high}] + broken: + - name: beta + patterns: [{name: beta, regex: '(?Pb)', confidence: high}] +""", + ) + + with pytest.raises(PrivacyGuardError) as exception_info: + RegexScanner.from_yaml(path, "selected") + + assert exception_info.value.code is ErrorCode.SCANNER_CONFIG_INVALID + assert exception_info.value.kind is ErrorKind.INVALID_INPUT + + +def test_excessive_yaml_nesting_is_rejected_safely(tmp_path: Path) -> None: + document = "value" + for _ in range(20): + document = f"[{document}]" + path = _write(tmp_path / "nested.yaml", document) + + with pytest.raises(PrivacyGuardError) as exception_info: + RegexScanner.from_yaml(path) + + assert exception_info.value.code is ErrorCode.SCANNER_CONFIG_INVALID + assert exception_info.value.kind is ErrorKind.INVALID_INPUT + + +def test_contextual_zero_width_is_a_runtime_configuration_failure( + tmp_path: Path, +) -> None: + path = _write( + tmp_path / "contextual.yaml", + _single(" - name: zero\n regex: '(?=a)'\n confidence: high\n"), + ) + scanner = RegexScanner.from_yaml(path) + + with pytest.raises(PrivacyGuardError) as exception_info: + scanner.scan("a") + + assert exception_info.value.code is ErrorCode.SCANNER_CONFIG_INVALID + assert exception_info.value.kind is ErrorKind.INTERNAL + + +def test_standalone_scan_exposes_typed_budget_exhaustion(tmp_path: Path) -> None: + path = _write( + tmp_path / "budget.yaml", + _single(" - name: literal\n regex: 'x'\n confidence: high\n"), + ) + scanner = RegexScanner.from_yaml(path) + + with pytest.raises(ScanBudgetExceeded): + scanner.scan("x", budget=ScanBudget(deadline=monotonic() - 1)) + + +def test_scanner_instance_supports_concurrent_calls(tmp_path: Path) -> None: + path = _write( + tmp_path / "concurrent.yaml", + _single(" - name: literal\n regex: 'x'\n confidence: high\n"), + ) + scanner = RegexScanner.from_yaml(path) + + with ThreadPoolExecutor(max_workers=4) as executor: + results = tuple(executor.map(scanner.scan, ("x",) * 16)) + + assert all(len(findings) == 1 for findings in results) diff --git a/projects/privacy-guard/tests/scanners/test_passthrough.py b/projects/privacy-guard/tests/scanners/test_scanner_models.py similarity index 68% rename from projects/privacy-guard/tests/scanners/test_passthrough.py rename to projects/privacy-guard/tests/scanners/test_scanner_models.py index ccdf478..21deb94 100644 --- a/projects/privacy-guard/tests/scanners/test_passthrough.py +++ b/projects/privacy-guard/tests/scanners/test_scanner_models.py @@ -1,10 +1,15 @@ +import math +from types import MappingProxyType + import pytest from pydantic import ValidationError +from typing_extensions import override from privacy_guard.scanners import ( Finding, - PassthroughScanner, RequestBodyFinding, + ScanBudget, + Scanner, ScannerConfig, ) @@ -20,11 +25,21 @@ def test_finding_names_the_scanner_that_produced_it() -> None: assert finding.scanner_name == "example-scanner" -def test_passthrough_scanner_has_stable_name() -> None: - scanner = PassthroughScanner() +def test_finding_accepts_general_immutable_string_metadata() -> None: + finding = Finding( + entity="email", + scanner_name="scanner", + start_offset=0, + end_offset=1, + metadata={"rule": "common-email", "source": "customer-catalog"}, + ) - assert scanner.scanner_name == "passthrough" - assert scanner.supported_entity_types == frozenset() + assert finding.metadata == { + "rule": "common-email", + "source": "customer-catalog", + } + assert isinstance(finding.metadata, MappingProxyType) + assert "customer-catalog" not in repr(finding) def test_scanner_config_owns_identity_and_entity_types() -> None: @@ -36,13 +51,6 @@ def test_scanner_config_owns_identity_and_entity_types() -> None: setattr(config, "name", "changed") -def test_passthrough_uses_minimal_config_with_fixed_empty_catalog() -> None: - scanner = PassthroughScanner() - - assert type(scanner.config) is ScannerConfig - assert scanner.supported_entity_types == frozenset() - - @pytest.mark.parametrize( "values", [ @@ -100,6 +108,13 @@ def test_scanner_config_rejects_invalid_metadata(values: dict[str, object]) -> N "end_offset": 1, "confidence": "high", }, + { + "entity": "email", + "scanner_name": "scanner", + "start_offset": 0, + "end_offset": 1, + "metadata": {"key": 3}, + }, { "entity": "email", "scanner_name": "scanner", @@ -135,21 +150,26 @@ def test_request_body_finding_requires_hidden_path_and_models_are_frozen() -> No ) -@pytest.mark.parametrize( - "text_block", - [ - "", - "ordinary ASCII text", - "Unicode: café 🐍", - "first line\nsecond line", - '{"looks": "like JSON"}', - "Contact alice@example.com or call 555-0100", - ], -) -def test_passthrough_scanner_always_returns_immutable_empty_tuple( - text_block: str, -) -> None: - findings = PassthroughScanner().scan(text_block) +def test_scanner_initialize_runs_after_validated_config_is_available() -> None: + class InitializingScanner(Scanner[ScannerConfig]): + initialized_name: str | None = None + + @override + def _initialize(self) -> None: + self.initialized_name = self.scanner_name - assert findings == () - assert type(findings) is tuple + @override + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: + return () + + scanner = InitializingScanner( + ScannerConfig(name="initialized", entity_types=frozenset()) + ) + + assert scanner.initialized_name == "initialized" + + +@pytest.mark.parametrize("deadline", [True, math.inf, math.nan, "soon"]) +def test_scan_budget_uses_strict_pydantic_validation(deadline: object) -> None: + with pytest.raises(ValidationError): + ScanBudget.model_validate({"deadline": deadline}) diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 1160b2d..cf638ea 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -13,7 +13,7 @@ from privacy_guard.processor import RequestProcessor from privacy_guard.scanners import ScannerConfig from privacy_guard.service import server as server_module -from privacy_guard.service.server import MiddlewareServer, create_server, main, serve +from privacy_guard.service.server import MiddlewareServer, create_server, serve from privacy_guard.service.servicer import PrivacyGuardMiddleware from ..scanner_helpers import DeterministicEmailScanner @@ -54,15 +54,19 @@ async def wait_for_termination(self, timeout: float | None = None) -> bool: raise NotImplementedError +def _middleware() -> PrivacyGuardMiddleware: + scanner = DeterministicEmailScanner( + ScannerConfig(name="test_email", entity_types=frozenset({"email"})) + ) + return PrivacyGuardMiddleware(RequestProcessor([scanner])) + + def test_middleware_server_wires_scanner_and_has_a_default_listen_address( monkeypatch: pytest.MonkeyPatch, ) -> None: served: list[tuple[str, PrivacyGuardMiddleware]] = [] - async def record_serve( - listen: str, servicer: PrivacyGuardMiddleware | None = None - ) -> None: - assert servicer is not None + async def record_serve(servicer: PrivacyGuardMiddleware, listen: str) -> None: served.append((listen, servicer)) monkeypatch.setattr(server_module, "serve", record_serve) @@ -141,7 +145,7 @@ async def Describe( async def test_loopback_accepts_body_at_advertised_limit_and_rejects_larger_body() -> ( None ): - middleware = PrivacyGuardMiddleware() + middleware = _middleware() server = create_server(middleware) port = server.add_insecure_port("127.0.0.1:0") assert port != 0 @@ -255,7 +259,7 @@ async def wait_for_termination(self, timeout: float | None = None) -> bool: ) with pytest.raises(PrivacyGuardError) as exception_info: - await serve("invalid-sensitive-listen-8472") + await serve(_middleware(), "invalid-sensitive-listen-8472") assert exception_info.value.code is ErrorCode.SERVER_BIND_FAILED assert "Hint:" in str(exception_info.value) @@ -288,36 +292,6 @@ async def wait_for_termination(self, timeout: float | None = None) -> bool: ) with pytest.raises(RuntimeError, match="startup failed"): - await serve("127.0.0.1:12345") + await serve(_middleware(), "127.0.0.1:12345") assert fake_server.stopped is True - - -def test_main_parses_custom_and_default_listen_without_permanent_server( - monkeypatch: pytest.MonkeyPatch, -) -> None: - listens: list[str] = [] - - def recording_serve(server: MiddlewareServer, listen: str) -> None: - listens.append(listen) - - monkeypatch.setattr(MiddlewareServer, "serve", recording_serve) - - assert main(["--listen", "127.0.0.1:54321"]) == 0 - assert main([]) == 0 - assert listens == ["127.0.0.1:54321", "127.0.0.1:50051"] - - -def test_main_accepts_a_general_sequence( - monkeypatch: pytest.MonkeyPatch, -) -> None: - received: list[str] = [] - - def recording_serve(server: MiddlewareServer, listen: str) -> None: - received.append(listen) - - monkeypatch.setattr(MiddlewareServer, "serve", recording_serve) - argv: Sequence[str] = ("--listen", "127.0.0.1:50052") - - assert main(argv) == 0 - assert received == ["127.0.0.1:50052"] diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index ce10bf4..ca3d58c 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -25,7 +25,13 @@ ProcessingResult, ) from privacy_guard.processor import RequestProcessor -from privacy_guard.scanners import Finding, RequestBodyFinding, Scanner, ScannerConfig +from privacy_guard.scanners import ( + Finding, + RequestBodyFinding, + ScanBudget, + Scanner, + ScannerConfig, +) from privacy_guard.service.servicer import PrivacyGuardMiddleware from ..scanner_helpers import DeterministicEmailScanner @@ -88,8 +94,15 @@ def _evaluation( ) +def _middleware() -> PrivacyGuardMiddleware: + scanner = DeterministicEmailScanner( + ScannerConfig(name="test_email", entity_types=frozenset({"email"})) + ) + return PrivacyGuardMiddleware(RequestProcessor([scanner])) + + def test_describe_advertises_one_pre_credentials_http_binding() -> None: - manifest = PrivacyGuardMiddleware()._describe() + manifest = _middleware()._describe() assert manifest.name == SERVICE_NAME == "privacy-guard" assert manifest.service_version == SERVICE_VERSION == "0.1.0" @@ -109,9 +122,7 @@ def test_validate_config_accepts_defaults_and_actions( if action is not None: values["on_finding"] = {"action": action.value} - response = PrivacyGuardMiddleware()._validate_config( - pb2.ValidateConfigRequest(config=values) - ) + response = _middleware()._validate_config(pb2.ValidateConfigRequest(config=values)) assert response.valid is True assert response.reason == "" @@ -135,9 +146,7 @@ def test_validate_config_accepts_defaults_and_actions( def test_validate_config_rejects_malformed_values_without_echoing_them( config: Mapping[str, object], ) -> None: - response = PrivacyGuardMiddleware()._validate_config( - pb2.ValidateConfigRequest(config=config) - ) + response = _middleware()._validate_config(pb2.ValidateConfigRequest(config=config)) assert response.valid is False assert "Hint:" in response.reason @@ -202,8 +211,8 @@ def test_validate_config_uses_domain_parser_and_processor_validation() -> None: @pytest.mark.asyncio -async def test_passthrough_and_bodyless_requests_allow_without_replacement() -> None: - servicer = PrivacyGuardMiddleware() +async def test_requests_without_findings_and_bodyless_requests_are_unchanged() -> None: + servicer = _middleware() ordinary = await servicer._evaluate_http_request(_evaluation()) bodyless = await servicer._evaluate_http_request(_evaluation(body=b"")) @@ -373,6 +382,31 @@ async def test_findings_aggregate_in_first_observed_group_order() -> None: assert all(not finding.severity for finding in response.findings) +@pytest.mark.asyncio +async def test_pattern_metadata_is_included_in_aggregate_label() -> None: + finding = RequestBodyFinding( + finding=Finding( + entity="email", + metadata={"pattern_name": "common-email"}, + scanner_name="regex", + start_offset=0, + end_offset=1, + ), + text_block_path="path", + ) + processor = FakeProcessor( + ProcessingResult(decision=ProcessingDecision.ALLOW, findings=(finding,)) + ) + + response = await PrivacyGuardMiddleware(processor)._evaluate_http_request( + _evaluation() + ) + + assert [(item.type, item.label) for item in response.findings] == [ + ("regex", "email/common-email") + ] + + @pytest.mark.asyncio async def test_more_than_32_finding_groups_stably_denies() -> None: findings = tuple( @@ -418,7 +452,7 @@ async def test_invalid_input_maps_to_invalid_argument_with_safe_catalog_details( context = RecordingContext() with pytest.raises(AbortedRpc): - await PrivacyGuardMiddleware()._evaluate_rpc(evaluation, context) + await _middleware()._evaluate_rpc(evaluation, context) assert context.code is grpc.StatusCode.INVALID_ARGUMENT assert context.details is not None @@ -451,7 +485,7 @@ def process(self, request: InterceptedRequest) -> ProcessingResult: async def test_cataloged_scanner_failure_maps_to_internal_status() -> None: class RaisingScanner(Scanner[ScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: raise RuntimeError("sensitive-scanner-failure-8472") context = RecordingContext() @@ -468,7 +502,7 @@ def _scan(self, text_block: str) -> tuple[Finding, ...]: def test_servicer_repr_does_not_retain_request_or_config_content() -> None: - servicer = PrivacyGuardMiddleware() + servicer = _middleware() assert "sensitive-body-8472" not in repr(servicer) assert "sensitive-injection-8472" not in repr(servicer) diff --git a/projects/privacy-guard/tests/test_hardening.py b/projects/privacy-guard/tests/test_hardening.py index 1f0bdbc..79634a2 100644 --- a/projects/privacy-guard/tests/test_hardening.py +++ b/projects/privacy-guard/tests/test_hardening.py @@ -26,8 +26,8 @@ from privacy_guard.scanners import ( Confidence, Finding, - PassthroughScanner, RequestBodyFinding, + ScanBudget, Scanner, ScannerConfig, ) @@ -145,7 +145,13 @@ def test_processor_checks_handler_aggregate_boundaries_contextually( ) -> None: import privacy_guard.processor as processor_module - processor = RequestProcessor([PassthroughScanner()]) + processor = RequestProcessor( + [ + DeterministicEmailScanner( + ScannerConfig(name="test_email", entity_types=frozenset({"email"})) + ) + ] + ) monkeypatch.setattr(processor_module, "MAX_TEXT_BLOCKS", 2) monkeypatch.setattr(processor_module, "MAX_SCANNED_CHARACTERS", 2) assert ( @@ -192,7 +198,7 @@ def test_json_key_findings_are_observed_blocked_and_never_rewritten( def test_finding_criteria_are_owned_by_every_action(action_kind: str) -> None: class MixedScanner(Scanner[ScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: return ( Finding( entity="email", @@ -235,12 +241,12 @@ def test_unknown_entity_filter_is_content_safe_and_multi_scanner_union_is_valid( ): class EmailScanner(Scanner[ScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: return () class TokenScanner(Scanner[ScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: return () processor = RequestProcessor( @@ -288,7 +294,7 @@ def test_multi_scanner_overlap_is_retained_for_observe_and_resolved_for_redact() ): class FirstScanner(Scanner[ScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: return ( Finding( entity="short", @@ -300,7 +306,7 @@ def _scan(self, text_block: str) -> tuple[Finding, ...]: class SecondScanner(Scanner[ScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: return ( Finding( entity="long", @@ -344,7 +350,7 @@ def test_finding_excess_stably_denies_without_returning_partial_findings( class NoisyScanner(Scanner[ScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: return ( Finding( entity="a", @@ -388,7 +394,7 @@ def test_scanner_identity_is_non_empty_and_finding_entities_are_bounded() -> Non def test_request_finding_limit_exact_boundary(block_count: int, allowed: bool) -> None: class DenseScanner(Scanner[ScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: return tuple( Finding( entity="unit", @@ -419,7 +425,7 @@ async def test_slow_scan_does_not_stall_unrelated_rpc() -> None: class SlowScanner(Scanner[ScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: started.set() release.wait(timeout=2) return () @@ -454,7 +460,7 @@ async def test_cancelled_rpc_holds_scan_slot_until_worker_really_finishes() -> N class BlockingScanner(Scanner[ScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: nonlocal entered entered += 1 first_started.set() diff --git a/projects/privacy-guard/tests/test_processor.py b/projects/privacy-guard/tests/test_processor.py index f02cc0d..d2e37f3 100644 --- a/projects/privacy-guard/tests/test_processor.py +++ b/projects/privacy-guard/tests/test_processor.py @@ -1,3 +1,4 @@ +import math from collections.abc import Mapping import pytest @@ -15,8 +16,9 @@ from privacy_guard.request_body import FormatHandler, RequestBody, TextBlock from privacy_guard.scanners import ( Finding, - PassthroughScanner, RequestBodyFinding, + ScanBudget, + ScanBudgetExceeded, Scanner, ScannerConfig, ScannerContractError, @@ -39,7 +41,7 @@ def __init__( self.calls: list[str] = [] @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: self.calls.append(text_block) return self.findings_by_text.get(text_block, ()) @@ -146,7 +148,7 @@ def _request_body_finding( ) -def test_passthrough_scans_each_text_block_and_reconstructs_once() -> None: +def test_scanner_visits_each_text_block_and_reconstructs_once() -> None: scanner = RecordingScanner() handler = RecordingHandler( ( @@ -171,6 +173,58 @@ def test_empty_scanner_sequence_is_rejected_at_construction() -> None: assert exception_info.value.code is ErrorCode.SCANNER_OUTPUT_INVALID +@pytest.mark.parametrize( + "timeout", + [True, False, 0, -1, math.inf, math.nan, 31], +) +def test_processor_rejects_invalid_scan_timeout(timeout: float) -> None: + with pytest.raises(ValueError): + RequestProcessor([RecordingScanner()], scan_timeout_seconds=timeout) + + +def test_one_scan_budget_is_shared_across_blocks_and_exhaustion_discards_findings() -> ( + None +): + class BudgetScanner(Scanner[ScannerConfig]): + def __init__(self) -> None: + super().__init__( + ScannerConfig(name="budget", entity_types=frozenset({"secret"})) + ) + self.budgets: list[ScanBudget] = [] + + @override + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: + self.budgets.append(budget) + if text_block == "second": + raise ScanBudgetExceeded + return ( + Finding( + entity="secret", + scanner_name=self.scanner_name, + start_offset=0, + end_offset=1, + ), + ) + + scanner = BudgetScanner() + handler = RecordingHandler( + ( + TextBlock(path="first", text="first"), + TextBlock(path="second", text="second"), + ) + ) + + result = _processor(scanner, handler).process( + _request(policy_config=_config(action_kind=PolicyAction.OBSERVE.value)) + ) + + assert result.decision is ProcessingDecision.DENY + assert result.reason_code == "privacy_guard_limit_exceeded" + assert result.findings == () + assert len(scanner.budgets) == 2 + assert scanner.budgets[0] is scanner.budgets[1] + + def test_duplicate_scanner_names_are_rejected_at_construction() -> None: with pytest.raises(PrivacyGuardError) as exception_info: RequestProcessor([RecordingScanner(), RecordingScanner()]) @@ -184,7 +238,7 @@ class PrefixScannerConfig(ScannerConfig): class PrefixScanner(Scanner[PrefixScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: return ( Finding( entity=self.config.finding_entity, @@ -214,11 +268,8 @@ def _scan(self, text_block: str) -> tuple[Finding, ...]: ) -@pytest.mark.parametrize("scanner", [RecordingScanner(), PassthroughScanner()]) -def test_empty_supported_entity_catalog_rejects_enabled_filter( - scanner: Scanner[ScannerConfig], -) -> None: - processor = RequestProcessor([scanner]) +def test_empty_supported_entity_catalog_rejects_enabled_filter() -> None: + processor = RequestProcessor([RecordingScanner()]) with pytest.raises(PrivacyGuardError) as exception_info: processor.validate_policy_config( @@ -232,7 +283,7 @@ def test_empty_supported_entity_catalog_rejects_enabled_filter( def test_handler_registry_key_must_match_immutable_identity() -> None: with pytest.raises(PrivacyGuardError) as exception_info: - RequestProcessor([PassthroughScanner()], {"wrong": RecordingHandler()}) + RequestProcessor([RecordingScanner()], {"wrong": RecordingHandler()}) assert exception_info.value.code is ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID @@ -244,7 +295,7 @@ def test_malformed_normalize_output_maps_to_handler_contract_error( monkeypatch.setattr(handler, "_normalize", lambda raw_body, policy: object()) with pytest.raises(PrivacyGuardError) as exception_info: - _processor(PassthroughScanner(), handler).process(_request()) + _processor(RecordingScanner(), handler).process(_request()) assert exception_info.value.code is ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID @@ -260,14 +311,14 @@ def test_non_bytes_reconstruct_output_maps_to_handler_contract_error( ) with pytest.raises(PrivacyGuardError) as exception_info: - _processor(PassthroughScanner(), handler).process(_request()) + _processor(RecordingScanner(), handler).process(_request()) assert exception_info.value.code is ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID def test_validate_policy_config_selects_and_validates_without_processing() -> None: handler = RecordingHandler() - processor = _processor(PassthroughScanner(), handler) + processor = _processor(RecordingScanner(), handler) processor.validate_policy_config(_config()) @@ -421,7 +472,7 @@ def test_block_denies_with_findings_and_suppresses_reconstruction() -> None: def test_block_allows_and_reconstructs_when_no_findings_exist() -> None: handler = RecordingHandler((TextBlock(path="text_block::alpha", text="safe"),)) - result = _processor(PassthroughScanner(), handler).process( + result = _processor(RecordingScanner(), handler).process( _request(policy_config=_config(action_kind=PolicyAction.BLOCK.value)) ) @@ -579,7 +630,7 @@ def test_contextually_invalid_finding_is_rejected(finding: Finding) -> None: def test_finding_entity_must_be_declared_by_scanner_config() -> None: class UndeclaredEntityScanner(Scanner[ScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: return ( _finding(0, 1, entity="undeclared", scanner_name=self.scanner_name), ) @@ -621,7 +672,7 @@ def test_collaborator_exceptions_are_replaced_without_partial_result_or_content( class RaisingScanner(Scanner[ScannerConfig]): @override - def _scan(self, text_block: str) -> tuple[Finding, ...]: + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: raise RuntimeError(sentinel) class RaisingHandler(RecordingHandler): @@ -634,7 +685,7 @@ def _normalize( scanner: Scanner[ScannerConfig] = ( RaisingScanner(ScannerConfig(name="raising", entity_types=frozenset())) if collaborator == "scanner" - else PassthroughScanner() + else RecordingScanner() ) handler: FormatHandler = ( RaisingHandler() @@ -662,7 +713,7 @@ def _normalize( raise expected with pytest.raises(PrivacyGuardError) as exception_info: - _processor(PassthroughScanner(), CatalogRaisingHandler()).process(_request()) + _processor(RecordingScanner(), CatalogRaisingHandler()).process(_request()) assert exception_info.value is expected @@ -672,7 +723,7 @@ def test_reconstruction_equal_to_original_returns_no_replacement() -> None: (TextBlock(path="text_block::alpha", text="text"),), reconstructed=b"body" ) - result = _processor(PassthroughScanner(), handler).process(_request(b"body")) + result = _processor(RecordingScanner(), handler).process(_request(b"body")) assert result.replacement_body is None diff --git a/projects/privacy-guard/tests/test_single_block_composition.py b/projects/privacy-guard/tests/test_single_block_composition.py index fdb5063..3f75d59 100644 --- a/projects/privacy-guard/tests/test_single_block_composition.py +++ b/projects/privacy-guard/tests/test_single_block_composition.py @@ -4,7 +4,6 @@ from privacy_guard.config import PolicyConfig from privacy_guard.request_body import JsonHandler -from privacy_guard.scanners import PassthroughScanner @pytest.mark.parametrize( @@ -28,7 +27,7 @@ ), ], ) -def test_one_selected_text_block_can_be_scanned_and_explicitly_replaced( +def test_one_selected_text_block_can_be_explicitly_replaced( raw_body: bytes, selected_text_block_path: str, expected_value: object ) -> None: json_handler = JsonHandler() @@ -39,9 +38,6 @@ def test_one_selected_text_block_can_be_scanned_and_explicitly_replaced( if text_block.path == selected_text_block_path ) - findings = PassthroughScanner().scan(selected_text_block.text) - - assert findings == () test_suffix = " [test suffix]" replacement_text = selected_text_block.text + test_suffix reconstructed_body = json_handler.reconstruct( diff --git a/projects/privacy-guard/uv.lock b/projects/privacy-guard/uv.lock index 49f6485..78c72b3 100644 --- a/projects/privacy-guard/uv.lock +++ b/projects/privacy-guard/uv.lock @@ -106,6 +106,7 @@ dependencies = [ { name = "grpcio" }, { name = "protobuf" }, { name = "pydantic" }, + { name = "pyyaml" }, { name = "typing-extensions" }, ] @@ -122,6 +123,7 @@ requires-dist = [ { name = "grpcio", specifier = ">=1.81.1,<2" }, { name = "protobuf", specifier = ">=6.33.5,<7" }, { name = "pydantic", specifier = ">=2.11,<3" }, + { name = "pyyaml", specifier = ">=6.0.2,<7" }, { name = "typing-extensions", specifier = ">=4.12,<5" }, ] @@ -303,6 +305,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + [[package]] name = "ruff" version = "0.12.12" From 61330d812415e00f6fb355877392fb347bf619ac Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Wed, 22 Jul 2026 22:23:43 +0000 Subject: [PATCH 04/82] Harden regex scanner configuration loading --- projects/privacy-guard/REGEX_SCANNER_PLAN.md | 37 +++++++-------- .../src/privacy_guard/scanners/regex.py | 10 +++- .../tests/scanners/test_regex.py | 47 +++++++++++++++++++ 3 files changed, 73 insertions(+), 21 deletions(-) diff --git a/projects/privacy-guard/REGEX_SCANNER_PLAN.md b/projects/privacy-guard/REGEX_SCANNER_PLAN.md index 186988a..440d7e2 100644 --- a/projects/privacy-guard/REGEX_SCANNER_PLAN.md +++ b/projects/privacy-guard/REGEX_SCANNER_PLAN.md @@ -135,9 +135,9 @@ they are not blamed on the request that happened to expose the pattern defect. structural, and model failures into one content-safe scanner-configuration error that never includes paths, patterns, or matched text. Add a dedicated `SCANNER_CONFIG_INVALID` catalog code rather than reusing policy - `CONFIG_INVALID`. Add the selected safe-YAML parser and timeout-capable regex - engine as bounded project dependencies in `pyproject.toml` and `uv.lock`; - do not build either boundary on undocumented engine behavior. + `CONFIG_INVALID`. Add the selected safe-YAML parser as a bounded project + dependency in `pyproject.toml` and `uv.lock`; use the standard-library regex + engine and documented behavior. 2. Add `RegexScannerConfig`, derived from `ScannerConfig`, and `RegexScanner`. Compile each configured expression as `(?:expression)(?P)`. The trailing empty named marker @@ -163,13 +163,11 @@ they are not blamed on the request that happened to expose the pattern defect. default budget for standalone calls when none is supplied. The protected scanner hook receives the effective non-optional budget; simple scanners may ignore it, while scanners performing potentially unbounded work must - cooperate with it. The budget is therefore an automatic hard bound for - `RegexScanner`, but only a cooperative contract for custom scanners. Before - each pattern evaluation, - `RegexScanner` will pass the smaller of the remaining request budget and the - per-pattern ceiling to a timeout-capable regex engine. Check the budget while - consuming the engine's iterator, not only when constructing it. Budget - exhaustion or an engine timeout will raise one exported, content-safe + cooperate with it. The budget is a cooperative contract for `RegexScanner` + and custom scanners because the standard-library regex engine cannot + interrupt an active evaluation. `RegexScanner` checks the budget before and + after each evaluation and while consuming matches. Budget exhaustion raises + one exported, content-safe `ScanBudgetExceeded` exception from `Scanner.scan`, with no partial findings. `RequestProcessor` will catch only that typed condition, discard findings accumulated from earlier scanners or blocks, and produce the existing stable @@ -185,11 +183,11 @@ they are not blamed on the request that happened to expose the pattern defect. benchmarks. The scalability gate is 1,000 active entities and 10,000 active patterns on documented representative inputs and reference hardware. Record input size and match density with each result so the claim is reproducible. - The request-scoped budget remains the runtime CPU bound regardless of - catalog size. If separately evaluating 10,000 compiled patterns cannot meet - the default budget, introduce bounded batching, indexing, or a multi-pattern - engine before claiming that scale; preserve overlapping results and original - pattern identity in any optimization. If batching combines patterns, assign + The request-scoped budget bounds work between regex evaluations regardless + of catalog size. If separately evaluating 10,000 compiled patterns cannot + meet the default budget, introduce bounded batching, indexing, or a + multi-pattern engine before claiming that scale; preserve overlapping + results and original pattern identity in any optimization. If batching combines patterns, assign collision-free internal group identifiers even when configured names normalize to the same value. Implement conservative provisional file and count ceilings first, then finalize and document them only after this gate @@ -241,10 +239,11 @@ they are not blamed on the request that happened to expose the pattern defect. trailing named markers, numeric backreferences, reserved named-group and inline-flag rejection, load-time and contextual zero-width rejection, deterministic ordering, and finding limits. -- Test per-pattern and request-wide timeout behavior with adversarial - expressions, many patterns, many text blocks, and long inputs. Verify timeout - paths raise `ScanBudgetExceeded` for standalone calls and return a stable deny - through `RequestProcessor`, with no partial findings in either case. +- Test per-pattern and request-wide budget behavior with many patterns, many + text blocks, and long inputs. Verify exhaustion paths raise + `ScanBudgetExceeded` for standalone calls and return a stable deny through + `RequestProcessor`, with no partial findings in either case. Document that an + active standard-library regex evaluation cannot be interrupted. - Test that `RequestProcessor` and standalone `Scanner.scan` calls use safe default budgets when callers provide no timeout or budget, and that an explicit processor timeout applies one shared deadline across the request. diff --git a/projects/privacy-guard/src/privacy_guard/scanners/regex.py b/projects/privacy-guard/src/privacy_guard/scanners/regex.py index 0fd34c4..6867e32 100644 --- a/projects/privacy-guard/src/privacy_guard/scanners/regex.py +++ b/projects/privacy-guard/src/privacy_guard/scanners/regex.py @@ -146,7 +146,7 @@ def _initialize(self) -> None: for entity in self.config.entities for pattern in entity.patterns ) - except (ValueError, re.error): + except (RecursionError, ValueError, re.error): raise PrivacyGuardError(ErrorCode.SCANNER_CONFIG_INVALID) from None @classmethod @@ -159,7 +159,7 @@ def from_yaml( ) -> Self: """Load, validate, select, and compile a complete YAML catalog.""" try: - raw = Path(path).read_bytes() + raw = _read_bounded_file(path) value = _load_yaml(raw) catalogs = _parse_catalogs(value) for entities in catalogs.values(): @@ -177,6 +177,7 @@ def from_yaml( raise except ( OSError, + RecursionError, UnicodeError, ValueError, TypeError, @@ -242,6 +243,11 @@ def _validate_name(value: str) -> str: return value +def _read_bounded_file(path: str | Path) -> bytes: + with Path(path).open("rb") as stream: + return stream.read(MAX_SCANNER_CONFIG_BYTES + 1) + + def _compile_rule(entity_name: str, pattern: RegexPattern) -> _CompiledRule: flags = 0 if pattern.ignore_case: diff --git a/projects/privacy-guard/tests/scanners/test_regex.py b/projects/privacy-guard/tests/scanners/test_regex.py index b83783f..419b4f0 100644 --- a/projects/privacy-guard/tests/scanners/test_regex.py +++ b/projects/privacy-guard/tests/scanners/test_regex.py @@ -1,11 +1,14 @@ from __future__ import annotations from concurrent.futures import ThreadPoolExecutor +from io import BytesIO from pathlib import Path from time import monotonic import pytest +from typing_extensions import override +import privacy_guard.scanners.regex as regex_module from privacy_guard.errors import ErrorCode, ErrorKind, PrivacyGuardError from privacy_guard.scanners import RegexScanner, ScanBudget, ScanBudgetExceeded @@ -156,6 +159,30 @@ def test_invalid_yaml_and_patterns_use_one_content_safe_error( assert "8472" not in str(exception_info.value) +def test_configuration_read_is_bounded_before_parsing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class RecordingBytesIO(BytesIO): + def __init__(self) -> None: + super().__init__(b"x" * 100) + self.read_sizes: list[int | None] = [] + + @override + def read(self, size: int | None = -1, /) -> bytes: + self.read_sizes.append(size) + return super().read(size) + + stream = RecordingBytesIO() + monkeypatch.setattr(regex_module, "MAX_SCANNER_CONFIG_BYTES", 64) + monkeypatch.setattr(Path, "open", lambda path, mode: stream) + + with pytest.raises(PrivacyGuardError) as exception_info: + RegexScanner.from_yaml("ignored.yaml") + + assert exception_info.value.code is ErrorCode.SCANNER_CONFIG_INVALID + assert stream.read_sizes == [65] + + def test_inactive_profiles_are_fully_validated(tmp_path: Path) -> None: path = _write( tmp_path / "profiles.yaml", @@ -190,6 +217,26 @@ def test_excessive_yaml_nesting_is_rejected_safely(tmp_path: Path) -> None: assert exception_info.value.kind is ErrorKind.INVALID_INPUT +def test_deeply_nested_regex_is_a_content_safe_configuration_error( + tmp_path: Path, +) -> None: + nested_expression = "(" * 1_000 + "x" + ")" * 1_000 + path = _write( + tmp_path / "deep-regex.yaml", + _single( + " - name: nested\n" + f" regex: '{nested_expression}'\n" + " confidence: high\n" + ), + ) + + with pytest.raises(PrivacyGuardError) as exception_info: + RegexScanner.from_yaml(path) + + assert exception_info.value.code is ErrorCode.SCANNER_CONFIG_INVALID + assert exception_info.value.kind is ErrorKind.INVALID_INPUT + + def test_contextual_zero_width_is_a_runtime_configuration_failure( tmp_path: Path, ) -> None: From fe8e84a11ab96a19f32b85178d1c5781e9d3a29e Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Wed, 22 Jul 2026 22:28:08 +0000 Subject: [PATCH 05/82] Improve Privacy Guard operator guidance --- projects/privacy-guard/README.md | 6 ++++- .../examples/email-scanner/README.md | 5 ++++ .../src/privacy_guard/service/server.py | 15 +++++++----- .../tests/service/test_server.py | 23 ++++++++++++++++++- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 224adf9..ab77c19 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -152,7 +152,11 @@ and 4 MiB of scanned characters. Scanning is capped at 256 findings per block, 4,096 per request, four active scanner workers, and 16 concurrent gRPC calls. One default one-second monotonic scan budget is shared across every block and scanner in a request. `RegexScanner` checks it before and after each expression -evaluation and while consuming overlapping matches. +evaluation and while consuming overlapping matches. Because the standard-library +regex engine cannot interrupt an active evaluation, one backtracking-heavy +expression may outlast the deadline. Test catalogs against representative +worst-case inputs before deployment and avoid expressions with pathological +backtracking behavior. Shape excess is invalid input. Finding or outbound representation excess returns a stable `privacy_guard_limit_exceeded` deny with no body or partial findings, avoiding a failure-mode-dependent fail open. diff --git a/projects/privacy-guard/examples/email-scanner/README.md b/projects/privacy-guard/examples/email-scanner/README.md index 92deb9b..70af73e 100644 --- a/projects/privacy-guard/examples/email-scanner/README.md +++ b/projects/privacy-guard/examples/email-scanner/README.md @@ -49,6 +49,11 @@ Only the checked-out `gateway.toml` is edited. Do not copy it into ## 2. Start Privacy Guard +This development server uses unauthenticated plaintext gRPC and receives request +bodies that may contain sensitive content. Restrict it to a trusted network and +firewall the port. When possible, bind `--listen` to the specific host interface +that Docker must reach instead of every interface. + In terminal 1: ```bash diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 2dca880..a5f6add 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -71,12 +71,15 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--scanner-name", default="regex") parser.add_argument("--listen", default="127.0.0.1:50051") arguments = parser.parse_args(argv) - scanner = RegexScanner.from_yaml( - arguments.scanner_config, - arguments.profile, - scanner_name=arguments.scanner_name, - ) - MiddlewareServer(scanner=scanner).serve(arguments.listen) + try: + scanner = RegexScanner.from_yaml( + arguments.scanner_config, + arguments.profile, + scanner_name=arguments.scanner_name, + ) + MiddlewareServer(scanner=scanner).serve(arguments.listen) + except PrivacyGuardError as error: + parser.exit(status=1, message=f"{error}\n") return 0 diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index cf638ea..3737466 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -13,7 +13,7 @@ from privacy_guard.processor import RequestProcessor from privacy_guard.scanners import ScannerConfig from privacy_guard.service import server as server_module -from privacy_guard.service.server import MiddlewareServer, create_server, serve +from privacy_guard.service.server import MiddlewareServer, create_server, main, serve from privacy_guard.service.servicer import PrivacyGuardMiddleware from ..scanner_helpers import DeterministicEmailScanner @@ -81,6 +81,27 @@ async def record_serve(servicer: PrivacyGuardMiddleware, listen: str) -> None: assert isinstance(served[0][1], PrivacyGuardMiddleware) +def test_cli_reports_expected_configuration_failure_without_traceback( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + expected = PrivacyGuardError(ErrorCode.SCANNER_CONFIG_INVALID) + + def reject_configuration(*args: object, **kwargs: object) -> None: + raise expected + + monkeypatch.setattr(server_module.RegexScanner, "from_yaml", reject_configuration) + + with pytest.raises(SystemExit) as exception_info: + main(["--scanner-config", "sensitive-path-8472.yaml"]) + + assert exception_info.value.code == 1 + stderr = capsys.readouterr().err + assert stderr == f"{expected}\n" + assert "Traceback" not in stderr + assert "8472" not in stderr + + @pytest.mark.asyncio async def test_create_server_accepts_injected_servicer_and_serves_loopback_rpcs() -> ( None From c1a9a1d992460c622215c18b8b03fdf8223c6497 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Wed, 22 Jul 2026 22:32:03 +0000 Subject: [PATCH 06/82] Clarify regex scanner CLI configuration --- .../src/privacy_guard/scanners/regex.py | 1 + .../src/privacy_guard/service/server.py | 8 ++++- .../tests/service/test_server.py | 29 +++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/projects/privacy-guard/src/privacy_guard/scanners/regex.py b/projects/privacy-guard/src/privacy_guard/scanners/regex.py index 6867e32..d426165 100644 --- a/projects/privacy-guard/src/privacy_guard/scanners/regex.py +++ b/projects/privacy-guard/src/privacy_guard/scanners/regex.py @@ -244,6 +244,7 @@ def _validate_name(value: str) -> str: def _read_bounded_file(path: str | Path) -> bytes: + """Bound allocation before handing configuration bytes to PyYAML.""" with Path(path).open("rb") as stream: return stream.read(MAX_SCANNER_CONFIG_BYTES + 1) diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index a5f6add..3fad062 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -67,7 +67,13 @@ def main(argv: Sequence[str] | None = None) -> int: """Load the required regex catalog, then run the middleware server.""" parser = argparse.ArgumentParser(description="Run the Privacy Guard middleware") parser.add_argument("--scanner-config", required=True) - parser.add_argument("--profile") + parser.add_argument( + "--profile", + help=( + "Regex catalog profile; required only when --scanner-config contains " + "profiles" + ), + ) parser.add_argument("--scanner-name", default="regex") parser.add_argument("--listen", default="127.0.0.1:50051") arguments = parser.parse_args(argv) diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 3737466..2306624 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -102,6 +102,35 @@ def reject_configuration(*args: object, **kwargs: object) -> None: assert "8472" not in stderr +@pytest.mark.parametrize("profile", [None, "customer-support"]) +def test_cli_profile_is_optional_and_forwarded_only_when_supplied( + monkeypatch: pytest.MonkeyPatch, + profile: str | None, +) -> None: + calls: list[tuple[str, str | None, str]] = [] + scanner = DeterministicEmailScanner( + ScannerConfig(name="regex", entity_types=frozenset({"email"})) + ) + + def load_scanner( + path: str, + selected_profile: str | None, + *, + scanner_name: str, + ) -> DeterministicEmailScanner: + calls.append((path, selected_profile, scanner_name)) + return scanner + + monkeypatch.setattr(server_module.RegexScanner, "from_yaml", load_scanner) + monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) + arguments = ["--scanner-config", "entities.yaml"] + if profile is not None: + arguments.extend(("--profile", profile)) + + assert main(arguments) == 0 + assert calls == [("entities.yaml", profile, "regex")] + + @pytest.mark.asyncio async def test_create_server_accepts_injected_servicer_and_serves_loopback_rpcs() -> ( None From e8d0abf64d9ca490bcdb875ccd10f478be539b51 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Wed, 22 Jul 2026 22:36:47 +0000 Subject: [PATCH 07/82] Separate scanner-specific CLI options --- projects/privacy-guard/README.md | 15 ++++-- .../src/privacy_guard/service/server.py | 46 ++++++++++++---- .../tests/service/test_server.py | 52 +++++++++++++++++-- 3 files changed, 94 insertions(+), 19 deletions(-) diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index ab77c19..4df7cdc 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -87,11 +87,16 @@ winners deterministically by confidence, span length, offsets, scanner identity, and entity. A scanner sequence is passed to `RequestProcessor`; scanner names must be unique and remain visible in aggregated findings. -`RegexScanner` is the packaged command's default implementation. Its entity -catalog remains explicit: the command refuses to start without -`--scanner-config`. Single-profile files contain a non-empty entity list; -multi-profile files contain only a non-empty `profiles` mapping and require -`--profile`. See [examples/regex-configs](examples/regex-configs) for both forms. +`RegexScanner` is the packaged command's default implementation. Every scanner +requires `--scanner-config` on the standard command surface; the active scanner +owns that file's schema and interpretation. For `RegexScanner`, the path selects +its YAML catalog. Single-profile files contain a non-empty entity list; +multi-profile files contain only a non-empty `profiles` mapping and require the +scanner-specific `--profile` option after `--`. The separator is unnecessary +when no scanner-specific options are supplied. Use `privacy-guard --help` for +standard options and `privacy-guard --scanner-config PATH -- --help` for regex +scanner options. See [examples/regex-configs](examples/regex-configs) for both +configuration forms. ```bash uv run privacy-guard \ diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 3fad062..f5f108f 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -9,6 +9,7 @@ import argparse import asyncio +import sys from collections.abc import Sequence import grpc @@ -64,24 +65,24 @@ async def serve( def main(argv: Sequence[str] | None = None) -> int: - """Load the required regex catalog, then run the middleware server.""" + """Load the default scanner from its required config, then run the server.""" + command_arguments = tuple(sys.argv[1:] if argv is None else argv) + server_arguments, scanner_arguments = _split_scanner_arguments(command_arguments) parser = argparse.ArgumentParser(description="Run the Privacy Guard middleware") - parser.add_argument("--scanner-config", required=True) parser.add_argument( - "--profile", - help=( - "Regex catalog profile; required only when --scanner-config contains " - "profiles" - ), + "--scanner-config", + required=True, + help="Path to the active scanner's configuration", ) - parser.add_argument("--scanner-name", default="regex") parser.add_argument("--listen", default="127.0.0.1:50051") - arguments = parser.parse_args(argv) + arguments = parser.parse_args(server_arguments) + scanner_parser = _create_regex_scanner_parser(parser.prog) + scanner_options = scanner_parser.parse_args(scanner_arguments or ()) try: scanner = RegexScanner.from_yaml( arguments.scanner_config, - arguments.profile, - scanner_name=arguments.scanner_name, + scanner_options.profile, + scanner_name=scanner_options.scanner_name, ) MiddlewareServer(scanner=scanner).serve(arguments.listen) except PrivacyGuardError as error: @@ -89,5 +90,28 @@ def main(argv: Sequence[str] | None = None) -> int: return 0 +def _split_scanner_arguments( + arguments: tuple[str, ...], +) -> tuple[tuple[str, ...], tuple[str, ...] | None]: + try: + separator_index = arguments.index("--") + except ValueError: + return arguments, None + return arguments[:separator_index], arguments[separator_index + 1 :] + + +def _create_regex_scanner_parser(command_name: str) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog=f"{command_name} --", + description="Configure the default RegexScanner", + ) + parser.add_argument( + "--profile", + help="Select a profile from the RegexScanner configuration", + ) + parser.add_argument("--scanner-name", default="regex") + return parser + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 2306624..98cbc12 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -123,12 +123,58 @@ def load_scanner( monkeypatch.setattr(server_module.RegexScanner, "from_yaml", load_scanner) monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) - arguments = ["--scanner-config", "entities.yaml"] + arguments = ["--scanner-config", "scanner-config.yaml"] if profile is not None: - arguments.extend(("--profile", profile)) + arguments.extend(("--", "--profile", profile)) assert main(arguments) == 0 - assert calls == [("entities.yaml", profile, "regex")] + assert calls == [("scanner-config.yaml", profile, "regex")] + + +def test_cli_always_requires_scanner_configuration( + capsys: pytest.CaptureFixture[str], +) -> None: + with pytest.raises(SystemExit) as exception_info: + main(["--listen", "127.0.0.1:50051"]) + + assert exception_info.value.code == 2 + assert "--scanner-config" in capsys.readouterr().err + + +def test_cli_exposes_separate_server_and_scanner_help( + capsys: pytest.CaptureFixture[str], +) -> None: + with pytest.raises(SystemExit) as server_help: + main(["--help"]) + assert server_help.value.code == 0 + server_output = capsys.readouterr().out + assert "--listen" in server_output + assert "--scanner-config" in server_output + + with pytest.raises(SystemExit) as scanner_help: + main(["--scanner-config", "scanner-config.yaml", "--", "--help"]) + assert scanner_help.value.code == 0 + scanner_output = capsys.readouterr().out + assert "--profile" in scanner_output + assert "--scanner-config" not in scanner_output + assert "--listen" not in scanner_output + + +def test_cli_rejects_scanner_options_before_separator( + capsys: pytest.CaptureFixture[str], +) -> None: + with pytest.raises(SystemExit) as exception_info: + main( + [ + "--scanner-config", + "scanner-config.yaml", + "--profile", + "customer-support", + ] + ) + + assert exception_info.value.code == 2 + assert "unrecognized arguments: --profile" in capsys.readouterr().err @pytest.mark.asyncio From b4a4a38d3f52192f85c5440f2ab5869a2100da12 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Wed, 22 Jul 2026 22:38:48 +0000 Subject: [PATCH 08/82] Address scanner CLI review feedback --- projects/privacy-guard/REGEX_SCANNER_PLAN.md | 19 +++++----- .../examples/regex-configs/README.md | 11 ++++-- .../src/privacy_guard/service/server.py | 10 ++++- .../tests/service/test_server.py | 37 +++++++++++++++++++ 4 files changed, 63 insertions(+), 14 deletions(-) diff --git a/projects/privacy-guard/REGEX_SCANNER_PLAN.md b/projects/privacy-guard/REGEX_SCANNER_PLAN.md index 440d7e2..91ab2ac 100644 --- a/projects/privacy-guard/REGEX_SCANNER_PLAN.md +++ b/projects/privacy-guard/REGEX_SCANNER_PLAN.md @@ -68,10 +68,10 @@ profiles: ignore_case: true ``` -The command will require `--profile NAME` when the file contains `profiles`. It -will reject `--profile` for a single-profile file, avoiding an ignored or -ambiguous option. Exactly one selected entity catalog is active; profiles are -not merged implicitly. +The command will require `--profile NAME` after the scanner-option separator +when the file contains `profiles`. It will reject `--profile` for a +single-profile file, avoiding an ignored or ambiguous option. Exactly one +selected entity catalog is active; profiles are not merged implicitly. Scanner identity is runtime configuration rather than rule-set content. The default command will use the stable name `regex`, with an explicit @@ -213,11 +213,12 @@ they are not blamed on the request that happened to expose the pattern defect. separate OpenShell protocol change before increasing Privacy Guard's limit; the scanner implementation must not emit a response its consumer rejects. 7. Add a loader such as `RegexScanner.from_yaml(path, profile=None)`. Update the - packaged command to accept one required `--scanner-config PATH`, a - conditionally required `--profile NAME`, and an optional `--scanner-name`. - Load and compile the configuration before binding a listening socket. Restore - the `privacy-guard` script only in the same change, so the command cannot run - with an implicit allow-all scanner between implementations. + packaged command to accept one required, scanner-agnostic + `--scanner-config PATH`. Put the conditionally required `--profile NAME` and + optional `--scanner-name` after the scanner-option separator. Load and compile the + configuration before binding a listening socket. Restore the `privacy-guard` + script only in the same change, so the command cannot run with an implicit + allow-all scanner between implementations. 8. Add example rule sets under `examples/regex-configs/`: single-profile `customer.yaml` and `hipaa.yaml` files plus a `profiles.yaml` file showing both catalogs in the multi-profile shape. The HIPAA example must be diff --git a/projects/privacy-guard/examples/regex-configs/README.md b/projects/privacy-guard/examples/regex-configs/README.md index 03deabf..5b5cbb4 100644 --- a/projects/privacy-guard/examples/regex-configs/README.md +++ b/projects/privacy-guard/examples/regex-configs/README.md @@ -1,9 +1,14 @@ # Regex scanner configurations `customer.yaml` and `hipaa.yaml` demonstrate the single-profile entity-list -shape. `profiles.yaml` combines both catalogs under explicit profile names; run -Privacy Guard with `--profile customer-support` or `--profile hipaa` when using -that file. +shape. `profiles.yaml` combines both catalogs under explicit profile names. Its +profile is a RegexScanner option, so select one after the scanner-option +separator, for example: + +```bash +privacy-guard --scanner-config profiles.yaml -- --profile customer-support +privacy-guard --scanner-config profiles.yaml -- --profile hipaa +``` The HIPAA catalog is only a starting rule set. It is not a claim of compliance. Deployers remain responsible for validating detection behavior and implementing diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index f5f108f..a28cd8e 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -68,7 +68,13 @@ def main(argv: Sequence[str] | None = None) -> int: """Load the default scanner from its required config, then run the server.""" command_arguments = tuple(sys.argv[1:] if argv is None else argv) server_arguments, scanner_arguments = _split_scanner_arguments(command_arguments) - parser = argparse.ArgumentParser(description="Run the Privacy Guard middleware") + parser = argparse.ArgumentParser( + description="Run the Privacy Guard middleware", + epilog=( + "Scanner-specific options follow --. Run '%(prog)s --scanner-config " + "PATH -- --help' to list them." + ), + ) parser.add_argument( "--scanner-config", required=True, @@ -107,7 +113,7 @@ def _create_regex_scanner_parser(command_name: str) -> argparse.ArgumentParser: ) parser.add_argument( "--profile", - help="Select a profile from the RegexScanner configuration", + help="Profile required for a multi-profile RegexScanner configuration", ) parser.add_argument("--scanner-name", default="regex") return parser diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 98cbc12..6c04109 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -131,6 +131,41 @@ def load_scanner( assert calls == [("scanner-config.yaml", profile, "regex")] +def test_cli_forwards_custom_scanner_name_after_separator( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str | None, str]] = [] + scanner = DeterministicEmailScanner( + ScannerConfig(name="custom-regex", entity_types=frozenset({"email"})) + ) + + def load_scanner( + path: str, + selected_profile: str | None, + *, + scanner_name: str, + ) -> DeterministicEmailScanner: + calls.append((path, selected_profile, scanner_name)) + return scanner + + monkeypatch.setattr(server_module.RegexScanner, "from_yaml", load_scanner) + monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) + + assert ( + main( + [ + "--scanner-config", + "scanner-config.yaml", + "--", + "--scanner-name", + "custom-regex", + ] + ) + == 0 + ) + assert calls == [("scanner-config.yaml", None, "custom-regex")] + + def test_cli_always_requires_scanner_configuration( capsys: pytest.CaptureFixture[str], ) -> None: @@ -150,6 +185,8 @@ def test_cli_exposes_separate_server_and_scanner_help( server_output = capsys.readouterr().out assert "--listen" in server_output assert "--scanner-config" in server_output + assert "Scanner-specific options follow --" in server_output + assert "--scanner-config PATH" in server_output with pytest.raises(SystemExit) as scanner_help: main(["--scanner-config", "scanner-config.yaml", "--", "--help"]) From d7e251083c18a0876d34d6086e6b2db03038b5ce Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Wed, 22 Jul 2026 22:45:06 +0000 Subject: [PATCH 09/82] Adopt Typer for Privacy Guard CLI --- projects/privacy-guard/README.md | 22 +-- projects/privacy-guard/REGEX_SCANNER_PLAN.md | 19 +-- .../examples/regex-configs/README.md | 8 +- projects/privacy-guard/pyproject.toml | 3 +- .../src/privacy_guard/service/server.py | 104 ++++++------- .../tests/service/test_server.py | 143 +++++++++--------- projects/privacy-guard/uv.lock | 69 +++++++++ 7 files changed, 221 insertions(+), 147 deletions(-) diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 4df7cdc..14aeb36 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -87,21 +87,21 @@ winners deterministically by confidence, span length, offsets, scanner identity, and entity. A scanner sequence is passed to `RequestProcessor`; scanner names must be unique and remain visible in aggregated findings. -`RegexScanner` is the packaged command's default implementation. Every scanner -requires `--scanner-config` on the standard command surface; the active scanner -owns that file's schema and interpretation. For `RegexScanner`, the path selects -its YAML catalog. Single-profile files contain a non-empty entity list; -multi-profile files contain only a non-empty `profiles` mapping and require the -scanner-specific `--profile` option after `--`. The separator is unnecessary -when no scanner-specific options are supplied. Use `privacy-guard --help` for -standard options and `privacy-guard --scanner-config PATH -- --help` for regex -scanner options. See [examples/regex-configs](examples/regex-configs) for both -configuration forms. +The `privacy-guard` command runs built-in scanners. Every built-in requires +`--scanner-config` on the shared command surface and owns that file's schema and +interpretation. `RegexScanner` is available through the `regex` subcommand; its +config path selects a YAML catalog. Single-profile files contain a non-empty +entity list. Multi-profile files contain only a non-empty `profiles` mapping and +require the regex-specific `--profile` option. Use `privacy-guard --help` for +shared options and built-in scanners, or +`privacy-guard --scanner-config PATH regex --help` for regex options. See +[examples/regex-configs](examples/regex-configs) for both configuration forms. ```bash uv run privacy-guard \ --scanner-config examples/regex-configs/customer.yaml \ - --listen 127.0.0.1:50051 + --listen 127.0.0.1:50051 \ + regex ``` Each entity has a unique name and a non-empty `patterns` list. Patterns declare diff --git a/projects/privacy-guard/REGEX_SCANNER_PLAN.md b/projects/privacy-guard/REGEX_SCANNER_PLAN.md index 91ab2ac..f756acf 100644 --- a/projects/privacy-guard/REGEX_SCANNER_PLAN.md +++ b/projects/privacy-guard/REGEX_SCANNER_PLAN.md @@ -68,10 +68,10 @@ profiles: ignore_case: true ``` -The command will require `--profile NAME` after the scanner-option separator -when the file contains `profiles`. It will reject `--profile` for a -single-profile file, avoiding an ignored or ambiguous option. Exactly one -selected entity catalog is active; profiles are not merged implicitly. +The `regex` built-in command will require `--profile NAME` when the file +contains `profiles`. It will reject `--profile` for a single-profile file, +avoiding an ignored or ambiguous option. Exactly one selected entity catalog is +active; profiles are not merged implicitly. Scanner identity is runtime configuration rather than rule-set content. The default command will use the stable name `regex`, with an explicit @@ -214,11 +214,12 @@ they are not blamed on the request that happened to expose the pattern defect. the scanner implementation must not emit a response its consumer rejects. 7. Add a loader such as `RegexScanner.from_yaml(path, profile=None)`. Update the packaged command to accept one required, scanner-agnostic - `--scanner-config PATH`. Put the conditionally required `--profile NAME` and - optional `--scanner-name` after the scanner-option separator. Load and compile the - configuration before binding a listening socket. Restore the `privacy-guard` - script only in the same change, so the command cannot run with an implicit - allow-all scanner between implementations. + `--scanner-config PATH`. Expose RegexScanner as the `regex` built-in command, + with the conditionally required `--profile NAME` and optional + `--scanner-name`. Load and compile the configuration before binding a + listening socket. Restore the `privacy-guard` script only in the same change, + so the command cannot run with an implicit allow-all scanner between + implementations. 8. Add example rule sets under `examples/regex-configs/`: single-profile `customer.yaml` and `hipaa.yaml` files plus a `profiles.yaml` file showing both catalogs in the multi-profile shape. The HIPAA example must be diff --git a/projects/privacy-guard/examples/regex-configs/README.md b/projects/privacy-guard/examples/regex-configs/README.md index 5b5cbb4..02990d3 100644 --- a/projects/privacy-guard/examples/regex-configs/README.md +++ b/projects/privacy-guard/examples/regex-configs/README.md @@ -2,12 +2,12 @@ `customer.yaml` and `hipaa.yaml` demonstrate the single-profile entity-list shape. `profiles.yaml` combines both catalogs under explicit profile names. Its -profile is a RegexScanner option, so select one after the scanner-option -separator, for example: +profile is a RegexScanner option, so select the `regex` built-in and pass one of +its profiles, for example: ```bash -privacy-guard --scanner-config profiles.yaml -- --profile customer-support -privacy-guard --scanner-config profiles.yaml -- --profile hipaa +privacy-guard --scanner-config profiles.yaml regex --profile customer-support +privacy-guard --scanner-config profiles.yaml regex --profile hipaa ``` The HIPAA catalog is only a starting rule set. It is not a claim of compliance. diff --git a/projects/privacy-guard/pyproject.toml b/projects/privacy-guard/pyproject.toml index 7b8d5a1..b7449bc 100644 --- a/projects/privacy-guard/pyproject.toml +++ b/projects/privacy-guard/pyproject.toml @@ -13,11 +13,12 @@ dependencies = [ "protobuf>=6.33.5,<7", "pydantic>=2.11,<3", "pyyaml>=6.0.2,<7", + "typer>=0.16,<1", "typing-extensions>=4.12,<5", ] [project.scripts] -privacy-guard = "privacy_guard.service.server:main" +privacy-guard = "privacy_guard.service.server:app" [project.urls] Repository = "https://github.com/NVIDIA/OpenShell-Research" diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index a28cd8e..8a725c3 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -7,12 +7,13 @@ from __future__ import annotations -import argparse import asyncio -import sys -from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Annotated import grpc +import typer from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES @@ -21,6 +22,13 @@ from privacy_guard.scanners import RegexScanner, Scanner, ScannerConfig from privacy_guard.service.servicer import PrivacyGuardMiddleware +app = typer.Typer( + name="privacy-guard", + help="Run Privacy Guard with a built-in scanner.", + no_args_is_help=True, + add_completion=False, +) + class MiddlewareServer: """High-level server that wires a scanner into the Privacy Guard service.""" @@ -64,60 +72,54 @@ async def serve( await servicer.close() -def main(argv: Sequence[str] | None = None) -> int: - """Load the default scanner from its required config, then run the server.""" - command_arguments = tuple(sys.argv[1:] if argv is None else argv) - server_arguments, scanner_arguments = _split_scanner_arguments(command_arguments) - parser = argparse.ArgumentParser( - description="Run the Privacy Guard middleware", - epilog=( - "Scanner-specific options follow --. Run '%(prog)s --scanner-config " - "PATH -- --help' to list them." - ), - ) - parser.add_argument( - "--scanner-config", - required=True, - help="Path to the active scanner's configuration", - ) - parser.add_argument("--listen", default="127.0.0.1:50051") - arguments = parser.parse_args(server_arguments) - scanner_parser = _create_regex_scanner_parser(parser.prog) - scanner_options = scanner_parser.parse_args(scanner_arguments or ()) +@app.callback() +def main( + context: typer.Context, + scanner_config: Annotated[ + Path, + typer.Option(help="Path to the built-in scanner's configuration."), + ], + listen: Annotated[ + str, + typer.Option(help="Address on which the middleware server listens."), + ] = "127.0.0.1:50051", +) -> None: + """Collect options shared by every built-in scanner.""" + context.obj = _CliOptions(scanner_config=scanner_config, listen=listen) + + +@app.command("regex") +def run_regex( + context: typer.Context, + profile: Annotated[ + str | None, + typer.Option(help="Profile required for a multi-profile configuration."), + ] = None, + scanner_name: Annotated[ + str, + typer.Option(help="Scanner identity attached to findings."), + ] = "regex", +) -> None: + """Run Privacy Guard with the built-in RegexScanner.""" + options = context.find_object(_CliOptions) + assert options is not None try: scanner = RegexScanner.from_yaml( - arguments.scanner_config, - scanner_options.profile, - scanner_name=scanner_options.scanner_name, + options.scanner_config, + profile, + scanner_name=scanner_name, ) - MiddlewareServer(scanner=scanner).serve(arguments.listen) + MiddlewareServer(scanner=scanner).serve(options.listen) except PrivacyGuardError as error: - parser.exit(status=1, message=f"{error}\n") - return 0 + typer.echo(str(error), err=True) + raise typer.Exit(code=1) from None -def _split_scanner_arguments( - arguments: tuple[str, ...], -) -> tuple[tuple[str, ...], tuple[str, ...] | None]: - try: - separator_index = arguments.index("--") - except ValueError: - return arguments, None - return arguments[:separator_index], arguments[separator_index + 1 :] - - -def _create_regex_scanner_parser(command_name: str) -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog=f"{command_name} --", - description="Configure the default RegexScanner", - ) - parser.add_argument( - "--profile", - help="Profile required for a multi-profile RegexScanner configuration", - ) - parser.add_argument("--scanner-name", default="regex") - return parser +@dataclass(frozen=True) +class _CliOptions: + scanner_config: Path + listen: str if __name__ == "__main__": - raise SystemExit(main()) + app() diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 6c04109..deb4f79 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -1,9 +1,11 @@ from collections.abc import Sequence +from pathlib import Path import grpc import pytest from google.protobuf import empty_pb2, message_factory from google.protobuf.message import Message +from typer.testing import CliRunner from typing_extensions import override from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 @@ -13,7 +15,7 @@ from privacy_guard.processor import RequestProcessor from privacy_guard.scanners import ScannerConfig from privacy_guard.service import server as server_module -from privacy_guard.service.server import MiddlewareServer, create_server, main, serve +from privacy_guard.service.server import MiddlewareServer, app, create_server, serve from privacy_guard.service.servicer import PrivacyGuardMiddleware from ..scanner_helpers import DeterministicEmailScanner @@ -83,7 +85,6 @@ async def record_serve(servicer: PrivacyGuardMiddleware, listen: str) -> None: def test_cli_reports_expected_configuration_failure_without_traceback( monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], ) -> None: expected = PrivacyGuardError(ErrorCode.SCANNER_CONFIG_INVALID) @@ -92,14 +93,15 @@ def reject_configuration(*args: object, **kwargs: object) -> None: monkeypatch.setattr(server_module.RegexScanner, "from_yaml", reject_configuration) - with pytest.raises(SystemExit) as exception_info: - main(["--scanner-config", "sensitive-path-8472.yaml"]) + result = CliRunner().invoke( + app, + ["--scanner-config", "sensitive-path-8472.yaml", "regex"], + ) - assert exception_info.value.code == 1 - stderr = capsys.readouterr().err - assert stderr == f"{expected}\n" - assert "Traceback" not in stderr - assert "8472" not in stderr + assert result.exit_code == 1 + assert result.output == f"{expected}\n" + assert "Traceback" not in result.output + assert "8472" not in result.output @pytest.mark.parametrize("profile", [None, "customer-support"]) @@ -113,25 +115,27 @@ def test_cli_profile_is_optional_and_forwarded_only_when_supplied( ) def load_scanner( - path: str, + path: str | Path, selected_profile: str | None, *, scanner_name: str, ) -> DeterministicEmailScanner: - calls.append((path, selected_profile, scanner_name)) + calls.append((str(path), selected_profile, scanner_name)) return scanner monkeypatch.setattr(server_module.RegexScanner, "from_yaml", load_scanner) monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) - arguments = ["--scanner-config", "scanner-config.yaml"] + arguments = ["--scanner-config", "scanner-config.yaml", "regex"] if profile is not None: - arguments.extend(("--", "--profile", profile)) + arguments.extend(("--profile", profile)) + + result = CliRunner().invoke(app, arguments) - assert main(arguments) == 0 + assert result.exit_code == 0 assert calls == [("scanner-config.yaml", profile, "regex")] -def test_cli_forwards_custom_scanner_name_after_separator( +def test_cli_forwards_custom_scanner_name_to_builtin( monkeypatch: pytest.MonkeyPatch, ) -> None: calls: list[tuple[str, str | None, str]] = [] @@ -140,78 +144,75 @@ def test_cli_forwards_custom_scanner_name_after_separator( ) def load_scanner( - path: str, + path: str | Path, selected_profile: str | None, *, scanner_name: str, ) -> DeterministicEmailScanner: - calls.append((path, selected_profile, scanner_name)) + calls.append((str(path), selected_profile, scanner_name)) return scanner monkeypatch.setattr(server_module.RegexScanner, "from_yaml", load_scanner) monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) - assert ( - main( - [ - "--scanner-config", - "scanner-config.yaml", - "--", - "--scanner-name", - "custom-regex", - ] - ) - == 0 + result = CliRunner().invoke( + app, + [ + "--scanner-config", + "scanner-config.yaml", + "regex", + "--scanner-name", + "custom-regex", + ], ) + + assert result.exit_code == 0 assert calls == [("scanner-config.yaml", None, "custom-regex")] -def test_cli_always_requires_scanner_configuration( - capsys: pytest.CaptureFixture[str], -) -> None: - with pytest.raises(SystemExit) as exception_info: - main(["--listen", "127.0.0.1:50051"]) +def test_cli_always_requires_scanner_configuration() -> None: + result = CliRunner().invoke(app, ["regex"]) - assert exception_info.value.code == 2 - assert "--scanner-config" in capsys.readouterr().err + assert result.exit_code == 2 + assert "--scanner-config" in result.output -def test_cli_exposes_separate_server_and_scanner_help( - capsys: pytest.CaptureFixture[str], -) -> None: - with pytest.raises(SystemExit) as server_help: - main(["--help"]) - assert server_help.value.code == 0 - server_output = capsys.readouterr().out - assert "--listen" in server_output - assert "--scanner-config" in server_output - assert "Scanner-specific options follow --" in server_output - assert "--scanner-config PATH" in server_output - - with pytest.raises(SystemExit) as scanner_help: - main(["--scanner-config", "scanner-config.yaml", "--", "--help"]) - assert scanner_help.value.code == 0 - scanner_output = capsys.readouterr().out - assert "--profile" in scanner_output - assert "--scanner-config" not in scanner_output - assert "--listen" not in scanner_output - - -def test_cli_rejects_scanner_options_before_separator( - capsys: pytest.CaptureFixture[str], -) -> None: - with pytest.raises(SystemExit) as exception_info: - main( - [ - "--scanner-config", - "scanner-config.yaml", - "--profile", - "customer-support", - ] - ) - - assert exception_info.value.code == 2 - assert "unrecognized arguments: --profile" in capsys.readouterr().err +def test_cli_exposes_root_and_builtin_scanner_help() -> None: + root_help = CliRunner().invoke(app, ["--help"]) + + assert root_help.exit_code == 0 + assert "privacy-guard" in root_help.output + assert "--listen" in root_help.output + assert "--scanner-config" in root_help.output + assert "regex" in root_help.output + + scanner_help = CliRunner().invoke( + app, + ["--scanner-config", "scanner-config.yaml", "regex", "--help"], + ) + + assert scanner_help.exit_code == 0 + assert "built-in RegexScanner" in scanner_help.output + assert "--profile" in scanner_help.output + assert "--scanner-name" in scanner_help.output + assert "--scanner-config" not in scanner_help.output + assert "--listen" not in scanner_help.output + + +def test_cli_rejects_builtin_options_before_subcommand() -> None: + result = CliRunner().invoke( + app, + [ + "--scanner-config", + "scanner-config.yaml", + "--profile", + "customer-support", + "regex", + ], + ) + + assert result.exit_code == 2 + assert "--profile" in result.output @pytest.mark.asyncio diff --git a/projects/privacy-guard/uv.lock b/projects/privacy-guard/uv.lock index 78c72b3..b7f208b 100644 --- a/projects/privacy-guard/uv.lock +++ b/projects/privacy-guard/uv.lock @@ -2,6 +2,15 @@ version = 1 revision = 3 requires-python = ">=3.11" +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -80,6 +89,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -107,6 +137,7 @@ dependencies = [ { name = "protobuf" }, { name = "pydantic" }, { name = "pyyaml" }, + { name = "typer" }, { name = "typing-extensions" }, ] @@ -124,6 +155,7 @@ requires-dist = [ { name = "protobuf", specifier = ">=6.33.5,<7" }, { name = "pydantic", specifier = ">=2.11,<3" }, { name = "pyyaml", specifier = ">=6.0.2,<7" }, + { name = "typer", specifier = ">=0.16,<1" }, { name = "typing-extensions", specifier = ">=4.12,<5" }, ] @@ -360,6 +392,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + [[package]] name = "ruff" version = "0.12.12" @@ -386,6 +431,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/7e/61c42657f6e4614a4258f1c3b0c5b93adc4d1f8575f5229d1906b483099b/ruff-0.12.12-py3-none-win_arm64.whl", hash = "sha256:2a8199cab4ce4d72d158319b63370abf60991495fb733db96cd923a34c52d093", size = 12256762, upload-time = "2025-09-04T16:50:15.737Z" }, ] +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "ty" version = "0.0.61" @@ -411,6 +465,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/14/7caec26d93a943c0e7d15eb7374644508d08cbd387d112b722b12d14e044/ty-0.0.61-py3-none-win_arm64.whl", hash = "sha256:3e496f7698bc4b5bbb1eb66d8b5799ba87596d88d36604ca359083893fa2fc49", size = 11693485, upload-time = "2026-07-18T01:39:52.73Z" }, ] +[[package]] +name = "typer" +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430, upload-time = "2026-07-15T19:21:07.007Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716, upload-time = "2026-07-15T19:21:05.553Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" From 9e3d14af923169820a3d9c6de574e12a1ff22554 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Wed, 22 Jul 2026 22:49:02 +0000 Subject: [PATCH 10/82] Make built-in scanner help self-contained --- projects/privacy-guard/README.md | 17 +++++----- projects/privacy-guard/REGEX_SCANNER_PLAN.md | 13 ++++---- .../examples/regex-configs/README.md | 4 +-- .../src/privacy_guard/service/server.py | 31 ++++++------------- .../tests/service/test_server.py | 19 ++++-------- 5 files changed, 31 insertions(+), 53 deletions(-) diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 14aeb36..d7edc6e 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -88,20 +88,19 @@ and entity. A scanner sequence is passed to `RequestProcessor`; scanner names must be unique and remain visible in aggregated findings. The `privacy-guard` command runs built-in scanners. Every built-in requires -`--scanner-config` on the shared command surface and owns that file's schema and -interpretation. `RegexScanner` is available through the `regex` subcommand; its -config path selects a YAML catalog. Single-profile files contain a non-empty -entity list. Multi-profile files contain only a non-empty `profiles` mapping and -require the regex-specific `--profile` option. Use `privacy-guard --help` for -shared options and built-in scanners, or -`privacy-guard --scanner-config PATH regex --help` for regex options. See +`--scanner-config` and owns that file's schema and interpretation. +`RegexScanner` is available through the `regex` subcommand; its config path +selects a YAML catalog. Single-profile files contain a non-empty entity list. +Multi-profile files contain only a non-empty `profiles` mapping and require the +regex-specific `--profile` option. Use `privacy-guard --help` for the built-in +scanner list or `privacy-guard regex --help` for regex options. See [examples/regex-configs](examples/regex-configs) for both configuration forms. ```bash uv run privacy-guard \ + regex \ --scanner-config examples/regex-configs/customer.yaml \ - --listen 127.0.0.1:50051 \ - regex + --listen 127.0.0.1:50051 ``` Each entity has a unique name and a non-empty `patterns` list. Patterns declare diff --git a/projects/privacy-guard/REGEX_SCANNER_PLAN.md b/projects/privacy-guard/REGEX_SCANNER_PLAN.md index f756acf..ca8517f 100644 --- a/projects/privacy-guard/REGEX_SCANNER_PLAN.md +++ b/projects/privacy-guard/REGEX_SCANNER_PLAN.md @@ -213,13 +213,12 @@ they are not blamed on the request that happened to expose the pattern defect. separate OpenShell protocol change before increasing Privacy Guard's limit; the scanner implementation must not emit a response its consumer rejects. 7. Add a loader such as `RegexScanner.from_yaml(path, profile=None)`. Update the - packaged command to accept one required, scanner-agnostic - `--scanner-config PATH`. Expose RegexScanner as the `regex` built-in command, - with the conditionally required `--profile NAME` and optional - `--scanner-name`. Load and compile the configuration before binding a - listening socket. Restore the `privacy-guard` script only in the same change, - so the command cannot run with an implicit allow-all scanner between - implementations. + packaged command to expose RegexScanner as the `regex` built-in command with + one required `--scanner-config PATH`, the conditionally required + `--profile NAME`, and optional `--scanner-name`. Load and compile the + configuration before binding a listening socket. Restore the `privacy-guard` + script only in the same change, so the command cannot run with an implicit + allow-all scanner between implementations. 8. Add example rule sets under `examples/regex-configs/`: single-profile `customer.yaml` and `hipaa.yaml` files plus a `profiles.yaml` file showing both catalogs in the multi-profile shape. The HIPAA example must be diff --git a/projects/privacy-guard/examples/regex-configs/README.md b/projects/privacy-guard/examples/regex-configs/README.md index 02990d3..b97e734 100644 --- a/projects/privacy-guard/examples/regex-configs/README.md +++ b/projects/privacy-guard/examples/regex-configs/README.md @@ -6,8 +6,8 @@ profile is a RegexScanner option, so select the `regex` built-in and pass one of its profiles, for example: ```bash -privacy-guard --scanner-config profiles.yaml regex --profile customer-support -privacy-guard --scanner-config profiles.yaml regex --profile hipaa +privacy-guard regex --scanner-config profiles.yaml --profile customer-support +privacy-guard regex --scanner-config profiles.yaml --profile hipaa ``` The HIPAA catalog is only a starting rule set. It is not a claim of compliance. diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 8a725c3..71a45a1 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -8,7 +8,6 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass from pathlib import Path from typing import Annotated @@ -73,24 +72,20 @@ async def serve( @app.callback() -def main( - context: typer.Context, +def main() -> None: + """Select one of Privacy Guard's built-in scanners.""" + + +@app.command("regex") +def run_regex( scanner_config: Annotated[ Path, - typer.Option(help="Path to the built-in scanner's configuration."), + typer.Option(help="Path to the RegexScanner configuration."), ], listen: Annotated[ str, typer.Option(help="Address on which the middleware server listens."), ] = "127.0.0.1:50051", -) -> None: - """Collect options shared by every built-in scanner.""" - context.obj = _CliOptions(scanner_config=scanner_config, listen=listen) - - -@app.command("regex") -def run_regex( - context: typer.Context, profile: Annotated[ str | None, typer.Option(help="Profile required for a multi-profile configuration."), @@ -101,25 +96,17 @@ def run_regex( ] = "regex", ) -> None: """Run Privacy Guard with the built-in RegexScanner.""" - options = context.find_object(_CliOptions) - assert options is not None try: scanner = RegexScanner.from_yaml( - options.scanner_config, + scanner_config, profile, scanner_name=scanner_name, ) - MiddlewareServer(scanner=scanner).serve(options.listen) + MiddlewareServer(scanner=scanner).serve(listen) except PrivacyGuardError as error: typer.echo(str(error), err=True) raise typer.Exit(code=1) from None -@dataclass(frozen=True) -class _CliOptions: - scanner_config: Path - listen: str - - if __name__ == "__main__": app() diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index deb4f79..27b4d11 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -95,7 +95,7 @@ def reject_configuration(*args: object, **kwargs: object) -> None: result = CliRunner().invoke( app, - ["--scanner-config", "sensitive-path-8472.yaml", "regex"], + ["regex", "--scanner-config", "sensitive-path-8472.yaml"], ) assert result.exit_code == 1 @@ -125,7 +125,7 @@ def load_scanner( monkeypatch.setattr(server_module.RegexScanner, "from_yaml", load_scanner) monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) - arguments = ["--scanner-config", "scanner-config.yaml", "regex"] + arguments = ["regex", "--scanner-config", "scanner-config.yaml"] if profile is not None: arguments.extend(("--profile", profile)) @@ -158,9 +158,9 @@ def load_scanner( result = CliRunner().invoke( app, [ + "regex", "--scanner-config", "scanner-config.yaml", - "regex", "--scanner-name", "custom-regex", ], @@ -182,29 +182,22 @@ def test_cli_exposes_root_and_builtin_scanner_help() -> None: assert root_help.exit_code == 0 assert "privacy-guard" in root_help.output - assert "--listen" in root_help.output - assert "--scanner-config" in root_help.output assert "regex" in root_help.output - scanner_help = CliRunner().invoke( - app, - ["--scanner-config", "scanner-config.yaml", "regex", "--help"], - ) + scanner_help = CliRunner().invoke(app, ["regex", "--help"]) assert scanner_help.exit_code == 0 assert "built-in RegexScanner" in scanner_help.output + assert "--scanner-config" in scanner_help.output + assert "--listen" in scanner_help.output assert "--profile" in scanner_help.output assert "--scanner-name" in scanner_help.output - assert "--scanner-config" not in scanner_help.output - assert "--listen" not in scanner_help.output def test_cli_rejects_builtin_options_before_subcommand() -> None: result = CliRunner().invoke( app, [ - "--scanner-config", - "scanner-config.yaml", "--profile", "customer-support", "regex", From 20f69eeee0e6f8c5d89197a42bf1916a4d783773 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Wed, 22 Jul 2026 22:51:51 +0000 Subject: [PATCH 11/82] Rename built-in scanner config option --- projects/privacy-guard/README.md | 4 ++-- projects/privacy-guard/REGEX_SCANNER_PLAN.md | 2 +- .../privacy-guard/examples/regex-configs/README.md | 4 ++-- .../privacy-guard/src/privacy_guard/service/server.py | 4 ++-- projects/privacy-guard/tests/service/test_server.py | 10 +++++----- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index d7edc6e..844e7db 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -88,7 +88,7 @@ and entity. A scanner sequence is passed to `RequestProcessor`; scanner names must be unique and remain visible in aggregated findings. The `privacy-guard` command runs built-in scanners. Every built-in requires -`--scanner-config` and owns that file's schema and interpretation. +`--config` and owns that file's schema and interpretation. `RegexScanner` is available through the `regex` subcommand; its config path selects a YAML catalog. Single-profile files contain a non-empty entity list. Multi-profile files contain only a non-empty `profiles` mapping and require the @@ -99,7 +99,7 @@ scanner list or `privacy-guard regex --help` for regex options. See ```bash uv run privacy-guard \ regex \ - --scanner-config examples/regex-configs/customer.yaml \ + --config examples/regex-configs/customer.yaml \ --listen 127.0.0.1:50051 ``` diff --git a/projects/privacy-guard/REGEX_SCANNER_PLAN.md b/projects/privacy-guard/REGEX_SCANNER_PLAN.md index ca8517f..ef95d12 100644 --- a/projects/privacy-guard/REGEX_SCANNER_PLAN.md +++ b/projects/privacy-guard/REGEX_SCANNER_PLAN.md @@ -214,7 +214,7 @@ they are not blamed on the request that happened to expose the pattern defect. the scanner implementation must not emit a response its consumer rejects. 7. Add a loader such as `RegexScanner.from_yaml(path, profile=None)`. Update the packaged command to expose RegexScanner as the `regex` built-in command with - one required `--scanner-config PATH`, the conditionally required + one required `--config PATH`, the conditionally required `--profile NAME`, and optional `--scanner-name`. Load and compile the configuration before binding a listening socket. Restore the `privacy-guard` script only in the same change, so the command cannot run with an implicit diff --git a/projects/privacy-guard/examples/regex-configs/README.md b/projects/privacy-guard/examples/regex-configs/README.md index b97e734..f3e24a7 100644 --- a/projects/privacy-guard/examples/regex-configs/README.md +++ b/projects/privacy-guard/examples/regex-configs/README.md @@ -6,8 +6,8 @@ profile is a RegexScanner option, so select the `regex` built-in and pass one of its profiles, for example: ```bash -privacy-guard regex --scanner-config profiles.yaml --profile customer-support -privacy-guard regex --scanner-config profiles.yaml --profile hipaa +privacy-guard regex --config profiles.yaml --profile customer-support +privacy-guard regex --config profiles.yaml --profile hipaa ``` The HIPAA catalog is only a starting rule set. It is not a claim of compliance. diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 71a45a1..9c423b8 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -78,7 +78,7 @@ def main() -> None: @app.command("regex") def run_regex( - scanner_config: Annotated[ + config: Annotated[ Path, typer.Option(help="Path to the RegexScanner configuration."), ], @@ -98,7 +98,7 @@ def run_regex( """Run Privacy Guard with the built-in RegexScanner.""" try: scanner = RegexScanner.from_yaml( - scanner_config, + config, profile, scanner_name=scanner_name, ) diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 27b4d11..e6ba3cc 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -95,7 +95,7 @@ def reject_configuration(*args: object, **kwargs: object) -> None: result = CliRunner().invoke( app, - ["regex", "--scanner-config", "sensitive-path-8472.yaml"], + ["regex", "--config", "sensitive-path-8472.yaml"], ) assert result.exit_code == 1 @@ -125,7 +125,7 @@ def load_scanner( monkeypatch.setattr(server_module.RegexScanner, "from_yaml", load_scanner) monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) - arguments = ["regex", "--scanner-config", "scanner-config.yaml"] + arguments = ["regex", "--config", "scanner-config.yaml"] if profile is not None: arguments.extend(("--profile", profile)) @@ -159,7 +159,7 @@ def load_scanner( app, [ "regex", - "--scanner-config", + "--config", "scanner-config.yaml", "--scanner-name", "custom-regex", @@ -174,7 +174,7 @@ def test_cli_always_requires_scanner_configuration() -> None: result = CliRunner().invoke(app, ["regex"]) assert result.exit_code == 2 - assert "--scanner-config" in result.output + assert "--config" in result.output def test_cli_exposes_root_and_builtin_scanner_help() -> None: @@ -188,7 +188,7 @@ def test_cli_exposes_root_and_builtin_scanner_help() -> None: assert scanner_help.exit_code == 0 assert "built-in RegexScanner" in scanner_help.output - assert "--scanner-config" in scanner_help.output + assert "--config" in scanner_help.output assert "--listen" in scanner_help.output assert "--profile" in scanner_help.output assert "--scanner-name" in scanner_help.output From cfc086a9827ea6d9bebf9d52de7063e625602c60 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Wed, 22 Jul 2026 22:57:28 +0000 Subject: [PATCH 12/82] Harden Privacy Guard CLI bind failures --- .../examples/regex-configs/README.md | 8 ++- .../src/privacy_guard/service/server.py | 5 +- .../tests/service/test_server.py | 59 +++++++++++++++++++ 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/projects/privacy-guard/examples/regex-configs/README.md b/projects/privacy-guard/examples/regex-configs/README.md index f3e24a7..f8758be 100644 --- a/projects/privacy-guard/examples/regex-configs/README.md +++ b/projects/privacy-guard/examples/regex-configs/README.md @@ -6,8 +6,12 @@ profile is a RegexScanner option, so select the `regex` built-in and pass one of its profiles, for example: ```bash -privacy-guard regex --config profiles.yaml --profile customer-support -privacy-guard regex --config profiles.yaml --profile hipaa +privacy-guard regex \ + --config examples/regex-configs/profiles.yaml \ + --profile customer-support +privacy-guard regex \ + --config examples/regex-configs/profiles.yaml \ + --profile hipaa ``` The HIPAA catalog is only a starting rule set. It is not a claim of compliance. diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 9c423b8..a4c8107 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -61,7 +61,10 @@ async def serve( """Bind ``listen``, start the server, and serve until terminated.""" server = create_server(servicer) try: - bound_port = server.add_insecure_port(listen) + try: + bound_port = server.add_insecure_port(listen) + except RuntimeError: + raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) from None if bound_port == 0: raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) await server.start() diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index e6ba3cc..2b10230 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -104,6 +104,36 @@ def reject_configuration(*args: object, **kwargs: object) -> None: assert "8472" not in result.output +def test_cli_reports_bind_failure_without_traceback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + expected = PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) + scanner = DeterministicEmailScanner( + ScannerConfig(name="regex", entity_types=frozenset({"email"})) + ) + + monkeypatch.setattr( + server_module.RegexScanner, + "from_yaml", + lambda *args, **kwargs: scanner, + ) + + def reject_listen_address(self: MiddlewareServer, listen: str) -> None: + raise expected + + monkeypatch.setattr(MiddlewareServer, "serve", reject_listen_address) + + result = CliRunner().invoke( + app, + ["regex", "--config", "scanner-config.yaml", "--listen", "bad:8472"], + ) + + assert result.exit_code == 1 + assert result.output == f"{expected}\n" + assert "Traceback" not in result.output + assert "8472" not in result.output + + @pytest.mark.parametrize("profile", [None, "customer-support"]) def test_cli_profile_is_optional_and_forwarded_only_when_supplied( monkeypatch: pytest.MonkeyPatch, @@ -394,6 +424,35 @@ async def wait_for_termination(self, timeout: float | None = None) -> bool: assert fake_server.stopped is True +@pytest.mark.asyncio +async def test_serve_translates_runtime_bind_failure_and_stops_server( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class BindFailureServer(LifecycleServerFake): + @override + def add_insecure_port(self, address: str) -> int: + raise RuntimeError("invalid-sensitive-listen-8472") + + @override + async def start(self) -> None: + raise AssertionError("start must not run after bind failure") + + @override + async def wait_for_termination(self, timeout: float | None = None) -> bool: + raise AssertionError("wait must not run after bind failure") + + fake_server = BindFailureServer() + monkeypatch.setattr(server_module, "create_server", lambda _: fake_server) + + with pytest.raises(PrivacyGuardError) as exception_info: + await serve(_middleware(), "invalid-sensitive-listen-8472") + + assert exception_info.value.code is ErrorCode.SERVER_BIND_FAILED + assert exception_info.value.__cause__ is None + assert "8472" not in str(exception_info.value) + assert fake_server.stopped is True + + @pytest.mark.asyncio async def test_startup_failure_stops_server_and_propagates( monkeypatch: pytest.MonkeyPatch, From 92dbf86bb8b14bfd7af1451f96e9040ebe0b94b9 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 01:02:26 +0000 Subject: [PATCH 13/82] Add end-to-end regex scanner walkthrough --- projects/privacy-guard/README.md | 14 +- projects/privacy-guard/REGEX_SCANNER_PLAN.md | 280 ------------------ .../examples/regex-configs/README.md | 20 -- .../examples/regex-configs/hipaa.yaml | 8 - .../examples/regex-configs/profiles.yaml | 19 -- .../examples/regex-scanner/README.md | 172 +++++++++++ .../examples/regex-scanner/gateway.toml | 12 + .../examples/regex-scanner/policy.yaml | 47 +++ .../regex-scanner.yaml} | 0 .../tests/examples/test_regex_scanner.py | 36 +++ 10 files changed, 276 insertions(+), 332 deletions(-) delete mode 100644 projects/privacy-guard/REGEX_SCANNER_PLAN.md delete mode 100644 projects/privacy-guard/examples/regex-configs/README.md delete mode 100644 projects/privacy-guard/examples/regex-configs/hipaa.yaml delete mode 100644 projects/privacy-guard/examples/regex-configs/profiles.yaml create mode 100644 projects/privacy-guard/examples/regex-scanner/README.md create mode 100644 projects/privacy-guard/examples/regex-scanner/gateway.toml create mode 100644 projects/privacy-guard/examples/regex-scanner/policy.yaml rename projects/privacy-guard/examples/{regex-configs/customer.yaml => regex-scanner/regex-scanner.yaml} (100%) create mode 100644 projects/privacy-guard/tests/examples/test_regex_scanner.py diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 844e7db..2a90c11 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -12,9 +12,12 @@ original. > the generic JSON handler, safe gRPC adaptation, and a loopback server are > implemented. -The self-contained [email scanner example](examples/email-scanner/README.md) -provides a deterministic scanner, middleware entry point, gateway registration, -sandbox policy, and manual Claude Code workflow. +The self-contained +[built-in regex scanner walkthrough](examples/regex-scanner/README.md) provides +a scanner configuration, gateway registration, sandbox policy, and manual +Claude Code workflow for a typical deployment. The separate +[email scanner example](examples/email-scanner/README.md) demonstrates how to +implement a custom scanner. ## Request flow @@ -94,12 +97,13 @@ selects a YAML catalog. Single-profile files contain a non-empty entity list. Multi-profile files contain only a non-empty `profiles` mapping and require the regex-specific `--profile` option. Use `privacy-guard --help` for the built-in scanner list or `privacy-guard regex --help` for regex options. See -[examples/regex-configs](examples/regex-configs) for both configuration forms. +[the built-in regex scanner walkthrough](examples/regex-scanner/README.md) for a +complete manual deployment. ```bash uv run privacy-guard \ regex \ - --config examples/regex-configs/customer.yaml \ + --config examples/regex-scanner/regex-scanner.yaml \ --listen 127.0.0.1:50051 ``` diff --git a/projects/privacy-guard/REGEX_SCANNER_PLAN.md b/projects/privacy-guard/REGEX_SCANNER_PLAN.md deleted file mode 100644 index ef95d12..0000000 --- a/projects/privacy-guard/REGEX_SCANNER_PLAN.md +++ /dev/null @@ -1,280 +0,0 @@ -# Regex scanner implementation plan - -## Outcome - -`RegexScanner` will become Privacy Guard's default scanner implementation. It -will compile a strict YAML configuration at startup and emit findings for every -configured entity and pattern match. Operators will be able to maintain focused -configuration files for different environments, such as customer-specific data -or HIPAA-regulated identifiers, without changing Python code. - -The scanner type will be the default; the rule set will remain explicit. The -service must never silently start with an empty or missing rule set. - -“Supports” a catalog size means that the complete configuration can be parsed, -validated, compiled, and used to scan representative requests within the -default request budget. Merely accepting a large file that predictably times -out does not satisfy the scalability goal. Catalog capacity is independent of -match cardinality: a profile may define thousands of entities even though one -unusually dense request can still hit the existing per-block, per-request, or -protocol-result limits and be denied safely. - -## Proposed YAML contract - -A single-profile configuration is its complete entity catalog. Every listed -entity is active, so an additional top-level `entities` wrapper would add no -information: - -```yaml -- name: email - patterns: - - name: common-email - regex: '(?)`. The trailing empty named marker - makes the pattern identity observable without inserting a capture before the - user's expression, so unnamed group numbering and numeric backreferences are - preserved. Retain the original pattern name alongside the compiled rule. - Compile patterns separately and use overlapping iteration so matches can - overlap both within one pattern and across different patterns, while - normalized group names remain rule-local. Read the automatic marker to - identify the matched rule, then report the original, non-normalized pattern - name from immutable rule metadata. Scanner calls will remain stateless and - thread-safe. -3. Add an immutable request-scoped `ScanBudget` containing a monotonic deadline - and export it from `privacy_guard.scanners` for advanced scanner authors. - Begin with a conservative provisional package default, then document and - finalize it using the benchmark gate below. `RequestProcessor` will accept an - optional finite, positive, upper-bounded `scan_timeout_seconds` construction - argument with that safe default; callers do not need to configure it. Reject - booleans as well as non-finite, non-positive, and over-maximum numeric values. - `RequestProcessor.process` will create one budget immediately before scanning - and pass it through every `Scanner.scan` call for every block and scanner. - `Scanner.scan` will accept an optional keyword-only budget and create a fresh - default budget for standalone calls when none is supplied. The protected - scanner hook receives the effective non-optional budget; simple scanners may - ignore it, while scanners performing potentially unbounded work must - cooperate with it. The budget is a cooperative contract for `RegexScanner` - and custom scanners because the standard-library regex engine cannot - interrupt an active evaluation. `RegexScanner` checks the budget before and - after each evaluation and while consuming matches. Budget exhaustion raises - one exported, content-safe - `ScanBudgetExceeded` exception from `Scanner.scan`, with no partial findings. - `RequestProcessor` will catch only that typed condition, discard findings - accumulated from earlier scanners or blocks, and produce the existing stable - limit deny. Standalone scanner callers receive the typed exception instead of - a service-specific decision. This deliberately evolves the protected scanner - extension signature; all built-in examples, tests, and extension - documentation must migrate in the same change. -4. Define independent limits for configuration bytes, profiles, entities, - patterns, pattern bytes, matches per pattern, and request evaluation time. - Configuration capacity must not be derived from response aggregation limits: - profiles should support thousands of entities and patterns. Establish the - final hard ceilings with startup-time, startup-memory, and scan-time - benchmarks. The scalability gate is 1,000 active entities and 10,000 active - patterns on documented representative inputs and reference hardware. Record - input size and match density with each result so the claim is reproducible. - The request-scoped budget bounds work between regex evaluations regardless - of catalog size. If separately evaluating 10,000 compiled patterns cannot - meet the default budget, introduce bounded batching, indexing, or a - multi-pattern engine before claiming that scale; preserve overlapping - results and original pattern identity in any optimization. If batching combines patterns, assign - collision-free internal group identifiers even when configured names - normalize to the same value. Implement conservative provisional file and - count ceilings first, then finalize and document them only after this gate - passes. -5. Add optional bounded string metadata to the domain `Finding` so scanners can - attach scanner-specific attribution without adding fields to the shared model. - `RegexScanner` stores its configured pattern name under `pattern_name`. - Include that metadata value, or `""` when absent, after entity in - scanner-result, block-result, overlap-winner, and final-winner ordering keys - so otherwise identical findings remain deterministic without changing higher-priority - span or confidence semantics. -6. Include the pattern in audit-safe service reporting. Findings without a - pattern retain the existing `type=scanner_name` and `label=entity` shape. - Findings with a pattern retain `type=scanner_name` and use the unambiguous - `label=entity/pattern_name` shape. Group by scanner, entity, optional pattern, - and confidence; validate the combined UTF-8 label and encoded protobuf size - before returning it. Keep the current limit of 32 aggregated findings per - middleware stage: the checked-in OpenShell protocol explicitly documents - that receiver limit. It limits distinct groups matched in one response, not - entities or patterns configured in a profile. The existing 4,096 domain - findings per request and 4 KiB per-group encoded limits also remain. If more - than 32 distinct groups must be reported in one stage, coordinate that as a - separate OpenShell protocol change before increasing Privacy Guard's limit; - the scanner implementation must not emit a response its consumer rejects. -7. Add a loader such as `RegexScanner.from_yaml(path, profile=None)`. Update the - packaged command to expose RegexScanner as the `regex` built-in command with - one required `--config PATH`, the conditionally required - `--profile NAME`, and optional `--scanner-name`. Load and compile the - configuration before binding a listening socket. Restore the `privacy-guard` - script only in the same change, so the command cannot run with an implicit - allow-all scanner between implementations. -8. Add example rule sets under `examples/regex-configs/`: single-profile - `customer.yaml` and `hipaa.yaml` files plus a `profiles.yaml` file showing - both catalogs in the multi-profile shape. The HIPAA example must be - described as a starting rule set, not a claim of compliance; operational - controls and validation remain the deployer's responsibility. Migrate the - self-contained `examples/email-scanner/` example from its handwritten scanner - to `RegexScanner` and a colocated YAML file, so the primary manual example - exercises the new default without depending on another example directory. - -## Verification - -- Unit-test both YAML shapes, profile selection, missing and unexpected - selections, ordering, limits, duplicate keys, aliases, unsafe tags, excessive - nesting and scalar sizes, unsupported flags, malformed regexes, and - content-safe failures. Verify that every profile is validated even when it is - not selected. -- Test Unicode offsets, multiple entities and patterns, overlaps, confidence, - case handling, automatic hyphen normalization, normalized-name coexistence, - trailing named markers, numeric backreferences, reserved named-group and - inline-flag rejection, load-time and contextual zero-width rejection, - deterministic ordering, and finding limits. -- Test per-pattern and request-wide budget behavior with many patterns, many - text blocks, and long inputs. Verify exhaustion paths raise - `ScanBudgetExceeded` for standalone calls and return a stable deny through - `RequestProcessor`, with no partial findings in either case. Document that an - active standard-library regex evaluation cannot be interrupted. -- Test that `RequestProcessor` and standalone `Scanner.scan` calls use safe - default budgets when callers provide no timeout or budget, and that an - explicit processor timeout applies one shared deadline across the request. -- Test concurrent calls against one scanner instance. -- Exercise observe, redact, and block through `RequestProcessor` and the gRPC - boundary with both customer and HIPAA example configurations. Verify that - domain and aggregated service findings report the configured pattern name, - legacy findings without a pattern retain their current representation, and - profiles containing more than 32 entities load and scan sparse inputs - normally. Verify the exact 32-group protocol boundary, the 4,096 - domain-finding boundary, and the per-group encoded-size boundary; the - thirty-third distinct response group must produce the stable limit deny with - no partial findings. -- Extend the diagnostic benchmark with representative rule counts and matching - densities, including 100, 1,000, and 10,000 active patterns. Record parse and - compilation time, retained memory, scan latency, timeout rate, input size, - and match density before establishing the final configuration ceilings, - default request budget, or performance thresholds. - -## Delivery sequence - -Each delivery step includes its focused unit tests; the final step adds -cross-component and adversarial coverage rather than deferring all testing. - -1. Configuration models, bounded YAML loader, dependencies, and conservative - provisional safety limits. -2. Request-scoped scan budget, regex compilation, and scanning engine. -3. Scalability benchmark, engine optimization if required, and final limits and - default budget. -4. Backward-compatible finding metadata and pattern-aware service reporting. -5. Service CLI wiring and restoration of the package script. -6. Customer and HIPAA example configurations and user documentation. -7. Security-focused tests and full integration tests. diff --git a/projects/privacy-guard/examples/regex-configs/README.md b/projects/privacy-guard/examples/regex-configs/README.md deleted file mode 100644 index f8758be..0000000 --- a/projects/privacy-guard/examples/regex-configs/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Regex scanner configurations - -`customer.yaml` and `hipaa.yaml` demonstrate the single-profile entity-list -shape. `profiles.yaml` combines both catalogs under explicit profile names. Its -profile is a RegexScanner option, so select the `regex` built-in and pass one of -its profiles, for example: - -```bash -privacy-guard regex \ - --config examples/regex-configs/profiles.yaml \ - --profile customer-support -privacy-guard regex \ - --config examples/regex-configs/profiles.yaml \ - --profile hipaa -``` - -The HIPAA catalog is only a starting rule set. It is not a claim of compliance. -Deployers remain responsible for validating detection behavior and implementing -the operational, administrative, and technical controls their environment -requires. diff --git a/projects/privacy-guard/examples/regex-configs/hipaa.yaml b/projects/privacy-guard/examples/regex-configs/hipaa.yaml deleted file mode 100644 index 3e02d2e..0000000 --- a/projects/privacy-guard/examples/regex-configs/hipaa.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Starting rules only. This file does not establish HIPAA compliance; deployers -# remain responsible for operational controls, validation, and risk assessment. -- name: medical-record-number - patterns: - - name: prefixed-mrn - regex: '\bMRN[ :]?[0-9]{6,10}\b' - confidence: high - ignore_case: true diff --git a/projects/privacy-guard/examples/regex-configs/profiles.yaml b/projects/privacy-guard/examples/regex-configs/profiles.yaml deleted file mode 100644 index 0010674..0000000 --- a/projects/privacy-guard/examples/regex-configs/profiles.yaml +++ /dev/null @@ -1,19 +0,0 @@ -profiles: - customer-support: - - name: email - patterns: - - name: common-email - regex: '(? None: + scanner = RegexScanner.from_yaml(EXAMPLE_DIRECTORY / "regex-scanner.yaml") + + findings = scanner.scan("Email user@example.com, customer CUST-12345678") + + assert [finding.entity for finding in findings] == ["email", "customer-id"] + pattern_names: list[str] = [] + for finding in findings: + assert finding.metadata is not None + pattern_names.append(finding.metadata[PATTERN_NAME_METADATA_KEY]) + assert pattern_names == [ + "common-email", + "prefixed-customer-id", + ] + + +def test_example_configuration_and_walkthrough_are_aligned() -> None: + policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() + gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() + readme = (EXAMPLE_DIRECTORY / "README.md").read_text() + + assert "middleware: privacy-guard-regex-scanner" in policy + assert 'name = "privacy-guard-regex-scanner"' in gateway + assert "action: redact" in policy + assert "entity_types: [email, customer-id]" in policy + assert "privacy-guard regex" in readme + assert '--config "$PWD/regex-scanner.yaml"' in readme + assert "privacy-guard-regex-lab" in readme From e5233cb98aed5d76ddab047e4d92102dbd788a1f Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 01:10:25 +0000 Subject: [PATCH 14/82] Standardize Privacy Guard example setup --- .../examples/email-scanner/.gitignore | 1 + .../examples/email-scanner/README.md | 49 +++++++-------- .../examples/email-scanner/gateway.toml | 4 +- .../examples/regex-scanner/.gitignore | 1 + .../examples/regex-scanner/README.md | 59 ++++++++++--------- .../examples/regex-scanner/gateway.toml | 4 +- .../tests/examples/test_email_scanner.py | 7 ++- .../tests/examples/test_regex_scanner.py | 7 ++- 8 files changed, 76 insertions(+), 56 deletions(-) create mode 100644 projects/privacy-guard/examples/email-scanner/.gitignore create mode 100644 projects/privacy-guard/examples/regex-scanner/.gitignore diff --git a/projects/privacy-guard/examples/email-scanner/.gitignore b/projects/privacy-guard/examples/email-scanner/.gitignore new file mode 100644 index 0000000..b223234 --- /dev/null +++ b/projects/privacy-guard/examples/email-scanner/.gitignore @@ -0,0 +1 @@ +gateway.local.toml diff --git a/projects/privacy-guard/examples/email-scanner/README.md b/projects/privacy-guard/examples/email-scanner/README.md index 70af73e..91ac7c4 100644 --- a/projects/privacy-guard/examples/email-scanner/README.md +++ b/projects/privacy-guard/examples/email-scanner/README.md @@ -23,28 +23,30 @@ intended as comprehensive production PII detection. curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh ``` -Run the example commands from this directory: +Run all commands in this walkthrough from the Privacy Guard project root: ```bash -cd projects/privacy-guard/examples/email-scanner +cd projects/privacy-guard ``` -## 1. Edit the example config +In particular, every `uv run` command below assumes this working directory. -Find the host address that Docker can reach: +## 1. Generate the local gateway config -```bash -ipconfig getifaddr en0 -``` - -Replace `REPLACE_WITH_HOST_IP` in `gateway.toml` with that address. If `en0` -does not return an address, find the active interface with: +Generate `gateway.local.toml` with the address of the host interface Docker can +reach: ```bash -route get default | grep interface +HOST_INTERFACE="$(route get default | awk '/interface:/{print $2}')" +HOST_IP="$(ipconfig getifaddr "$HOST_INTERFACE")" +test -n "$HOST_IP" +sed "s/REPLACE_WITH_HOST_IP/$HOST_IP/" \ + examples/email-scanner/gateway.toml \ + > examples/email-scanner/gateway.local.toml ``` -Only the checked-out `gateway.toml` is edited. Do not copy it into +The checked-in `gateway.toml` remains an example-specific template. +`gateway.local.toml` is ignored by Git. Do not copy either file into `~/.config/openshell`. ## 2. Start Privacy Guard @@ -57,7 +59,7 @@ that Docker must reach instead of every interface. In terminal 1: ```bash -uv run --project ../.. python middleware_server.py \ +uv run python examples/email-scanner/middleware_server.py \ --listen 0.0.0.0:50051 ``` @@ -71,17 +73,16 @@ In terminal 2: brew services stop openshell OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" \ -openshell-gateway --config "$PWD/gateway.toml" +openshell-gateway --config "$PWD/examples/email-scanner/gateway.local.toml" ``` The first command stops the background service so the foreground gateway can use the standard port. The second command reuses the credentials and state created -by the recommended macOS installation, but loads `gateway.toml` from this -directory. It should stay in the foreground; `Server listening` means it is -ready. +by the recommended macOS installation, but loads the generated local config. It +should stay in the foreground; `Server listening` means it is ready. -Middleware registration is static. After editing `gateway.toml`, stop this -foreground process with `Ctrl-C` and run the second command again. +Middleware registration is static. After regenerating `gateway.local.toml`, stop +this foreground process with `Ctrl-C` and run the second command again. ## 4. Create the sandbox and run Claude @@ -94,7 +95,7 @@ openshell sandbox create \ --name privacy-guard-lab \ --from base \ --no-auto-providers \ - --policy "$PWD/policy.yaml" \ + --policy "$PWD/examples/email-scanner/policy.yaml" \ -- claude ``` @@ -118,11 +119,13 @@ an email finding. ## Change the behavior -Edit `policy.yaml` and change `on_finding.action` to `observe`, `block`, or -`redact`, then apply it without recreating the sandbox: +Edit `examples/email-scanner/policy.yaml` and change `on_finding.action` to +`observe`, `block`, or `redact`, then apply it without recreating the sandbox: ```bash -openshell policy set privacy-guard-lab --policy "$PWD/policy.yaml" --wait +openshell policy set privacy-guard-lab \ + --policy "$PWD/examples/email-scanner/policy.yaml" \ + --wait ``` - `redact` sends `[email]` to Claude. diff --git a/projects/privacy-guard/examples/email-scanner/gateway.toml b/projects/privacy-guard/examples/email-scanner/gateway.toml index 00ca9e7..713239d 100644 --- a/projects/privacy-guard/examples/email-scanner/gateway.toml +++ b/projects/privacy-guard/examples/email-scanner/gateway.toml @@ -1,6 +1,6 @@ # OpenShell gateway configuration for the email-scanner example. -# Replace REPLACE_WITH_HOST_IP, then pass this file to openshell-gateway with -# --config. Do not copy it into ~/.config/openshell. +# Generate the ignored gateway.local.toml from this template using the README. +# Do not copy either file into ~/.config/openshell. [openshell] version = 1 diff --git a/projects/privacy-guard/examples/regex-scanner/.gitignore b/projects/privacy-guard/examples/regex-scanner/.gitignore new file mode 100644 index 0000000..b223234 --- /dev/null +++ b/projects/privacy-guard/examples/regex-scanner/.gitignore @@ -0,0 +1 @@ +gateway.local.toml diff --git a/projects/privacy-guard/examples/regex-scanner/README.md b/projects/privacy-guard/examples/regex-scanner/README.md index a2e4483..98393a9 100644 --- a/projects/privacy-guard/examples/regex-scanner/README.md +++ b/projects/privacy-guard/examples/regex-scanner/README.md @@ -20,12 +20,14 @@ directory. curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh ``` -Run the remaining commands from this directory: +Run all commands in this walkthrough from the Privacy Guard project root: ```bash -cd projects/privacy-guard/examples/regex-scanner +cd projects/privacy-guard ``` +In particular, every `uv run` command below assumes this working directory. + ## 1. Review the scanner configuration `regex-scanner.yaml` configures two entities: @@ -36,22 +38,22 @@ cd projects/privacy-guard/examples/regex-scanner The patterns are intentionally small and understandable. Treat them as a starting point, not comprehensive production detection. -## 2. Configure the gateway address - -Find the host address that Docker can reach: - -```bash -ipconfig getifaddr en0 -``` +## 2. Generate the local gateway config -Replace `REPLACE_WITH_HOST_IP` in `gateway.toml` with that address. If `en0` -does not return an address, find the active interface with: +Generate `gateway.local.toml` with the address of the host interface Docker can +reach: ```bash -route get default | grep interface +HOST_INTERFACE="$(route get default | awk '/interface:/{print $2}')" +HOST_IP="$(ipconfig getifaddr "$HOST_INTERFACE")" +test -n "$HOST_IP" +sed "s/REPLACE_WITH_HOST_IP/$HOST_IP/" \ + examples/regex-scanner/gateway.toml \ + > examples/regex-scanner/gateway.local.toml ``` -Only the checked-out `gateway.toml` is edited. Do not copy it into +The checked-in `gateway.toml` remains an example-specific template. +`gateway.local.toml` is ignored by Git. Do not copy either file into `~/.config/openshell`. ## 3. Start Privacy Guard @@ -64,8 +66,8 @@ interface that Docker must reach. In terminal 1: ```bash -uv run --project ../.. privacy-guard regex \ - --config "$PWD/regex-scanner.yaml" \ +uv run privacy-guard regex \ + --config examples/regex-scanner/regex-scanner.yaml \ --listen 0.0.0.0:50051 ``` @@ -80,16 +82,16 @@ In terminal 2: brew services stop openshell OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" \ -openshell-gateway --config "$PWD/gateway.toml" +openshell-gateway --config "$PWD/examples/regex-scanner/gateway.local.toml" ``` The first command stops the background service so the foreground gateway can use the standard port. The second command reuses the credentials and state created -by the recommended macOS installation, but loads `gateway.toml` from this -directory. Keep it in the foreground; `Server listening` means it is ready. +by the recommended macOS installation, but loads the generated local config. +Keep it in the foreground; `Server listening` means it is ready. -Middleware registration is static. After editing `gateway.toml`, stop this -foreground process with `Ctrl-C` and run the second command again. +Middleware registration is static. After regenerating `gateway.local.toml`, stop +this foreground process with `Ctrl-C` and run the second command again. ## 5. Create the sandbox and run Claude @@ -102,7 +104,7 @@ openshell sandbox create \ --name privacy-guard-regex-lab \ --from base \ --no-auto-providers \ - --policy "$PWD/policy.yaml" \ + --policy "$PWD/examples/regex-scanner/policy.yaml" \ -- claude ``` @@ -126,20 +128,23 @@ findings for both configured entities. ## Change the behavior -To change enforcement, edit `policy.yaml` and set `on_finding.action` to -`observe`, `block`, or `redact`, then apply it without recreating the sandbox: +To change enforcement, edit `examples/regex-scanner/policy.yaml` and set +`on_finding.action` to `observe`, `block`, or `redact`, then apply it without +recreating the sandbox: ```bash -openshell policy set privacy-guard-regex-lab --policy "$PWD/policy.yaml" --wait +openshell policy set privacy-guard-regex-lab \ + --policy "$PWD/examples/regex-scanner/policy.yaml" \ + --wait ``` - `redact` sends `[email]` and `[customer-id]` to Claude. - `observe` records findings but sends the original values. - `block` denies a request containing either configured entity. -To change detection, edit `regex-scanner.yaml`, stop Privacy Guard with `Ctrl-C`, -and run the terminal 1 command again. Scanner configuration is loaded only at -startup. +To change detection, edit `examples/regex-scanner/regex-scanner.yaml`, stop +Privacy Guard with `Ctrl-C`, and run the terminal 1 command again. Scanner +configuration is loaded only at startup. To reconnect later: diff --git a/projects/privacy-guard/examples/regex-scanner/gateway.toml b/projects/privacy-guard/examples/regex-scanner/gateway.toml index 8eed17a..a56d9b2 100644 --- a/projects/privacy-guard/examples/regex-scanner/gateway.toml +++ b/projects/privacy-guard/examples/regex-scanner/gateway.toml @@ -1,6 +1,6 @@ # OpenShell gateway configuration for the built-in regex-scanner example. -# Replace REPLACE_WITH_HOST_IP, then pass this file to openshell-gateway with -# --config. Do not copy it into ~/.config/openshell. +# Generate the ignored gateway.local.toml from this template using the README. +# Do not copy either file into ~/.config/openshell. [openshell] version = 1 diff --git a/projects/privacy-guard/tests/examples/test_email_scanner.py b/projects/privacy-guard/tests/examples/test_email_scanner.py index c5694a4..d0f2063 100644 --- a/projects/privacy-guard/tests/examples/test_email_scanner.py +++ b/projects/privacy-guard/tests/examples/test_email_scanner.py @@ -27,10 +27,15 @@ def test_example_scanner_detects_email_and_server_entry_point_imports() -> None: def test_example_configuration_targets_its_local_middleware() -> None: policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() + gitignore = (EXAMPLE_DIRECTORY / ".gitignore").read_text() readme = (EXAMPLE_DIRECTORY / "README.md").read_text() assert "middleware: privacy-guard-email-scanner" in policy assert 'name = "privacy-guard-email-scanner"' in gateway assert "action: redact" in policy assert "entity_types: [email]" in policy - assert "python middleware_server.py" in readme + assert "cd projects/privacy-guard" in readme + assert 'sed "s/REPLACE_WITH_HOST_IP/$HOST_IP/"' in readme + assert "uv run python examples/email-scanner/middleware_server.py" in readme + assert "examples/email-scanner/gateway.local.toml" in readme + assert "gateway.local.toml" in gitignore diff --git a/projects/privacy-guard/tests/examples/test_regex_scanner.py b/projects/privacy-guard/tests/examples/test_regex_scanner.py index 9be0790..9f59a60 100644 --- a/projects/privacy-guard/tests/examples/test_regex_scanner.py +++ b/projects/privacy-guard/tests/examples/test_regex_scanner.py @@ -25,12 +25,17 @@ def test_example_scanner_detects_email_and_customer_id() -> None: def test_example_configuration_and_walkthrough_are_aligned() -> None: policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() + gitignore = (EXAMPLE_DIRECTORY / ".gitignore").read_text() readme = (EXAMPLE_DIRECTORY / "README.md").read_text() assert "middleware: privacy-guard-regex-scanner" in policy assert 'name = "privacy-guard-regex-scanner"' in gateway assert "action: redact" in policy assert "entity_types: [email, customer-id]" in policy + assert "cd projects/privacy-guard" in readme + assert 'sed "s/REPLACE_WITH_HOST_IP/$HOST_IP/"' in readme assert "privacy-guard regex" in readme - assert '--config "$PWD/regex-scanner.yaml"' in readme + assert "--config examples/regex-scanner/regex-scanner.yaml" in readme + assert "examples/regex-scanner/gateway.local.toml" in readme assert "privacy-guard-regex-lab" in readme + assert "gateway.local.toml" in gitignore From fd54837060815c6248f29e3354ffb5a7674b3a48 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 01:14:10 +0000 Subject: [PATCH 15/82] Simplify example gateway setup --- projects/privacy-guard/examples/email-scanner/README.md | 7 +++---- projects/privacy-guard/examples/regex-scanner/README.md | 7 +++---- .../privacy-guard/tests/examples/test_email_scanner.py | 2 +- .../privacy-guard/tests/examples/test_regex_scanner.py | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/projects/privacy-guard/examples/email-scanner/README.md b/projects/privacy-guard/examples/email-scanner/README.md index 91ac7c4..eb96feb 100644 --- a/projects/privacy-guard/examples/email-scanner/README.md +++ b/projects/privacy-guard/examples/email-scanner/README.md @@ -37,14 +37,13 @@ Generate `gateway.local.toml` with the address of the host interface Docker can reach: ```bash -HOST_INTERFACE="$(route get default | awk '/interface:/{print $2}')" -HOST_IP="$(ipconfig getifaddr "$HOST_INTERFACE")" -test -n "$HOST_IP" -sed "s/REPLACE_WITH_HOST_IP/$HOST_IP/" \ +sed "s/REPLACE_WITH_HOST_IP/$(ipconfig getifaddr en0)/" \ examples/email-scanner/gateway.toml \ > examples/email-scanner/gateway.local.toml ``` +If the active interface is not `en0`, replace it in the command. + The checked-in `gateway.toml` remains an example-specific template. `gateway.local.toml` is ignored by Git. Do not copy either file into `~/.config/openshell`. diff --git a/projects/privacy-guard/examples/regex-scanner/README.md b/projects/privacy-guard/examples/regex-scanner/README.md index 98393a9..70791a0 100644 --- a/projects/privacy-guard/examples/regex-scanner/README.md +++ b/projects/privacy-guard/examples/regex-scanner/README.md @@ -44,14 +44,13 @@ Generate `gateway.local.toml` with the address of the host interface Docker can reach: ```bash -HOST_INTERFACE="$(route get default | awk '/interface:/{print $2}')" -HOST_IP="$(ipconfig getifaddr "$HOST_INTERFACE")" -test -n "$HOST_IP" -sed "s/REPLACE_WITH_HOST_IP/$HOST_IP/" \ +sed "s/REPLACE_WITH_HOST_IP/$(ipconfig getifaddr en0)/" \ examples/regex-scanner/gateway.toml \ > examples/regex-scanner/gateway.local.toml ``` +If the active interface is not `en0`, replace it in the command. + The checked-in `gateway.toml` remains an example-specific template. `gateway.local.toml` is ignored by Git. Do not copy either file into `~/.config/openshell`. diff --git a/projects/privacy-guard/tests/examples/test_email_scanner.py b/projects/privacy-guard/tests/examples/test_email_scanner.py index d0f2063..895a749 100644 --- a/projects/privacy-guard/tests/examples/test_email_scanner.py +++ b/projects/privacy-guard/tests/examples/test_email_scanner.py @@ -35,7 +35,7 @@ def test_example_configuration_targets_its_local_middleware() -> None: assert "action: redact" in policy assert "entity_types: [email]" in policy assert "cd projects/privacy-guard" in readme - assert 'sed "s/REPLACE_WITH_HOST_IP/$HOST_IP/"' in readme + assert 'sed "s/REPLACE_WITH_HOST_IP/$(ipconfig getifaddr en0)/"' in readme assert "uv run python examples/email-scanner/middleware_server.py" in readme assert "examples/email-scanner/gateway.local.toml" in readme assert "gateway.local.toml" in gitignore diff --git a/projects/privacy-guard/tests/examples/test_regex_scanner.py b/projects/privacy-guard/tests/examples/test_regex_scanner.py index 9f59a60..50e16c3 100644 --- a/projects/privacy-guard/tests/examples/test_regex_scanner.py +++ b/projects/privacy-guard/tests/examples/test_regex_scanner.py @@ -33,7 +33,7 @@ def test_example_configuration_and_walkthrough_are_aligned() -> None: assert "action: redact" in policy assert "entity_types: [email, customer-id]" in policy assert "cd projects/privacy-guard" in readme - assert 'sed "s/REPLACE_WITH_HOST_IP/$HOST_IP/"' in readme + assert 'sed "s/REPLACE_WITH_HOST_IP/$(ipconfig getifaddr en0)/"' in readme assert "privacy-guard regex" in readme assert "--config examples/regex-scanner/regex-scanner.yaml" in readme assert "examples/regex-scanner/gateway.local.toml" in readme From f91bf4990c40d9c9a805a1c4ea02d0889eb5f8b2 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 01:22:39 +0000 Subject: [PATCH 16/82] Use OpenShell host alias in examples --- .../examples/email-scanner/.gitignore | 1 - .../examples/email-scanner/README.md | 50 ++++++----------- .../examples/email-scanner/gateway.toml | 6 +- .../examples/regex-scanner/.gitignore | 1 - .../examples/regex-scanner/README.md | 55 +++++++------------ .../examples/regex-scanner/gateway.toml | 6 +- .../tests/examples/test_email_scanner.py | 11 ++-- .../tests/examples/test_regex_scanner.py | 11 ++-- 8 files changed, 52 insertions(+), 89 deletions(-) delete mode 100644 projects/privacy-guard/examples/email-scanner/.gitignore delete mode 100644 projects/privacy-guard/examples/regex-scanner/.gitignore diff --git a/projects/privacy-guard/examples/email-scanner/.gitignore b/projects/privacy-guard/examples/email-scanner/.gitignore deleted file mode 100644 index b223234..0000000 --- a/projects/privacy-guard/examples/email-scanner/.gitignore +++ /dev/null @@ -1 +0,0 @@ -gateway.local.toml diff --git a/projects/privacy-guard/examples/email-scanner/README.md b/projects/privacy-guard/examples/email-scanner/README.md index eb96feb..ce066b6 100644 --- a/projects/privacy-guard/examples/email-scanner/README.md +++ b/projects/privacy-guard/examples/email-scanner/README.md @@ -23,32 +23,16 @@ intended as comprehensive production PII detection. curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh ``` -Run all commands in this walkthrough from the Privacy Guard project root: +Run all commands in this walkthrough from the example directory: ```bash -cd projects/privacy-guard +cd projects/privacy-guard/examples/email-scanner ``` -In particular, every `uv run` command below assumes this working directory. +`uv` automatically discovers the Privacy Guard project in the parent +directories, so the commands do not need a `--project` option. -## 1. Generate the local gateway config - -Generate `gateway.local.toml` with the address of the host interface Docker can -reach: - -```bash -sed "s/REPLACE_WITH_HOST_IP/$(ipconfig getifaddr en0)/" \ - examples/email-scanner/gateway.toml \ - > examples/email-scanner/gateway.local.toml -``` - -If the active interface is not `en0`, replace it in the command. - -The checked-in `gateway.toml` remains an example-specific template. -`gateway.local.toml` is ignored by Git. Do not copy either file into -`~/.config/openshell`. - -## 2. Start Privacy Guard +## 1. Start Privacy Guard This development server uses unauthenticated plaintext gRPC and receives request bodies that may contain sensitive content. Restrict it to a trusted network and @@ -58,13 +42,13 @@ that Docker must reach instead of every interface. In terminal 1: ```bash -uv run python examples/email-scanner/middleware_server.py \ +uv run python middleware_server.py \ --listen 0.0.0.0:50051 ``` Leave it running. -## 3. Run the installed gateway with the example config +## 2. Run the installed gateway with the example config In terminal 2: @@ -72,18 +56,18 @@ In terminal 2: brew services stop openshell OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" \ -openshell-gateway --config "$PWD/examples/email-scanner/gateway.local.toml" +openshell-gateway --config "$PWD/gateway.toml" ``` The first command stops the background service so the foreground gateway can use the standard port. The second command reuses the credentials and state created -by the recommended macOS installation, but loads the generated local config. It -should stay in the foreground; `Server listening` means it is ready. +by the recommended macOS installation, but loads this example's `gateway.toml`. +It should stay in the foreground; `Server listening` means it is ready. -Middleware registration is static. After regenerating `gateway.local.toml`, stop -this foreground process with `Ctrl-C` and run the second command again. +Middleware registration is static. After editing `gateway.toml`, stop this +foreground process with `Ctrl-C` and run the second command again. -## 4. Create the sandbox and run Claude +## 3. Create the sandbox and run Claude In terminal 3: @@ -94,7 +78,7 @@ openshell sandbox create \ --name privacy-guard-lab \ --from base \ --no-auto-providers \ - --policy "$PWD/examples/email-scanner/policy.yaml" \ + --policy "$PWD/policy.yaml" \ -- claude ``` @@ -118,12 +102,12 @@ an email finding. ## Change the behavior -Edit `examples/email-scanner/policy.yaml` and change `on_finding.action` to -`observe`, `block`, or `redact`, then apply it without recreating the sandbox: +Edit `policy.yaml` and change `on_finding.action` to `observe`, `block`, or +`redact`, then apply it without recreating the sandbox: ```bash openshell policy set privacy-guard-lab \ - --policy "$PWD/examples/email-scanner/policy.yaml" \ + --policy "$PWD/policy.yaml" \ --wait ``` diff --git a/projects/privacy-guard/examples/email-scanner/gateway.toml b/projects/privacy-guard/examples/email-scanner/gateway.toml index 713239d..0783572 100644 --- a/projects/privacy-guard/examples/email-scanner/gateway.toml +++ b/projects/privacy-guard/examples/email-scanner/gateway.toml @@ -1,12 +1,12 @@ # OpenShell gateway configuration for the email-scanner example. -# Generate the ignored gateway.local.toml from this template using the README. -# Do not copy either file into ~/.config/openshell. +# Pass this example-specific config directly to openshell-gateway. +# Do not copy it into ~/.config/openshell. [openshell] version = 1 [[openshell.supervisor.middleware]] name = "privacy-guard-email-scanner" -grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051" +grpc_endpoint = "http://host.openshell.internal:50051" max_body_bytes = 4194304 timeout = "5s" diff --git a/projects/privacy-guard/examples/regex-scanner/.gitignore b/projects/privacy-guard/examples/regex-scanner/.gitignore deleted file mode 100644 index b223234..0000000 --- a/projects/privacy-guard/examples/regex-scanner/.gitignore +++ /dev/null @@ -1 +0,0 @@ -gateway.local.toml diff --git a/projects/privacy-guard/examples/regex-scanner/README.md b/projects/privacy-guard/examples/regex-scanner/README.md index 70791a0..368a0a1 100644 --- a/projects/privacy-guard/examples/regex-scanner/README.md +++ b/projects/privacy-guard/examples/regex-scanner/README.md @@ -20,13 +20,14 @@ directory. curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh ``` -Run all commands in this walkthrough from the Privacy Guard project root: +Run all commands in this walkthrough from the example directory: ```bash -cd projects/privacy-guard +cd projects/privacy-guard/examples/regex-scanner ``` -In particular, every `uv run` command below assumes this working directory. +`uv` automatically discovers the Privacy Guard project in the parent +directories, so the commands do not need a `--project` option. ## 1. Review the scanner configuration @@ -38,24 +39,7 @@ In particular, every `uv run` command below assumes this working directory. The patterns are intentionally small and understandable. Treat them as a starting point, not comprehensive production detection. -## 2. Generate the local gateway config - -Generate `gateway.local.toml` with the address of the host interface Docker can -reach: - -```bash -sed "s/REPLACE_WITH_HOST_IP/$(ipconfig getifaddr en0)/" \ - examples/regex-scanner/gateway.toml \ - > examples/regex-scanner/gateway.local.toml -``` - -If the active interface is not `en0`, replace it in the command. - -The checked-in `gateway.toml` remains an example-specific template. -`gateway.local.toml` is ignored by Git. Do not copy either file into -`~/.config/openshell`. - -## 3. Start Privacy Guard +## 2. Start Privacy Guard This development server uses unauthenticated plaintext gRPC and receives request bodies that may contain sensitive content. Restrict it to a trusted network and @@ -66,14 +50,14 @@ In terminal 1: ```bash uv run privacy-guard regex \ - --config examples/regex-scanner/regex-scanner.yaml \ + --config regex-scanner.yaml \ --listen 0.0.0.0:50051 ``` Leave it running. Privacy Guard loads and compiles the scanner configuration before it binds the port. -## 4. Run the installed gateway with the example config +## 3. Run the installed gateway with the example config In terminal 2: @@ -81,18 +65,18 @@ In terminal 2: brew services stop openshell OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" \ -openshell-gateway --config "$PWD/examples/regex-scanner/gateway.local.toml" +openshell-gateway --config "$PWD/gateway.toml" ``` The first command stops the background service so the foreground gateway can use the standard port. The second command reuses the credentials and state created -by the recommended macOS installation, but loads the generated local config. +by the recommended macOS installation, but loads this example's `gateway.toml`. Keep it in the foreground; `Server listening` means it is ready. -Middleware registration is static. After regenerating `gateway.local.toml`, stop -this foreground process with `Ctrl-C` and run the second command again. +Middleware registration is static. After editing `gateway.toml`, stop this +foreground process with `Ctrl-C` and run the second command again. -## 5. Create the sandbox and run Claude +## 4. Create the sandbox and run Claude In terminal 3: @@ -103,7 +87,7 @@ openshell sandbox create \ --name privacy-guard-regex-lab \ --from base \ --no-auto-providers \ - --policy "$PWD/examples/regex-scanner/policy.yaml" \ + --policy "$PWD/policy.yaml" \ -- claude ``` @@ -127,13 +111,12 @@ findings for both configured entities. ## Change the behavior -To change enforcement, edit `examples/regex-scanner/policy.yaml` and set -`on_finding.action` to `observe`, `block`, or `redact`, then apply it without -recreating the sandbox: +To change enforcement, edit `policy.yaml` and set `on_finding.action` to +`observe`, `block`, or `redact`, then apply it without recreating the sandbox: ```bash openshell policy set privacy-guard-regex-lab \ - --policy "$PWD/examples/regex-scanner/policy.yaml" \ + --policy "$PWD/policy.yaml" \ --wait ``` @@ -141,9 +124,9 @@ openshell policy set privacy-guard-regex-lab \ - `observe` records findings but sends the original values. - `block` denies a request containing either configured entity. -To change detection, edit `examples/regex-scanner/regex-scanner.yaml`, stop -Privacy Guard with `Ctrl-C`, and run the terminal 1 command again. Scanner -configuration is loaded only at startup. +To change detection, edit `regex-scanner.yaml`, stop Privacy Guard with `Ctrl-C`, +and run the terminal 1 command again. Scanner configuration is loaded only at +startup. To reconnect later: diff --git a/projects/privacy-guard/examples/regex-scanner/gateway.toml b/projects/privacy-guard/examples/regex-scanner/gateway.toml index a56d9b2..a6a07fe 100644 --- a/projects/privacy-guard/examples/regex-scanner/gateway.toml +++ b/projects/privacy-guard/examples/regex-scanner/gateway.toml @@ -1,12 +1,12 @@ # OpenShell gateway configuration for the built-in regex-scanner example. -# Generate the ignored gateway.local.toml from this template using the README. -# Do not copy either file into ~/.config/openshell. +# Pass this example-specific config directly to openshell-gateway. +# Do not copy it into ~/.config/openshell. [openshell] version = 1 [[openshell.supervisor.middleware]] name = "privacy-guard-regex-scanner" -grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051" +grpc_endpoint = "http://host.openshell.internal:50051" max_body_bytes = 4194304 timeout = "5s" diff --git a/projects/privacy-guard/tests/examples/test_email_scanner.py b/projects/privacy-guard/tests/examples/test_email_scanner.py index 895a749..39b010f 100644 --- a/projects/privacy-guard/tests/examples/test_email_scanner.py +++ b/projects/privacy-guard/tests/examples/test_email_scanner.py @@ -27,15 +27,14 @@ def test_example_scanner_detects_email_and_server_entry_point_imports() -> None: def test_example_configuration_targets_its_local_middleware() -> None: policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() - gitignore = (EXAMPLE_DIRECTORY / ".gitignore").read_text() readme = (EXAMPLE_DIRECTORY / "README.md").read_text() assert "middleware: privacy-guard-email-scanner" in policy assert 'name = "privacy-guard-email-scanner"' in gateway + assert 'grpc_endpoint = "http://host.openshell.internal:50051"' in gateway assert "action: redact" in policy assert "entity_types: [email]" in policy - assert "cd projects/privacy-guard" in readme - assert 'sed "s/REPLACE_WITH_HOST_IP/$(ipconfig getifaddr en0)/"' in readme - assert "uv run python examples/email-scanner/middleware_server.py" in readme - assert "examples/email-scanner/gateway.local.toml" in readme - assert "gateway.local.toml" in gitignore + assert "cd projects/privacy-guard/examples/email-scanner" in readme + assert "uv run python middleware_server.py" in readme + assert 'openshell-gateway --config "$PWD/gateway.toml"' in readme + assert "uv run --project" not in readme diff --git a/projects/privacy-guard/tests/examples/test_regex_scanner.py b/projects/privacy-guard/tests/examples/test_regex_scanner.py index 50e16c3..cbb5f61 100644 --- a/projects/privacy-guard/tests/examples/test_regex_scanner.py +++ b/projects/privacy-guard/tests/examples/test_regex_scanner.py @@ -25,17 +25,16 @@ def test_example_scanner_detects_email_and_customer_id() -> None: def test_example_configuration_and_walkthrough_are_aligned() -> None: policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() - gitignore = (EXAMPLE_DIRECTORY / ".gitignore").read_text() readme = (EXAMPLE_DIRECTORY / "README.md").read_text() assert "middleware: privacy-guard-regex-scanner" in policy assert 'name = "privacy-guard-regex-scanner"' in gateway + assert 'grpc_endpoint = "http://host.openshell.internal:50051"' in gateway assert "action: redact" in policy assert "entity_types: [email, customer-id]" in policy - assert "cd projects/privacy-guard" in readme - assert 'sed "s/REPLACE_WITH_HOST_IP/$(ipconfig getifaddr en0)/"' in readme + assert "cd projects/privacy-guard/examples/regex-scanner" in readme assert "privacy-guard regex" in readme - assert "--config examples/regex-scanner/regex-scanner.yaml" in readme - assert "examples/regex-scanner/gateway.local.toml" in readme + assert "--config regex-scanner.yaml" in readme + assert 'openshell-gateway --config "$PWD/gateway.toml"' in readme assert "privacy-guard-regex-lab" in readme - assert "gateway.local.toml" in gitignore + assert "uv run --project" not in readme From fe422d94a532abccd6f385eec20033c52fb58b71 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 01:29:21 +0000 Subject: [PATCH 17/82] Log Privacy Guard CLI startup --- .../src/privacy_guard/service/server.py | 13 +++++++ .../tests/service/test_server.py | 36 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index a4c8107..6f685f2 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -8,6 +8,7 @@ from __future__ import annotations import asyncio +import logging from pathlib import Path from typing import Annotated @@ -77,6 +78,10 @@ async def serve( @app.callback() def main() -> None: """Select one of Privacy Guard's built-in scanners.""" + logging.basicConfig( + level=logging.INFO, + format="%(levelname)s %(name)s: %(message)s", + ) @app.command("regex") @@ -105,11 +110,19 @@ def run_regex( profile, scanner_name=scanner_name, ) + _LOGGER.info( + "privacy_guard_server_starting scanner=%s listen=%s", + scanner.scanner_name, + listen, + ) MiddlewareServer(scanner=scanner).serve(listen) except PrivacyGuardError as error: typer.echo(str(error), err=True) raise typer.Exit(code=1) from None +_LOGGER = logging.getLogger(__name__) + + if __name__ == "__main__": app() diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 2b10230..86fe932 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -1,3 +1,4 @@ +import logging from collections.abc import Sequence from pathlib import Path @@ -200,6 +201,41 @@ def load_scanner( assert calls == [("scanner-config.yaml", None, "custom-regex")] +def test_cli_logs_server_start_without_configuration_details( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + scanner = DeterministicEmailScanner( + ScannerConfig(name="custom-regex", entity_types=frozenset({"email"})) + ) + monkeypatch.setattr( + server_module.RegexScanner, + "from_yaml", + lambda *args, **kwargs: scanner, + ) + monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) + + with caplog.at_level(logging.INFO, logger="privacy_guard.service.server"): + result = CliRunner().invoke( + app, + [ + "regex", + "--config", + "sensitive-config-8472.yaml", + "--listen", + "127.0.0.1:50051", + "--scanner-name", + "custom-regex", + ], + ) + + assert result.exit_code == 0 + assert "privacy_guard_server_starting" in caplog.text + assert "scanner=custom-regex" in caplog.text + assert "listen=127.0.0.1:50051" in caplog.text + assert "8472" not in caplog.text + + def test_cli_always_requires_scanner_configuration() -> None: result = CliRunner().invoke(app, ["regex"]) From 05d66fec3db7d2621355a2e0273571e8d0a0c2f2 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 01:31:48 +0000 Subject: [PATCH 18/82] Fix example middleware endpoint --- .../privacy-guard/examples/email-scanner/README.md | 11 +++++++---- .../privacy-guard/examples/email-scanner/gateway.toml | 2 +- .../privacy-guard/examples/regex-scanner/README.md | 11 +++++++---- .../privacy-guard/examples/regex-scanner/gateway.toml | 2 +- .../tests/examples/test_email_scanner.py | 3 ++- .../tests/examples/test_regex_scanner.py | 3 ++- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/projects/privacy-guard/examples/email-scanner/README.md b/projects/privacy-guard/examples/email-scanner/README.md index ce066b6..402f7ad 100644 --- a/projects/privacy-guard/examples/email-scanner/README.md +++ b/projects/privacy-guard/examples/email-scanner/README.md @@ -35,15 +35,14 @@ directories, so the commands do not need a `--project` option. ## 1. Start Privacy Guard This development server uses unauthenticated plaintext gRPC and receives request -bodies that may contain sensitive content. Restrict it to a trusted network and -firewall the port. When possible, bind `--listen` to the specific host interface -that Docker must reach instead of every interface. +bodies that may contain sensitive content. It listens only on loopback because +the gateway and Privacy Guard both run on the host. In terminal 1: ```bash uv run python middleware_server.py \ - --listen 0.0.0.0:50051 + --listen 127.0.0.1:50051 ``` Leave it running. @@ -64,6 +63,10 @@ the standard port. The second command reuses the credentials and state created by the recommended macOS installation, but loads this example's `gateway.toml`. It should stay in the foreground; `Server listening` means it is ready. +The gateway connects to Privacy Guard over host loopback. The +`host.openshell.internal` hostname is only needed when a process inside a +sandbox connects back to a service on the host. + Middleware registration is static. After editing `gateway.toml`, stop this foreground process with `Ctrl-C` and run the second command again. diff --git a/projects/privacy-guard/examples/email-scanner/gateway.toml b/projects/privacy-guard/examples/email-scanner/gateway.toml index 0783572..d3689b6 100644 --- a/projects/privacy-guard/examples/email-scanner/gateway.toml +++ b/projects/privacy-guard/examples/email-scanner/gateway.toml @@ -7,6 +7,6 @@ version = 1 [[openshell.supervisor.middleware]] name = "privacy-guard-email-scanner" -grpc_endpoint = "http://host.openshell.internal:50051" +grpc_endpoint = "http://127.0.0.1:50051" max_body_bytes = 4194304 timeout = "5s" diff --git a/projects/privacy-guard/examples/regex-scanner/README.md b/projects/privacy-guard/examples/regex-scanner/README.md index 368a0a1..8cd36c3 100644 --- a/projects/privacy-guard/examples/regex-scanner/README.md +++ b/projects/privacy-guard/examples/regex-scanner/README.md @@ -42,16 +42,15 @@ starting point, not comprehensive production detection. ## 2. Start Privacy Guard This development server uses unauthenticated plaintext gRPC and receives request -bodies that may contain sensitive content. Restrict it to a trusted network and -firewall the port. When possible, replace `0.0.0.0` with the specific host -interface that Docker must reach. +bodies that may contain sensitive content. It listens only on loopback because +the gateway and Privacy Guard both run on the host. In terminal 1: ```bash uv run privacy-guard regex \ --config regex-scanner.yaml \ - --listen 0.0.0.0:50051 + --listen 127.0.0.1:50051 ``` Leave it running. Privacy Guard loads and compiles the scanner configuration @@ -73,6 +72,10 @@ the standard port. The second command reuses the credentials and state created by the recommended macOS installation, but loads this example's `gateway.toml`. Keep it in the foreground; `Server listening` means it is ready. +The gateway connects to Privacy Guard over host loopback. The +`host.openshell.internal` hostname is only needed when a process inside a +sandbox connects back to a service on the host. + Middleware registration is static. After editing `gateway.toml`, stop this foreground process with `Ctrl-C` and run the second command again. diff --git a/projects/privacy-guard/examples/regex-scanner/gateway.toml b/projects/privacy-guard/examples/regex-scanner/gateway.toml index a6a07fe..ccc9fe8 100644 --- a/projects/privacy-guard/examples/regex-scanner/gateway.toml +++ b/projects/privacy-guard/examples/regex-scanner/gateway.toml @@ -7,6 +7,6 @@ version = 1 [[openshell.supervisor.middleware]] name = "privacy-guard-regex-scanner" -grpc_endpoint = "http://host.openshell.internal:50051" +grpc_endpoint = "http://127.0.0.1:50051" max_body_bytes = 4194304 timeout = "5s" diff --git a/projects/privacy-guard/tests/examples/test_email_scanner.py b/projects/privacy-guard/tests/examples/test_email_scanner.py index 39b010f..18598cd 100644 --- a/projects/privacy-guard/tests/examples/test_email_scanner.py +++ b/projects/privacy-guard/tests/examples/test_email_scanner.py @@ -31,10 +31,11 @@ def test_example_configuration_targets_its_local_middleware() -> None: assert "middleware: privacy-guard-email-scanner" in policy assert 'name = "privacy-guard-email-scanner"' in gateway - assert 'grpc_endpoint = "http://host.openshell.internal:50051"' in gateway + assert 'grpc_endpoint = "http://127.0.0.1:50051"' in gateway assert "action: redact" in policy assert "entity_types: [email]" in policy assert "cd projects/privacy-guard/examples/email-scanner" in readme assert "uv run python middleware_server.py" in readme + assert "--listen 127.0.0.1:50051" in readme assert 'openshell-gateway --config "$PWD/gateway.toml"' in readme assert "uv run --project" not in readme diff --git a/projects/privacy-guard/tests/examples/test_regex_scanner.py b/projects/privacy-guard/tests/examples/test_regex_scanner.py index cbb5f61..75807bc 100644 --- a/projects/privacy-guard/tests/examples/test_regex_scanner.py +++ b/projects/privacy-guard/tests/examples/test_regex_scanner.py @@ -29,12 +29,13 @@ def test_example_configuration_and_walkthrough_are_aligned() -> None: assert "middleware: privacy-guard-regex-scanner" in policy assert 'name = "privacy-guard-regex-scanner"' in gateway - assert 'grpc_endpoint = "http://host.openshell.internal:50051"' in gateway + assert 'grpc_endpoint = "http://127.0.0.1:50051"' in gateway assert "action: redact" in policy assert "entity_types: [email, customer-id]" in policy assert "cd projects/privacy-guard/examples/regex-scanner" in readme assert "privacy-guard regex" in readme assert "--config regex-scanner.yaml" in readme + assert "--listen 127.0.0.1:50051" in readme assert 'openshell-gateway --config "$PWD/gateway.toml"' in readme assert "privacy-guard-regex-lab" in readme assert "uv run --project" not in readme From 1a295e827afe6ece4ed6557f21a4a474ea405758 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 01:36:29 +0000 Subject: [PATCH 19/82] Document local gateway registration --- .../examples/email-scanner/README.md | 20 ++++++++++++++----- .../examples/regex-scanner/README.md | 19 +++++++++++++----- .../tests/examples/test_email_scanner.py | 3 +++ .../tests/examples/test_regex_scanner.py | 3 +++ 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/projects/privacy-guard/examples/email-scanner/README.md b/projects/privacy-guard/examples/email-scanner/README.md index 402f7ad..d72d043 100644 --- a/projects/privacy-guard/examples/email-scanner/README.md +++ b/projects/privacy-guard/examples/email-scanner/README.md @@ -3,9 +3,10 @@ This self-contained example supplies a deterministic custom email scanner, its Privacy Guard server entry point, and the OpenShell gateway and sandbox policy needed to try redaction with Claude Code. It temporarily runs the installed -OpenShell gateway with the config from this directory. It does not create or modify -`~/.config/openshell/gateway.toml`, and it does not create a project-local state -directory. +OpenShell gateway with the config from this directory. It does not create or +modify `~/.config/openshell/gateway.toml` or create a project-local state +directory. It registers the standard local gateway endpoint with the OpenShell +CLI. The `EmailScanner` class in `middleware_server.py` demonstrates how to implement the scanner extension contract directly. It detects email-shaped text in Claude @@ -75,6 +76,11 @@ foreground process with `Ctrl-C` and run the second command again. In terminal 3: ```bash +openshell gateway add \ + https://127.0.0.1:17670 \ + --local \ + --name openshell + openshell status openshell sandbox create \ @@ -85,6 +91,10 @@ openshell sandbox create \ -- claude ``` +`gateway add` saves the endpoint as the CLI's active gateway and refreshes its +package-managed mTLS credentials. The registration continues to target the +normal local gateway after this walkthrough. + Choose Claude Code's subscription-account login and complete authentication. Then enter: @@ -141,8 +151,8 @@ background gateway: brew services start openshell ``` -Stop Privacy Guard with `Ctrl-C` in terminal 1. No default OpenShell config was -changed. +Stop Privacy Guard with `Ctrl-C` in terminal 1. The example-specific gateway +configuration was never installed as the default configuration. This example uses Claude Code because subscription prompts are sent in inspectable HTTP request bodies. ChatGPT-subscription Codex currently sends prompts in diff --git a/projects/privacy-guard/examples/regex-scanner/README.md b/projects/privacy-guard/examples/regex-scanner/README.md index 8cd36c3..e196fbe 100644 --- a/projects/privacy-guard/examples/regex-scanner/README.md +++ b/projects/privacy-guard/examples/regex-scanner/README.md @@ -6,9 +6,9 @@ email address and customer ID before Claude Code sends a request to Anthropic. It is intended to be worked through by hand. The example temporarily runs the installed OpenShell gateway with the config in -this directory. It does not create or modify -`~/.config/openshell/gateway.toml`, and it does not create a project-local state -directory. +this directory. It does not create or modify `~/.config/openshell/gateway.toml` +or create a project-local state directory. It registers the standard local +gateway endpoint with the OpenShell CLI. ## Prerequisites @@ -84,6 +84,11 @@ foreground process with `Ctrl-C` and run the second command again. In terminal 3: ```bash +openshell gateway add \ + https://127.0.0.1:17670 \ + --local \ + --name openshell + openshell status openshell sandbox create \ @@ -94,6 +99,10 @@ openshell sandbox create \ -- claude ``` +`gateway add` saves the endpoint as the CLI's active gateway and refreshes its +package-managed mTLS credentials. The registration continues to target the +normal local gateway after this walkthrough. + Choose Claude Code's subscription-account login and complete authentication. Then enter: @@ -154,8 +163,8 @@ background gateway: brew services start openshell ``` -Stop Privacy Guard with `Ctrl-C` in terminal 1. No default OpenShell config was -changed. +Stop Privacy Guard with `Ctrl-C` in terminal 1. The example-specific gateway +configuration was never installed as the default configuration. This example uses Claude Code because subscription prompts are sent in inspectable HTTP request bodies. ChatGPT-subscription Codex currently sends diff --git a/projects/privacy-guard/tests/examples/test_email_scanner.py b/projects/privacy-guard/tests/examples/test_email_scanner.py index 18598cd..d697df4 100644 --- a/projects/privacy-guard/tests/examples/test_email_scanner.py +++ b/projects/privacy-guard/tests/examples/test_email_scanner.py @@ -38,4 +38,7 @@ def test_example_configuration_targets_its_local_middleware() -> None: assert "uv run python middleware_server.py" in readme assert "--listen 127.0.0.1:50051" in readme assert 'openshell-gateway --config "$PWD/gateway.toml"' in readme + assert "openshell gateway add" in readme + assert "https://127.0.0.1:17670" in readme + assert "--name openshell" in readme assert "uv run --project" not in readme diff --git a/projects/privacy-guard/tests/examples/test_regex_scanner.py b/projects/privacy-guard/tests/examples/test_regex_scanner.py index 75807bc..2cd4b58 100644 --- a/projects/privacy-guard/tests/examples/test_regex_scanner.py +++ b/projects/privacy-guard/tests/examples/test_regex_scanner.py @@ -37,5 +37,8 @@ def test_example_configuration_and_walkthrough_are_aligned() -> None: assert "--config regex-scanner.yaml" in readme assert "--listen 127.0.0.1:50051" in readme assert 'openshell-gateway --config "$PWD/gateway.toml"' in readme + assert "openshell gateway add" in readme + assert "https://127.0.0.1:17670" in readme + assert "--name openshell" in readme assert "privacy-guard-regex-lab" in readme assert "uv run --project" not in readme From bd2e32a3530da4af108c21cda423533ccd54ab5d Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 01:41:11 +0000 Subject: [PATCH 20/82] Make example middleware reachable from sandboxes --- .../examples/email-scanner/.gitignore | 1 + .../examples/email-scanner/README.md | 39 ++++++++++++------- .../examples/email-scanner/gateway.toml | 2 +- .../email-scanner/generate_gateway_config.py | 29 ++++++++++++++ .../examples/regex-scanner/.gitignore | 1 + .../examples/regex-scanner/README.md | 39 ++++++++++++------- .../examples/regex-scanner/gateway.toml | 2 +- .../regex-scanner/generate_gateway_config.py | 29 ++++++++++++++ .../tests/examples/test_email_scanner.py | 10 +++-- .../tests/examples/test_regex_scanner.py | 10 +++-- 10 files changed, 124 insertions(+), 38 deletions(-) create mode 100644 projects/privacy-guard/examples/email-scanner/.gitignore create mode 100644 projects/privacy-guard/examples/email-scanner/generate_gateway_config.py create mode 100644 projects/privacy-guard/examples/regex-scanner/.gitignore create mode 100644 projects/privacy-guard/examples/regex-scanner/generate_gateway_config.py diff --git a/projects/privacy-guard/examples/email-scanner/.gitignore b/projects/privacy-guard/examples/email-scanner/.gitignore new file mode 100644 index 0000000..b223234 --- /dev/null +++ b/projects/privacy-guard/examples/email-scanner/.gitignore @@ -0,0 +1 @@ +gateway.local.toml diff --git a/projects/privacy-guard/examples/email-scanner/README.md b/projects/privacy-guard/examples/email-scanner/README.md index d72d043..5710c3e 100644 --- a/projects/privacy-guard/examples/email-scanner/README.md +++ b/projects/privacy-guard/examples/email-scanner/README.md @@ -33,22 +33,34 @@ cd projects/privacy-guard/examples/email-scanner `uv` automatically discovers the Privacy Guard project in the parent directories, so the commands do not need a `--project` option. -## 1. Start Privacy Guard +## 1. Generate the local gateway config + +Generate a local config containing a host address that both the gateway and +sandboxes can reach: + +```bash +uv run python generate_gateway_config.py +``` + +This writes the ignored `gateway.local.toml`. The checked-in `gateway.toml` +remains a reusable template. + +## 2. Start Privacy Guard This development server uses unauthenticated plaintext gRPC and receives request -bodies that may contain sensitive content. It listens only on loopback because -the gateway and Privacy Guard both run on the host. +bodies that may contain sensitive content. Restrict access to port 50051 with a +host firewall because sandboxes must be able to reach it. In terminal 1: ```bash uv run python middleware_server.py \ - --listen 127.0.0.1:50051 + --listen 0.0.0.0:50051 ``` Leave it running. -## 2. Run the installed gateway with the example config +## 3. Run the installed gateway with the example config In terminal 2: @@ -56,22 +68,19 @@ In terminal 2: brew services stop openshell OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" \ -openshell-gateway --config "$PWD/gateway.toml" +openshell-gateway --config "$PWD/gateway.local.toml" ``` The first command stops the background service so the foreground gateway can use the standard port. The second command reuses the credentials and state created -by the recommended macOS installation, but loads this example's `gateway.toml`. -It should stay in the foreground; `Server listening` means it is ready. - -The gateway connects to Privacy Guard over host loopback. The -`host.openshell.internal` hostname is only needed when a process inside a -sandbox connects back to a service on the host. +by the recommended macOS installation, but loads this example's generated +`gateway.local.toml`. It should stay in the foreground; `Server listening` means +it is ready. -Middleware registration is static. After editing `gateway.toml`, stop this -foreground process with `Ctrl-C` and run the second command again. +Middleware registration is static. After regenerating `gateway.local.toml`, stop +this foreground process with `Ctrl-C` and run the second command again. -## 3. Create the sandbox and run Claude +## 4. Create the sandbox and run Claude In terminal 3: diff --git a/projects/privacy-guard/examples/email-scanner/gateway.toml b/projects/privacy-guard/examples/email-scanner/gateway.toml index d3689b6..510586c 100644 --- a/projects/privacy-guard/examples/email-scanner/gateway.toml +++ b/projects/privacy-guard/examples/email-scanner/gateway.toml @@ -7,6 +7,6 @@ version = 1 [[openshell.supervisor.middleware]] name = "privacy-guard-email-scanner" -grpc_endpoint = "http://127.0.0.1:50051" +grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051" max_body_bytes = 4194304 timeout = "5s" diff --git a/projects/privacy-guard/examples/email-scanner/generate_gateway_config.py b/projects/privacy-guard/examples/email-scanner/generate_gateway_config.py new file mode 100644 index 0000000..6a4081e --- /dev/null +++ b/projects/privacy-guard/examples/email-scanner/generate_gateway_config.py @@ -0,0 +1,29 @@ +"""Generate a gateway config that both the host and sandboxes can reach.""" + +from __future__ import annotations + +import socket +from pathlib import Path + + +def main() -> None: + example_directory = Path(__file__).parent + template = (example_directory / "gateway.toml").read_text() + placeholder = "REPLACE_WITH_HOST_IP" + if template.count(placeholder) != 1: + raise RuntimeError("gateway.toml must contain exactly one host placeholder") + + host_ip = _discover_host_ipv4() + output_path = example_directory / "gateway.local.toml" + output_path.write_text(template.replace(placeholder, host_ip)) + print(f"Wrote {output_path.name} with middleware host {host_ip}") + + +def _discover_host_ipv4() -> str: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe: + probe.connect(("192.0.2.1", 80)) + return str(probe.getsockname()[0]) + + +if __name__ == "__main__": + main() diff --git a/projects/privacy-guard/examples/regex-scanner/.gitignore b/projects/privacy-guard/examples/regex-scanner/.gitignore new file mode 100644 index 0000000..b223234 --- /dev/null +++ b/projects/privacy-guard/examples/regex-scanner/.gitignore @@ -0,0 +1 @@ +gateway.local.toml diff --git a/projects/privacy-guard/examples/regex-scanner/README.md b/projects/privacy-guard/examples/regex-scanner/README.md index e196fbe..28289ab 100644 --- a/projects/privacy-guard/examples/regex-scanner/README.md +++ b/projects/privacy-guard/examples/regex-scanner/README.md @@ -39,24 +39,36 @@ directories, so the commands do not need a `--project` option. The patterns are intentionally small and understandable. Treat them as a starting point, not comprehensive production detection. -## 2. Start Privacy Guard +## 2. Generate the local gateway config + +Generate a local config containing a host address that both the gateway and +sandboxes can reach: + +```bash +uv run python generate_gateway_config.py +``` + +This writes the ignored `gateway.local.toml`. The checked-in `gateway.toml` +remains a reusable template. + +## 3. Start Privacy Guard This development server uses unauthenticated plaintext gRPC and receives request -bodies that may contain sensitive content. It listens only on loopback because -the gateway and Privacy Guard both run on the host. +bodies that may contain sensitive content. Restrict access to port 50051 with a +host firewall because sandboxes must be able to reach it. In terminal 1: ```bash uv run privacy-guard regex \ --config regex-scanner.yaml \ - --listen 127.0.0.1:50051 + --listen 0.0.0.0:50051 ``` Leave it running. Privacy Guard loads and compiles the scanner configuration before it binds the port. -## 3. Run the installed gateway with the example config +## 4. Run the installed gateway with the example config In terminal 2: @@ -64,22 +76,19 @@ In terminal 2: brew services stop openshell OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" \ -openshell-gateway --config "$PWD/gateway.toml" +openshell-gateway --config "$PWD/gateway.local.toml" ``` The first command stops the background service so the foreground gateway can use the standard port. The second command reuses the credentials and state created -by the recommended macOS installation, but loads this example's `gateway.toml`. -Keep it in the foreground; `Server listening` means it is ready. - -The gateway connects to Privacy Guard over host loopback. The -`host.openshell.internal` hostname is only needed when a process inside a -sandbox connects back to a service on the host. +by the recommended macOS installation, but loads this example's generated +`gateway.local.toml`. Keep it in the foreground; `Server listening` means it is +ready. -Middleware registration is static. After editing `gateway.toml`, stop this -foreground process with `Ctrl-C` and run the second command again. +Middleware registration is static. After regenerating `gateway.local.toml`, stop +this foreground process with `Ctrl-C` and run the second command again. -## 4. Create the sandbox and run Claude +## 5. Create the sandbox and run Claude In terminal 3: diff --git a/projects/privacy-guard/examples/regex-scanner/gateway.toml b/projects/privacy-guard/examples/regex-scanner/gateway.toml index ccc9fe8..8a6aaf3 100644 --- a/projects/privacy-guard/examples/regex-scanner/gateway.toml +++ b/projects/privacy-guard/examples/regex-scanner/gateway.toml @@ -7,6 +7,6 @@ version = 1 [[openshell.supervisor.middleware]] name = "privacy-guard-regex-scanner" -grpc_endpoint = "http://127.0.0.1:50051" +grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051" max_body_bytes = 4194304 timeout = "5s" diff --git a/projects/privacy-guard/examples/regex-scanner/generate_gateway_config.py b/projects/privacy-guard/examples/regex-scanner/generate_gateway_config.py new file mode 100644 index 0000000..6a4081e --- /dev/null +++ b/projects/privacy-guard/examples/regex-scanner/generate_gateway_config.py @@ -0,0 +1,29 @@ +"""Generate a gateway config that both the host and sandboxes can reach.""" + +from __future__ import annotations + +import socket +from pathlib import Path + + +def main() -> None: + example_directory = Path(__file__).parent + template = (example_directory / "gateway.toml").read_text() + placeholder = "REPLACE_WITH_HOST_IP" + if template.count(placeholder) != 1: + raise RuntimeError("gateway.toml must contain exactly one host placeholder") + + host_ip = _discover_host_ipv4() + output_path = example_directory / "gateway.local.toml" + output_path.write_text(template.replace(placeholder, host_ip)) + print(f"Wrote {output_path.name} with middleware host {host_ip}") + + +def _discover_host_ipv4() -> str: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe: + probe.connect(("192.0.2.1", 80)) + return str(probe.getsockname()[0]) + + +if __name__ == "__main__": + main() diff --git a/projects/privacy-guard/tests/examples/test_email_scanner.py b/projects/privacy-guard/tests/examples/test_email_scanner.py index d697df4..3fa0ae9 100644 --- a/projects/privacy-guard/tests/examples/test_email_scanner.py +++ b/projects/privacy-guard/tests/examples/test_email_scanner.py @@ -27,17 +27,21 @@ def test_example_scanner_detects_email_and_server_entry_point_imports() -> None: def test_example_configuration_targets_its_local_middleware() -> None: policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() + gitignore = (EXAMPLE_DIRECTORY / ".gitignore").read_text() readme = (EXAMPLE_DIRECTORY / "README.md").read_text() assert "middleware: privacy-guard-email-scanner" in policy assert 'name = "privacy-guard-email-scanner"' in gateway - assert 'grpc_endpoint = "http://127.0.0.1:50051"' in gateway + assert 'grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051"' in gateway + assert "gateway.local.toml" in gitignore + assert (EXAMPLE_DIRECTORY / "generate_gateway_config.py").is_file() assert "action: redact" in policy assert "entity_types: [email]" in policy assert "cd projects/privacy-guard/examples/email-scanner" in readme assert "uv run python middleware_server.py" in readme - assert "--listen 127.0.0.1:50051" in readme - assert 'openshell-gateway --config "$PWD/gateway.toml"' in readme + assert "uv run python generate_gateway_config.py" in readme + assert "--listen 0.0.0.0:50051" in readme + assert 'openshell-gateway --config "$PWD/gateway.local.toml"' in readme assert "openshell gateway add" in readme assert "https://127.0.0.1:17670" in readme assert "--name openshell" in readme diff --git a/projects/privacy-guard/tests/examples/test_regex_scanner.py b/projects/privacy-guard/tests/examples/test_regex_scanner.py index 2cd4b58..98004d3 100644 --- a/projects/privacy-guard/tests/examples/test_regex_scanner.py +++ b/projects/privacy-guard/tests/examples/test_regex_scanner.py @@ -25,18 +25,22 @@ def test_example_scanner_detects_email_and_customer_id() -> None: def test_example_configuration_and_walkthrough_are_aligned() -> None: policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() + gitignore = (EXAMPLE_DIRECTORY / ".gitignore").read_text() readme = (EXAMPLE_DIRECTORY / "README.md").read_text() assert "middleware: privacy-guard-regex-scanner" in policy assert 'name = "privacy-guard-regex-scanner"' in gateway - assert 'grpc_endpoint = "http://127.0.0.1:50051"' in gateway + assert 'grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051"' in gateway + assert "gateway.local.toml" in gitignore + assert (EXAMPLE_DIRECTORY / "generate_gateway_config.py").is_file() assert "action: redact" in policy assert "entity_types: [email, customer-id]" in policy assert "cd projects/privacy-guard/examples/regex-scanner" in readme assert "privacy-guard regex" in readme assert "--config regex-scanner.yaml" in readme - assert "--listen 127.0.0.1:50051" in readme - assert 'openshell-gateway --config "$PWD/gateway.toml"' in readme + assert "uv run python generate_gateway_config.py" in readme + assert "--listen 0.0.0.0:50051" in readme + assert 'openshell-gateway --config "$PWD/gateway.local.toml"' in readme assert "openshell gateway add" in readme assert "https://127.0.0.1:17670" in readme assert "--name openshell" in readme From d008566ec496ecd0444d77c903c0286412f957eb Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 01:55:54 +0000 Subject: [PATCH 21/82] Disable telemetry in regex walkthrough --- projects/privacy-guard/examples/regex-scanner/README.md | 6 +++++- projects/privacy-guard/tests/examples/test_regex_scanner.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/privacy-guard/examples/regex-scanner/README.md b/projects/privacy-guard/examples/regex-scanner/README.md index 28289ab..d1850a2 100644 --- a/projects/privacy-guard/examples/regex-scanner/README.md +++ b/projects/privacy-guard/examples/regex-scanner/README.md @@ -105,13 +105,17 @@ openshell sandbox create \ --from base \ --no-auto-providers \ --policy "$PWD/policy.yaml" \ - -- claude + -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude ``` `gateway add` saves the endpoint as the CLI's active gateway and refreshes its package-managed mTLS credentials. The registration continues to target the normal local gateway after this walkthrough. +Disabling nonessential Claude Code traffic keeps telemetry and error-reporting +batches out of this focused redaction exercise. Privacy Guard still processes +the model requests sent to `api.anthropic.com`. + Choose Claude Code's subscription-account login and complete authentication. Then enter: diff --git a/projects/privacy-guard/tests/examples/test_regex_scanner.py b/projects/privacy-guard/tests/examples/test_regex_scanner.py index 98004d3..bde574b 100644 --- a/projects/privacy-guard/tests/examples/test_regex_scanner.py +++ b/projects/privacy-guard/tests/examples/test_regex_scanner.py @@ -44,5 +44,6 @@ def test_example_configuration_and_walkthrough_are_aligned() -> None: assert "openshell gateway add" in readme assert "https://127.0.0.1:17670" in readme assert "--name openshell" in readme + assert "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude" in readme assert "privacy-guard-regex-lab" in readme assert "uv run --project" not in readme From 786c1ff5c43530eaf532cd1d0804a280cb6b515e Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 02:14:41 +0000 Subject: [PATCH 22/82] Add request pipeline debug logging --- .../examples/regex-scanner/README.md | 27 +++++++ .../src/privacy_guard/processor.py | 78 +++++++++++++++++++ .../src/privacy_guard/service/server.py | 52 +++++++++++-- .../src/privacy_guard/service/servicer.py | 13 +++- .../tests/service/test_server.py | 48 ++++++++++++ .../privacy-guard/tests/test_processor.py | 51 ++++++++++++ 6 files changed, 263 insertions(+), 6 deletions(-) diff --git a/projects/privacy-guard/examples/regex-scanner/README.md b/projects/privacy-guard/examples/regex-scanner/README.md index d1850a2..d40fb4f 100644 --- a/projects/privacy-guard/examples/regex-scanner/README.md +++ b/projects/privacy-guard/examples/regex-scanner/README.md @@ -68,6 +68,33 @@ uv run privacy-guard regex \ Leave it running. Privacy Guard loads and compiles the scanner configuration before it binds the port. +To investigate request latency without logging request content, restart Privacy +Guard with `--debug` before the `regex` command: + +```bash +uv run privacy-guard --debug regex \ + --config regex-scanner.yaml \ + --listen 0.0.0.0:50051 +``` + +Debug output reports body size, text-block count, scanned character count, and +normalize, scan, and reconstruction timings. It never reports body content, +matched values, JSON paths, headers, or scanner configuration. + +To compare the complete body received by Privacy Guard with the body it returns +to OpenShell, use the explicit content-logging mode instead: + +```bash +uv run privacy-guard --debug-log-content regex \ + --config regex-scanner.yaml \ + --listen 0.0.0.0:50051 +``` + +Look for `stage=received` and `stage=forwarded` entries with the same +`request_id`. This mode also enables phase timings. It logs secrets by design, +so use it only for local debugging and disable it after capturing the failing +request. + ## 4. Run the installed gateway with the example config In terminal 2: diff --git a/projects/privacy-guard/src/privacy_guard/processor.py b/projects/privacy-guard/src/privacy_guard/processor.py index dffc3ae..133465a 100644 --- a/projects/privacy-guard/src/privacy_guard/processor.py +++ b/projects/privacy-guard/src/privacy_guard/processor.py @@ -2,7 +2,9 @@ from __future__ import annotations +import logging import math +import time from collections.abc import Mapping, Sequence from dataclasses import dataclass from string import Formatter @@ -64,6 +66,7 @@ def __init__( format_handlers: Mapping[str, FormatHandler] = DEFAULT_FORMAT_HANDLERS, *, scan_timeout_seconds: float = DEFAULT_SCAN_TIMEOUT_SECONDS, + log_request_content: bool = False, ) -> None: if ( isinstance(scan_timeout_seconds, bool) @@ -89,6 +92,7 @@ def __init__( self._scanners: tuple[Scanner[ScannerConfig], ...] = scanner_tuple self._supported_entities = frozenset(supported_entities) self._scan_timeout_seconds = float(scan_timeout_seconds) + self._log_request_content = log_request_content self._format_handlers: dict[str, FormatHandler] = {} for format_name, handler in format_handlers.items(): if not isinstance(handler, FormatHandler): @@ -102,17 +106,69 @@ def validate_policy_config(self, policy_config: PolicyConfig) -> None: self._validate_entity_filter(policy_config.on_finding) def process(self, request: InterceptedRequest) -> ProcessingResult: + if self._log_request_content: + _LOGGER.debug( + "privacy_guard_request_content request_id=%s stage=received body=%r", + request.request_id, + request.raw_body, + ) + _LOGGER.debug( + "privacy_guard_processing_started request_id=%s body_bytes=%d", + request.request_id, + len(request.raw_body), + ) policy = request.policy_config action = policy.on_finding handler = self._select_handler(policy.body_format) self._validate_entity_filter(action) if request.raw_body == b"": + _LOGGER.debug( + "privacy_guard_processing_bodyless request_id=%s", + request.request_id, + ) + if self._log_request_content: + _LOGGER.debug( + "privacy_guard_request_content request_id=%s " + "stage=forwarded body=%r", + request.request_id, + request.raw_body, + ) return ProcessingResult(decision=ProcessingDecision.ALLOW) + phase_started = time.monotonic() + _LOGGER.debug( + "privacy_guard_processing_phase_started request_id=%s phase=normalize", + request.request_id, + ) request_body = _normalize_request_body(handler, request) verified_body = _validate_normalized_body(request_body, request.raw_body) + scanned_characters = sum( + len(text_block.text) for text_block in verified_body.text_blocks + ) + _LOGGER.debug( + "privacy_guard_processing_phase_completed request_id=%s " + "phase=normalize duration_ms=%.3f text_block_count=%d " + "scanned_characters=%d", + request.request_id, + (time.monotonic() - phase_started) * 1000, + len(verified_body.text_blocks), + scanned_characters, + ) budget = ScanBudget.from_timeout(self._scan_timeout_seconds) + phase_started = time.monotonic() + _LOGGER.debug( + "privacy_guard_processing_phase_started request_id=%s phase=scan", + request.request_id, + ) scan_result = self._scan_text_blocks(verified_body.text_blocks, action, budget) + _LOGGER.debug( + "privacy_guard_processing_phase_completed request_id=%s phase=scan " + "duration_ms=%.3f finding_count=%d limit_exceeded=%s", + request.request_id, + (time.monotonic() - phase_started) * 1000, + len(scan_result.findings), + scan_result.limit_exceeded, + ) if scan_result.limit_exceeded: return ProcessingResult( decision=ProcessingDecision.DENY, reason_code=LIMIT_REASON_CODE @@ -155,11 +211,30 @@ def process(self, request: InterceptedRequest) -> ProcessingResult: replacement, projected_size = redaction redacted_text_bytes += projected_size replacements[block.path] = replacement + phase_started = time.monotonic() + _LOGGER.debug( + "privacy_guard_processing_phase_started request_id=%s phase=reconstruct", + request.request_id, + ) reconstructed = _reconstruct_body(handler, verified_body.source, replacements) + _LOGGER.debug( + "privacy_guard_processing_phase_completed request_id=%s " + "phase=reconstruct duration_ms=%.3f output_bytes=%d transformed=%s", + request.request_id, + (time.monotonic() - phase_started) * 1000, + len(reconstructed), + reconstructed != request.raw_body, + ) if len(reconstructed) > MAX_BODY_BYTES: return ProcessingResult( decision=ProcessingDecision.DENY, reason_code=LIMIT_REASON_CODE ) + if self._log_request_content: + _LOGGER.debug( + "privacy_guard_request_content request_id=%s stage=forwarded body=%r", + request.request_id, + reconstructed, + ) return ProcessingResult( decision=ProcessingDecision.ALLOW, replacement_body=reconstructed @@ -437,3 +512,6 @@ def _finding_pattern_name(finding: Finding) -> str: if finding.metadata is None: return "" return finding.metadata.get(PATTERN_NAME_METADATA_KEY, "") + + +_LOGGER = logging.getLogger(__name__) diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 6f685f2..1ff7a65 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -33,8 +33,18 @@ class MiddlewareServer: """High-level server that wires a scanner into the Privacy Guard service.""" - def __init__(self, *, scanner: Scanner[ScannerConfig]) -> None: - self._servicer = PrivacyGuardMiddleware(RequestProcessor([scanner])) + def __init__( + self, + *, + scanner: Scanner[ScannerConfig], + log_request_content: bool = False, + ) -> None: + self._servicer = PrivacyGuardMiddleware( + RequestProcessor( + [scanner], + log_request_content=log_request_content, + ) + ) def serve(self, listen: str = "127.0.0.1:50051") -> None: """Serve until termination using a managed synchronous entry point.""" @@ -76,16 +86,45 @@ async def serve( @app.callback() -def main() -> None: +def main( + context: typer.Context, + debug: Annotated[ + bool, + typer.Option( + "--debug", + help="Log content-safe request processing and phase timings.", + ), + ] = False, + debug_log_content: Annotated[ + bool, + typer.Option( + "--debug-log-content", + help=( + "DANGEROUS: log complete request bodies before and after " + "Privacy Guard processing." + ), + ), + ] = False, +) -> None: """Select one of Privacy Guard's built-in scanners.""" logging.basicConfig( level=logging.INFO, - format="%(levelname)s %(name)s: %(message)s", + format="%(asctime)s %(levelname)s %(name)s: %(message)s", ) + logging.getLogger("privacy_guard").setLevel( + logging.DEBUG if debug or debug_log_content else logging.INFO + ) + context.obj = debug_log_content + if debug_log_content: + _LOGGER.warning( + "privacy_guard_request_content_logging_enabled " + "complete_request_bodies_may_contain_secrets" + ) @app.command("regex") def run_regex( + context: typer.Context, config: Annotated[ Path, typer.Option(help="Path to the RegexScanner configuration."), @@ -115,7 +154,10 @@ def run_regex( scanner.scanner_name, listen, ) - MiddlewareServer(scanner=scanner).serve(listen) + MiddlewareServer( + scanner=scanner, + log_request_content=context.obj is True, + ).serve(listen) except PrivacyGuardError as error: typer.echo(str(error), err=True) raise typer.Exit(code=1) from None diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index 4d2aeab..b6f9012 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -132,6 +132,11 @@ async def _evaluate_rpc( """Run one evaluation using only the context's abort capability.""" started = time.monotonic() request_id = request.context.request_id + _LOGGER.debug( + "privacy_guard_evaluation_received request_id=%s body_bytes=%d", + request_id, + len(request.body), + ) failure: PrivacyGuardError | None = None action = "error" finding_count = 0 @@ -153,7 +158,13 @@ async def _evaluate_rpc( failure=failure, ) _LOGGER.info( - "privacy_guard_evaluation", + "privacy_guard_evaluation request_id=%s duration_ms=%.3f " + "action=%s finding_count=%d error_code=%s", + log_extra["request_id"], + log_extra["duration_ms"], + log_extra["action"], + log_extra["finding_count"], + log_extra["error_code"] or "none", extra=log_extra, ) diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 86fe932..b72e91b 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -249,6 +249,8 @@ def test_cli_exposes_root_and_builtin_scanner_help() -> None: assert root_help.exit_code == 0 assert "privacy-guard" in root_help.output assert "regex" in root_help.output + assert "--debug" in root_help.output + assert "--debug-log-content" in root_help.output scanner_help = CliRunner().invoke(app, ["regex", "--help"]) @@ -260,6 +262,52 @@ def test_cli_exposes_root_and_builtin_scanner_help() -> None: assert "--scanner-name" in scanner_help.output +def test_cli_enables_explicit_request_content_logging( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + processor_options: list[bool] = [] + scanner = DeterministicEmailScanner( + ScannerConfig(name="regex", entity_types=frozenset({"email"})) + ) + real_request_processor = server_module.RequestProcessor + + def create_processor( + scanners: Sequence[DeterministicEmailScanner], + *, + log_request_content: bool = False, + ) -> RequestProcessor: + processor_options.append(log_request_content) + return real_request_processor( + scanners, + log_request_content=log_request_content, + ) + + monkeypatch.setattr( + server_module.RegexScanner, + "from_yaml", + lambda *args, **kwargs: scanner, + ) + monkeypatch.setattr(server_module, "RequestProcessor", create_processor) + monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) + + with caplog.at_level(logging.WARNING, logger="privacy_guard.service.server"): + result = CliRunner().invoke( + app, + [ + "--debug-log-content", + "regex", + "--config", + "scanner-config.yaml", + ], + ) + + assert result.exit_code == 0 + assert processor_options == [True] + assert "privacy_guard_request_content_logging_enabled" in caplog.text + assert "complete_request_bodies_may_contain_secrets" in caplog.text + + def test_cli_rejects_builtin_options_before_subcommand() -> None: result = CliRunner().invoke( app, diff --git a/projects/privacy-guard/tests/test_processor.py b/projects/privacy-guard/tests/test_processor.py index d2e37f3..8645fd1 100644 --- a/projects/privacy-guard/tests/test_processor.py +++ b/projects/privacy-guard/tests/test_processor.py @@ -1,3 +1,4 @@ +import logging import math from collections.abc import Mapping @@ -166,6 +167,56 @@ def test_scanner_visits_each_text_block_and_reconstructs_once() -> None: assert handler.replacements == {} +def test_debug_log_reports_safe_processing_phase_metrics( + caplog: pytest.LogCaptureFixture, +) -> None: + sentinel = "sensitive@example.com" + scanner = RecordingScanner( + {sentinel: (_finding(0, len(sentinel), entity="email"),)} + ) + handler = RecordingHandler( + (TextBlock(path="sensitive-path-8472", text=sentinel),), + reconstructed=b"[email]", + ) + + with caplog.at_level(logging.DEBUG, logger="privacy_guard.processor"): + _processor(scanner, handler).process(_request(raw_body=sentinel.encode())) + + assert "phase=normalize" in caplog.text + assert "phase=scan" in caplog.text + assert "phase=reconstruct" in caplog.text + assert f"body_bytes={len(sentinel.encode())}" in caplog.text + assert "text_block_count=1" in caplog.text + assert f"scanned_characters={len(sentinel)}" in caplog.text + assert "finding_count=1" in caplog.text + assert sentinel not in caplog.text + assert "8472" not in caplog.text + + +def test_request_content_logging_reports_received_and_forwarded_bodies( + caplog: pytest.LogCaptureFixture, +) -> None: + sentinel = "sensitive@example.com" + scanner = RecordingScanner( + {sentinel: (_finding(0, len(sentinel), entity="email"),)} + ) + handler = RecordingHandler( + (TextBlock(path="email", text=sentinel),), + reconstructed=b"[email]", + ) + processor = RequestProcessor( + [scanner], + {handler.format_name: handler}, + log_request_content=True, + ) + + with caplog.at_level(logging.DEBUG, logger="privacy_guard.processor"): + processor.process(_request(raw_body=sentinel.encode())) + + assert f"stage=received body={sentinel.encode()!r}" in caplog.text + assert "stage=forwarded body=b'[email]'" in caplog.text + + def test_empty_scanner_sequence_is_rejected_at_construction() -> None: with pytest.raises(PrivacyGuardError) as exception_info: RequestProcessor([]) From d9482b2493551fd3f18cac39b900138df957d12e Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 16:16:02 -0400 Subject: [PATCH 23/82] Polish Privacy Guard documentation and workflows --- .github/workflows/privacy-guard.yml | 49 ++++ docs/documentation/index.md | 8 +- .../architecture/format-handlers.md | 193 ++++++++++++++ .../privacy-guard/architecture/index.md | 130 +++++++++ .../architecture/request-lifecycle.md | 146 ++++++++++ .../architecture/safety-and-limits.md | 188 +++++++++++++ .../privacy-guard/architecture/scanners.md | 175 ++++++++++++ .../architecture/service-boundary.md | 173 ++++++++++++ projects/privacy-guard/AGENTS.md | 105 ++++++++ projects/privacy-guard/CONTRIBUTING.md | 55 ---- projects/privacy-guard/Makefile | 82 ++++++ projects/privacy-guard/README.md | 71 ++++- projects/privacy-guard/docs/architecture | 1 + projects/privacy-guard/docs/index.md | 6 + .../examples/email-scanner/README.md | 86 ++++-- .../email-scanner/generate_gateway_config.py | 29 -- .../examples/email-scanner/policy.yaml | 2 + .../examples/regex-scanner/.gitignore | 3 + .../examples/regex-scanner/README.md | 251 +++++++++++++----- .../regex-scanner/generate_gateway_config.py | 29 -- .../regex-scanner/pi-models.template.json | 16 ++ .../regex-scanner/policy.pi.template.yaml | 42 +++ .../examples/regex-scanner/policy.yaml | 2 + .../middleware-dev-manifest.json | 14 +- .../scripts/{validate.sh => check.sh} | 2 +- .../src/privacy_guard/bindings/__init__.py | 2 +- .../supervisor_middleware_pb2_grpc.py | 186 ++++++------- .../tests/examples/test_email_scanner.py | 13 +- .../tests/examples/test_regex_scanner.py | 73 ++++- zensical.toml | 12 +- 30 files changed, 1816 insertions(+), 328 deletions(-) create mode 100644 .github/workflows/privacy-guard.yml create mode 100644 docs/documentation/privacy-guard/architecture/format-handlers.md create mode 100644 docs/documentation/privacy-guard/architecture/index.md create mode 100644 docs/documentation/privacy-guard/architecture/request-lifecycle.md create mode 100644 docs/documentation/privacy-guard/architecture/safety-and-limits.md create mode 100644 docs/documentation/privacy-guard/architecture/scanners.md create mode 100644 docs/documentation/privacy-guard/architecture/service-boundary.md create mode 100644 projects/privacy-guard/AGENTS.md delete mode 100644 projects/privacy-guard/CONTRIBUTING.md create mode 100644 projects/privacy-guard/Makefile create mode 120000 projects/privacy-guard/docs/architecture create mode 100644 projects/privacy-guard/docs/index.md delete mode 100644 projects/privacy-guard/examples/email-scanner/generate_gateway_config.py delete mode 100644 projects/privacy-guard/examples/regex-scanner/generate_gateway_config.py create mode 100644 projects/privacy-guard/examples/regex-scanner/pi-models.template.json create mode 100644 projects/privacy-guard/examples/regex-scanner/policy.pi.template.yaml rename projects/privacy-guard/scripts/{validate.sh => check.sh} (86%) diff --git a/.github/workflows/privacy-guard.yml b/.github/workflows/privacy-guard.yml new file mode 100644 index 0000000..fa36cc2 --- /dev/null +++ b/.github/workflows/privacy-guard.yml @@ -0,0 +1,49 @@ +name: Privacy Guard + +"on": + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: privacy-guard-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + check: + name: Check Privacy Guard (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - "3.11" + - "3.14" + defaults: + run: + working-directory: projects/privacy-guard + env: + UV_CACHE_DIR: ${{ runner.temp }}/privacy-guard-uv-cache + UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/privacy-guard-venv + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up uv and Python + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.11.31" + python-version: ${{ matrix.python-version }} + + - name: Install locked dependencies + run: uv sync --frozen + + - name: Run project checks + run: make check diff --git a/docs/documentation/index.md b/docs/documentation/index.md index 628d196..5579bdd 100644 --- a/docs/documentation/index.md +++ b/docs/documentation/index.md @@ -6,5 +6,9 @@ agent_markdown: true # Documentation -As Dev Notes introduce reusable software, this section will collect the technical -documentation and references needed to install, use, and reproduce that work. +Technical documentation and references for installing, using, and extending +OpenShell Research projects. + +- [Privacy Guard architecture](privacy-guard/architecture/index.md): component + boundaries, request flow, extension interfaces, concurrency, failures, and + limits diff --git a/docs/documentation/privacy-guard/architecture/format-handlers.md b/docs/documentation/privacy-guard/architecture/format-handlers.md new file mode 100644 index 0000000..e229901 --- /dev/null +++ b/docs/documentation/privacy-guard/architecture/format-handlers.md @@ -0,0 +1,193 @@ +--- +title: Format handlers +description: Request-body normalization, opaque paths, and reconstruction contracts. +agent_markdown: true +--- + +# Format handlers + +A format handler translates raw body bytes into text blocks and reconstructs the +body after redaction. It owns all format-specific parsing and path syntax. + +The processor uses the same interface for every format and never parses a path. + +## Format handler interface + +A format handler implements two protected methods: + +| Method | Responsibility | +| --- | --- | +| `_normalize(raw_body, policy_config)` | Parse bytes and return a `RequestBody` | +| `_reconstruct(request_body, replacements_by_path)` | Apply replacements and return bytes | + +Applications call the public `normalize` and `reconstruct` methods. These +wrappers validate extension output. + +The handler constructor declares a stable `format_name`. The processor registry +key must match it. + +## Normalized body + +`RequestBody` contains: + +| Field | Purpose | +| --- | --- | +| `text_blocks` | `TextBlock` values selected for scanning | +| `parsed_value` | Opaque handler-owned reconstruction state | +| `original_bytes` | Exact input bytes | + +`TextBlock` contains: + +| Field | Purpose | +| --- | --- | +| `path` | Opaque handler-defined address | +| `text` | String passed to scanners | +| `replaceable` | Whether redaction may rewrite this block | + +The processor requires unique paths. It may compare paths and use them as +mapping keys, but it must not parse them. + +The handler must not mutate `parsed_value` during reconstruction. It may copy +the value before applying replacements. + +## Unchanged and rewritten bodies + +When `replacements_by_path` is empty, return `original_bytes` exactly. + +When replacements exist: + +- replace only the addressed text blocks +- reject invalid paths or replacement types +- preserve untouched values semantically +- return bytes + +A rewritten body does not need to preserve whitespace, object formatting, or +other serialization details. + +## JSON handler + +`JsonHandler` is the only built-in format handler. It accepts strict UTF-8 JSON +and rejects: + +- invalid UTF-8 +- duplicate object keys +- non-finite numbers +- invalid Unicode scalar values +- excessive nesting or text shape + +It scans: + +- every JSON string value +- every object key + +It does not scan numbers, booleans, or null. + +### Paths + +String values use JSON Pointer paths: + +```text +{"items": [{"email": "a@example.com"}]} +``` + +```text +/items/0/email +``` + +The root string uses the empty path. Pointer tokens escape `~` as `~0` and `/` +as `~1`. + +Object keys use an internal `#key:` prefix: + +```text +#key:/items/0/email +``` + +Key paths are not reconstruction paths. Their text blocks set +`replaceable=False`. + +### Reconstruction + +`JsonHandler` copies its parsed tree, resolves each JSON Pointer, verifies the +target is a string, applies the replacement, and serializes compact UTF-8 JSON. + +Object keys are never rewritten. If redact policy selects a finding in a key, +the processor denies the request. + +## Minimal plain-text handler + +```python +from collections.abc import Mapping + +from privacy_guard.config import PolicyConfig +from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.request_body import FormatHandler, RequestBody, TextBlock + + +class PlainTextHandler(FormatHandler): + def __init__(self) -> None: + super().__init__(format_name="text") + + def _normalize( + self, + raw_body: bytes, + policy_config: PolicyConfig, + ) -> RequestBody: + try: + text = raw_body.decode("utf-8") + except UnicodeDecodeError: + raise PrivacyGuardError(ErrorCode.BODY_ENCODING_INVALID) from None + return RequestBody( + text_blocks=(TextBlock(path="", text=text),), + parsed_value=None, + original_bytes=raw_body, + ) + + def _reconstruct( + self, + request_body: RequestBody, + replacements_by_path: Mapping[str, str], + ) -> bytes: + if not replacements_by_path: + return request_body.original_bytes + if set(replacements_by_path) != {""}: + raise PrivacyGuardError(ErrorCode.BODY_RECONSTRUCTION_INVALID) + return replacements_by_path[""].encode("utf-8") +``` + +Register it when constructing the processor: + +```python +processor = RequestProcessor( + scanners=[scanner], + format_handlers={"text": PlainTextHandler()}, +) +``` + +## Concurrency + +One handler instance serves concurrent requests. Keep all parsed request state +inside the returned `RequestBody`. + +Do not store request bytes, parsed bodies, text blocks, paths, or replacements +on the handler instance. + +## Testing a handler + +Test: + +- valid and invalid input encoding +- parser edge cases specific to the format +- stable and unique text-block paths +- text selection and `replaceable` flags +- exact no-op byte preservation +- replacement of every supported path shape +- invalid and stale replacement paths +- reconstruction does not mutate parsed state +- shape limits +- concurrent calls + +Processor tests should cover cross-handler rules such as path uniqueness, +original-byte equality, and aggregate text limits. + +[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/index.md b/docs/documentation/privacy-guard/architecture/index.md new file mode 100644 index 0000000..c2954ed --- /dev/null +++ b/docs/documentation/privacy-guard/architecture/index.md @@ -0,0 +1,130 @@ +--- +title: Privacy Guard architecture +description: System structure, component boundaries, and request flow for Privacy Guard. +agent_markdown: true +--- + +# Privacy Guard architecture + +Privacy Guard is OpenShell middleware that scans provider-bound HTTP request +bodies and applies policy to detected sensitive data. OpenShell calls it before +adding provider credentials. + +Privacy Guard can allow the request unchanged, allow it with a replacement body, +or deny it. It never sends the provider request itself. + +## Where Privacy Guard runs + +```text +Sandbox process + | + | provider-bound HTTP request + v +OpenShell supervisor + | + | gRPC: SupervisorMiddleware + v +Privacy Guard + | + | allow, replace body, or deny + v +OpenShell supervisor + | + | credentials added only after middleware allows + v +Provider +``` + +Privacy Guard implements the `SupervisorMiddleware` gRPC service. Its manifest +registers one binding: HTTP requests in the pre-credentials phase. + +## Component boundaries + +Source paths on these pages are relative to +`projects/privacy-guard/src/privacy_guard/`. + +- `service/` owns gRPC, protobuf conversion, worker scheduling, and finding + aggregation. It does not scan or apply policy. Outside generated `bindings/`, + no other package imports gRPC or generated bindings. +- `processor.py` coordinates format handlers and scanners, applies policy, and + redacts bodies. It does not import gRPC or protobuf. +- `request_body/` parses and reconstructs bodies. It does not scan or apply + policy. +- `scanners/` detects values within one text block. It does not depend on + request formats, policy, gRPC, or protobuf. +- `config.py` defines policy configuration, not scanner configuration. +- `payloads/` defines transport-independent request and result records. + +## Request flow + +```text +HttpRequestEvaluation protobuf + | + v +PrivacyGuardMiddleware + validates transport input + parses policy + creates InterceptedRequest + | + v +RequestProcessor + selects FormatHandler + | + v +FormatHandler.normalize + returns RequestBody + TextBlock values + | + v +Scanner.scan + returns block-relative Finding values + | + v +RequestProcessor + validates and filters findings + observes, blocks, or redacts + | + +-- policy deny or pre-reconstruction limit --> ProcessingResult + | + +-- continue + | + v + FormatHandler.reconstruct + | + v + output-size check + | + v + ProcessingResult +``` + +Both paths return a `ProcessingResult` to `PrivacyGuardMiddleware`, which +aggregates findings and creates the `HttpRequestResult` protobuf. + +The processor is synchronous. The service runs it in a dedicated thread pool +so scanner work does not block the gRPC event loop. + +## Core data types + +| Type | Meaning | +| --- | --- | +| `PolicyConfig` | Body format and action to apply to selected findings | +| `InterceptedRequest` | Protobuf-free request body, request ID, content type, and parsed policy | +| `RequestBody` | Handler-owned parsed state, original bytes, and text blocks | +| `TextBlock` | One string to scan, with an opaque path and replacement flag | +| `Finding` | One scanner result with block-relative offsets | +| `RequestBodyFinding` | A finding paired with its text-block path | +| `ProcessingResult` | Allow or deny decision, optional replacement body, findings, and reason code | + +Domain model fields are strict and frozen. `RequestBody.parsed_value` is +handler-owned and may contain mutable objects; reconstruction must not mutate +it. Fields containing request content are excluded from normal representations. + +## Read next + +- [Request lifecycle](request-lifecycle.md) explains normalization, scanning, + policy filtering, and redaction. +- [Scanners](scanners.md) defines the scanner extension contract. +- [Format handlers](format-handlers.md) defines body parsing and reconstruction. +- [Service boundary](service-boundary.md) covers gRPC adaptation and concurrency. +- [Safety and limits](safety-and-limits.md) records failure behavior and resource + bounds. diff --git a/docs/documentation/privacy-guard/architecture/request-lifecycle.md b/docs/documentation/privacy-guard/architecture/request-lifecycle.md new file mode 100644 index 0000000..d9a30ed --- /dev/null +++ b/docs/documentation/privacy-guard/architecture/request-lifecycle.md @@ -0,0 +1,146 @@ +--- +title: Request lifecycle +description: How Privacy Guard normalizes, scans, filters, and transforms one request. +agent_markdown: true +--- + +# Request lifecycle + +`RequestProcessor` owns the complete protobuf-free request flow. It composes +format handlers and scanners, applies policy, and returns a `ProcessingResult`. + +## Inputs + +The processor receives an `InterceptedRequest` containing: + +- the exact request body bytes +- a parsed `PolicyConfig` +- the request ID +- the original content type, retained as context but not used for format + selection + +Policy selects the format through `body_format`; the processor does not infer it +from request content or metadata. + +After validating the configured format and entity filter, the processor allows +an empty body without normalization or scanning. + +## Processing stages + +### 1. Select the format handler + +The processor looks up `PolicyConfig.body_format` in its registered format +handlers. The handler's own `format_name` must match its registry key. + +It also verifies that every entity named by policy appears in at least one +active scanner's `ScannerConfig.entity_types`. An unknown format or entity is +invalid configuration. + +### 2. Normalize the body + +The selected handler parses the original bytes and returns a `RequestBody`. +The processor then verifies: + +- `RequestBody.original_bytes` exactly matches the input +- every `TextBlock.path` is unique +- the number of text blocks is within the request limit +- the total text length is within the scan limit + +The processor does not parse text-block paths. Paths belong to the format +handler. + +### 3. Scan and filter findings + +The processor creates one request-wide `ScanBudget`. It passes every text block +to every configured scanner in registration order. It validates each result, +then selects findings by: + +- `entity_types` +- `minimum_confidence` + +Policy never changes scanner behavior. `entity_types: null` selects all entity +types. An empty list selects none. +`minimum_confidence: null` accepts every confidence level. + +`Scanner.scan` validates the returned tuple and finding models. +`RequestProcessor` validates scanner identity, declared entities, offsets, and +selected finding totals. See +[Validation ownership](safety-and-limits.md#validation-ownership). + +### 4. Apply the action + +| Action | No selected findings | Selected findings | +| --- | --- | --- | +| `observe` | Allow unchanged | Allow unchanged and report findings | +| `block` | Allow unchanged | Deny and report findings | +| `redact` | Allow unchanged | Replace selected spans, or deny if safe replacement is impossible | + +The processor scans all text blocks for every action. The action controls how +selected findings affect the result. + +### 5. Reconstruct the body + +For redaction, the processor creates replacement text for each affected, +replaceable block. It passes a path-to-text mapping back to the same format +handler that normalized the body. + +A selected block action, non-replaceable redaction, or limit result returns +before reconstruction. Every other non-empty request is reconstructed once. +Observe and block-without-findings pass an empty replacement map. + +With an empty replacement map, a handler returns the original bytes exactly. +After a rewrite, untouched values must remain semantically equivalent, but +byte formatting may change. + +The processor returns a replacement body only when reconstructed bytes differ +from the original. + +## Redaction behavior + +Findings may overlap. Every action retains all selected findings for reporting. +Redaction resolves overlaps only when choosing spans to replace; losing overlaps +remain in the result. + +Redaction chooses non-overlapping replacement spans in this order: + +1. higher confidence +2. longer span +3. earlier start offset +4. earlier end offset +5. scanner name +6. entity name +7. pattern name + +The stable ordering makes redaction independent of incidental scanner return +order. + +The redaction template accepts static text and `{entity}`. Formatting options, +conversions, and other fields are invalid. + +Before allocating replacement text, the processor projects its UTF-8 size +against the body limit. It checks the serialized body again after +reconstruction. + +## Non-replaceable text + +A format handler may expose text that can be observed but not safely rewritten. +It marks that `TextBlock` with `replaceable=False`. + +The JSON handler uses this for object keys. Observe reports findings in keys, +and block denies normally. Redact denies the request when a selected finding is +in a key because changing a key could create a collision. + +## Output + +The processor returns a `ProcessingResult`: + +- `ALLOW` with no replacement for an unchanged request +- `ALLOW` with replacement bytes for a successful redaction +- `DENY` with `privacy_guard_blocked` for a policy block +- `DENY` with `privacy_guard_limit_exceeded` when safe bounded output cannot be + produced + +Findings remain path-aware in the domain result. The service later aggregates +them for the protobuf response. + +[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md new file mode 100644 index 0000000..3bf8eb8 --- /dev/null +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -0,0 +1,188 @@ +--- +title: Safety and limits +description: Validation ownership, bounded processing, failure behavior, and logging rules. +agent_markdown: true +--- + +# Safety and limits + +Privacy Guard bounds input shape, scanning work, findings, replacement size, and +transport output. Limits are part of the architecture, not optional tuning +defaults. + +Package-wide values live in `constants.py`. + +## Validation ownership + +Each layer validates facts it can establish: + +| Layer | Validates | +| --- | --- | +| Policy and domain models | Strict field types, shape, and local field rules | +| Scanner public wrapper | Scanner return container and finding models | +| Format-handler public wrappers | Normalized body and reconstructed byte types | +| JSON handler | Early JSON nesting, text-block, and character limits | +| Processor | Registry consistency, original bytes, paths, finding identity, offsets, and authoritative request-wide limits | +| Service | Phase, transport body size, protobuf conversion, response size, and finding aggregation | + +Keep validation at these owners. Early checks may bound allocation, while the +next trust boundary repeats authoritative checks for extension output. Do not +move request-wide checks into a scanner or scanner semantics into the service. + +## Request limits + +| Limit | Constant | Value | Owner | +| --- | --- | ---: | --- | +| Incoming request body | `MAX_BODY_BYTES` | 4 MiB | Service | +| Projected or reconstructed replacement body | `MAX_BODY_BYTES` | 4 MiB | Processor | +| Serialized replacement body | `MAX_BODY_BYTES` | 4 MiB | Service | +| gRPC receive message | `MAX_RECEIVE_MESSAGE_BYTES` | 5 MiB | gRPC server | +| JSON nesting | `MAX_JSON_NESTING` | 64 levels | JSON handler | +| Text blocks per request | `MAX_TEXT_BLOCKS` | 4,096 | JSON handler + processor | +| Characters scanned per request | `MAX_SCANNED_CHARACTERS` | 4,194,304 | JSON handler + processor | +| Findings returned by one scanner call | `MAX_FINDINGS_PER_BLOCK` | 256 | Scanner wrapper | +| Selected findings per text block | `MAX_FINDINGS_PER_BLOCK` | 256 | Processor | +| Selected findings per request | `MAX_FINDINGS_PER_REQUEST` | 4,096 | Processor | +| Default request scan budget | `DEFAULT_SCAN_TIMEOUT_SECONDS` | 1 second | Processor | +| Maximum configured scan budget | `MAX_SCAN_TIMEOUT_SECONDS` | 30 seconds | Processor | +| Active scanner workers | `MAX_CONCURRENT_SCANS` | 4 | Service | +| Concurrent gRPC calls | `MAX_CONCURRENT_RPCS` | 16 | gRPC server | + +The gRPC receive limit includes 1 MiB beyond the body limit for the protobuf +envelope. The service still enforces the advertised 4 MiB body limit. + +The JSON handler checks text-block and character limits while parsing. The +processor enforces them authoritatively for every handler. + +The scan budget is shared across all scanners and text blocks in a request. It +is cooperative: a scanner that calls `remaining_seconds()` after the deadline +causes a limit deny. The processor cannot detect or interrupt an opaque scanner +operation that does not check the budget. + +## Name, finding, and response limits + +| Limit | Constant | Value | +| --- | --- | ---: | +| Entity, scanner, and metadata strings | `MAX_SCANNER_METADATA_BYTES` | 1,024 UTF-8 bytes | +| Metadata entries per finding | `MAX_FINDING_METADATA_ENTRIES` | 32 | +| Aggregated protobuf finding groups | `MAX_PROTO_FINDING_GROUPS` | 32 | +| Encoded size per protobuf finding | `MAX_PROTO_FINDING_BYTES` | 4 KiB | + +The string limit covers policy entity filters, finding entities, finding scanner +names, and finding metadata keys and values. + +The service aggregates findings before applying protobuf limits. If a safe +result cannot be represented, it returns a limit deny with no partial findings. + +## Regex configuration limits + +| Limit | Constant | Value | +| --- | --- | ---: | +| YAML file size | `MAX_SCANNER_CONFIG_BYTES` | 16 MiB | +| YAML nesting | `MAX_SCANNER_CONFIG_NESTING` | 16 | +| YAML nodes | `MAX_SCANNER_CONFIG_NODES` | 250,000 | +| YAML scalar | `MAX_SCANNER_CONFIG_SCALAR_BYTES` | 16 KiB | +| Name length | `MAX_REGEX_NAME_BYTES` | 128 ASCII bytes | +| Profiles | `MAX_REGEX_PROFILES` | 32 | +| Entities per profile | `MAX_REGEX_ENTITIES_PER_PROFILE` | 2,000 | +| Patterns per profile | `MAX_REGEX_PATTERNS_PER_PROFILE` | 10,000 | +| Entities across the file | `MAX_REGEX_ENTITIES_TOTAL` | 10,000 | +| Patterns across the file | `MAX_REGEX_PATTERNS_TOTAL` | 50,000 | +| Regex pattern | `MAX_REGEX_PATTERN_BYTES` | 16 KiB | +| Matches per pattern and text block | `MAX_MATCHES_PER_PATTERN` | 256 | + +These catalog limits are separate from request finding limits. A large catalog +may still produce only a few findings for a request. + +## When a limit is exceeded + +Privacy Guard returns a successful deny result with +`privacy_guard_limit_exceeded` when: + +- a scanner observes an expired scan budget +- findings exceed block or request limits +- projected redaction exceeds the body limit +- a reconstructed body exceeds the body limit +- aggregated findings exceed protobuf limits +- a deny reason code cannot be represented safely + +The limit result contains no replacement body and no partial findings. + +During evaluation, malformed input and internal failures instead abort the RPC +with a cataloged gRPC error. OpenShell then applies the middleware +registration's failure mode. + +## Error catalog + +Production failures use `PrivacyGuardError` and a stable `ErrorCode`. Each error +spec defines: + +- whether the failure is invalid input or internal +- the responsible component +- the failed operation +- a content-safe summary +- a remediation hint + +Caught extension exceptions are translated at their boundary. Their messages +and exception chains are not exposed. + +Use a new catalog entry for a new externally observable failure. Do not return +raw parser, scanner, Pydantic, or protobuf exceptions. + +## Logging + +Default operational logs include: + +- request ID +- duration +- action +- finding count +- stable error code +- phase timing and aggregate sizes at debug level + +They do not include: + +- request or replacement bodies +- text blocks +- matches +- paths +- headers or credentials +- scanner configuration +- arbitrary exception text + +Sensitive model fields use `repr=False` to reduce accidental exposure. This is +not a substitute for safe exception handling. + +`--debug` enables content-safe phase diagnostics. `--debug-log-content` +deliberately logs complete bodies and emits a warning; use it only in a +controlled development environment. + +## Extension requirements + +Scanner and format-handler extensions should: + +- return the types declared by the extension hook +- keep request data in local variables or returned request models +- translate expected parse and configuration failures to `PrivacyGuardError` +- leave extension-output validation to the public wrapper +- check applicable limits before allocating output proportional to input or + findings +- avoid logging input or caught exception text + +Do not add retries, fallbacks, or extra validation without a concrete failure +mode and a clear owning layer. + +## Changing a limit + +A limit change may affect the middleware manifest, processor behavior, protobuf +serialization, tests, examples, and deployment configuration. + +Before changing a limit: + +1. identify every layer that enforces or advertises it +2. update exact-boundary tests +3. check failure behavior above the boundary +4. run `make check` +5. benchmark changes that affect request processing + +[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/scanners.md b/docs/documentation/privacy-guard/architecture/scanners.md new file mode 100644 index 0000000..62e9192 --- /dev/null +++ b/docs/documentation/privacy-guard/architecture/scanners.md @@ -0,0 +1,175 @@ +--- +title: Scanners +description: Scanner configuration, lifecycle, output contract, and extension rules. +agent_markdown: true +--- + +# Scanners + +A scanner detects sensitive data in one text block. It has no access to the +request format, text-block path, active policy, gRPC request, or protobuf +messages. + +## Scanner interface + +A scanner: + +1. declares a concrete `ScannerConfig` type +2. receives validated configuration at construction +3. receives one text block and a `ScanBudget` in `_scan` +4. returns a tuple of block-relative `Finding` values + +The public `scan` method owns output validation. Extensions implement `_scan` +and do not override `scan`. + +## Configuration + +`ScannerConfig` contains: + +| Field | Purpose | +| --- | --- | +| `name` | Stable scanner identity copied into findings | +| `entity_types` | Complete set of entity names the scanner can emit | + +Custom scanners subclass `ScannerConfig` to add detection-specific fields. +Configuration fields are strict and frozen. Use immutable types for custom +fields because nested values are not deep-frozen. Scanner configuration remains +independent of policy. + +The base constructor infers the concrete config type from +`Scanner[ConcreteConfig]`, validates the supplied model, stores scanner metadata, +and then calls `_initialize`. + +Use `_initialize` for reusable derived state such as compiled expressions. Do +not define a scanner `__init__` for custom initialization. + +## Minimal scanner + +```python +from pydantic import Field + +from privacy_guard.scanners import Finding, ScanBudget, Scanner, ScannerConfig + + +class KeywordScannerConfig(ScannerConfig): + keyword: str = Field(min_length=1) + + +class KeywordScanner(Scanner[KeywordScannerConfig]): + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: + start = text_block.find(self.config.keyword) + if start < 0: + return () + return ( + Finding( + entity="keyword", + scanner_name=self.scanner_name, + start_offset=start, + end_offset=start + len(self.config.keyword), + ), + ) +``` + +Construct it with an explicit entity catalog: + +```python +scanner = KeywordScanner( + KeywordScannerConfig( + name="keywords", + entity_types=frozenset({"keyword"}), + keyword="secret", + ) +) +``` + +## Finding model + +A `Finding` contains: + +| Field | Meaning | +| --- | --- | +| `entity` | Entity name declared in `ScannerConfig.entity_types` | +| `scanner_name` | Name of the scanner that produced the finding | +| `start_offset` | Inclusive character offset within the text block | +| `end_offset` | Exclusive character offset within the text block | +| `confidence` | `low`, `medium`, or `high` | +| `metadata` | Optional bounded, immutable scanner attribution | + +Offsets count Python string characters, not UTF-8 bytes. A finding must cover a +non-empty span. + +The processor adds the text-block path later by creating a +`RequestBodyFinding`. Scanners never create or interpret paths. + +## Output validation + +`Scanner.scan` validates the returned tuple and findings. `RequestProcessor` +validates scanner identity, declared entities, offsets, and aggregate limits. +The processor validates every finding, then enforces block and request totals on +findings selected by policy. + +See [Validation ownership](safety-and-limits.md#validation-ownership). + +## Budgets and blocking work + +After normalization, the processor creates one monotonic budget for the scan +phase and shares it across every scanner and text block. The budget is +cooperative. + +Iterative scanners should call `budget.remaining_seconds()` before and after +each bounded unit of work. The method returns the remaining time or raises +`ScanBudgetExceeded`. + +A budget cannot interrupt a blocking operation already in progress. For +example, Python's standard regex engine cannot stop an expression while it is +evaluating. Avoid operations without a tested worst-case bound. + +## Concurrency + +One scanner instance serves concurrent requests through the service worker +pool. Scanner state must be immutable after initialization. + +Do not store: + +- text blocks +- matches from a request +- request IDs +- mutable per-request counters + +Keep per-request data local to `_scan`; do not mutate shared objects from it. + +## Built-in regex scanner + +`RegexScanner` loads a bounded YAML catalog. It compiles configured patterns in +`_initialize` and stores immutable compiled rules. + +Each regex finding includes `pattern_name` metadata. The processor uses entity +names for policy filtering. The service renders the audit label as +`entity/pattern-name`. + +The regex scanner: + +- rejects empty matches +- rejects named groups because it reserves an internal marker group +- rejects inline flags in favor of explicit config fields +- supports overlapping matches by advancing from the previous match start +- limits matches per pattern and findings per block +- checks the shared budget around each regex evaluation + +## Testing a scanner + +Test scanners without gRPC: + +- valid and invalid configuration +- no-match and match cases +- Unicode offsets +- declared entity and scanner identity +- output ordering when relevant +- finding limits +- budget exhaustion for iterative work +- concurrent calls when the scanner keeps derived state + +Processor tests should cover only behavior that requires multiple scanners, +text-block paths, policy, or request-wide limits. + +[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md new file mode 100644 index 0000000..10dd18b --- /dev/null +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -0,0 +1,173 @@ +--- +title: Service boundary +description: gRPC adaptation, worker scheduling, result aggregation, and server lifecycle. +agent_markdown: true +--- + +# Service boundary + +Outside the generated `bindings/` package, `service/` is the only layer that +imports gRPC or generated protobuf bindings. It translates between OpenShell's +transport contract and Privacy Guard's domain types. + +## gRPC methods + +Privacy Guard implements three `SupervisorMiddleware` methods: + +| RPC | Behavior | +| --- | --- | +| `Describe` | Advertises the service name, version, pre-credentials HTTP binding, and body limit | +| `ValidateConfig` | Parses policy and asks the processor to validate referenced formats and entities | +| `EvaluateHttpRequest` | Validates transport input, runs the processor, and returns a decision | + +`Describe` advertises only +`SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS`. The service rejects evaluations +for any other phase. + +## Incoming requests + +For each evaluation, the servicer: + +1. validates the phase and body size +2. converts the protobuf `Struct` into a strict `PolicyConfig` +3. extracts the request ID, content type, and body +4. constructs an `InterceptedRequest` +5. passes that domain record to `RequestProcessor` + +Transport-only fields that processing does not need remain at the service +boundary. + +`content_type` is retained as request context, but the processor selects the +format from policy. It does not infer format from this header. + +## Outgoing results + +The processor returns `ProcessingResult`. The servicer converts it to +`HttpRequestResult`: + +| Domain result | Protobuf result | +| --- | --- | +| Allow without replacement | `DECISION_ALLOW`, `has_body=false` | +| Allow with replacement | `DECISION_ALLOW`, `has_body=true`, replacement bytes | +| Deny | `DECISION_DENY`, no body, stable reason code | + +The service checks replacement size again before serialization. + +### Finding aggregation + +Domain findings include text-block paths and exact offsets. The protobuf result +contains count-based audit findings. + +The service groups findings by: + +- scanner name +- entity +- optional `pattern_name` metadata +- confidence + +It emits: + +- `type`: scanner name +- `label`: entity or `entity/pattern-name` +- `confidence` +- `count` + +Paths, offsets, and matched request content never cross the protobuf boundary. + +## Concurrency model + +The gRPC server allows up to 16 concurrent RPCs. Synchronous processing runs in +a dedicated pool with four workers. + +```text +gRPC event loop + | + | acquire scan slot + v +4-slot semaphore + | + v +4-thread executor + | + v +RequestProcessor.process +``` + +This prevents scanner work from blocking the async event loop and bounds active +synchronous scans. + +Scanner and format-handler instances are shared by those workers. They must be +safe for concurrent use. + +## Cancellation + +Cancelling an async RPC cannot stop Python code already running in a worker +thread. The service shields the worker future and keeps its semaphore slot until +the worker actually exits. + +This rule prevents cancellation from creating more than four active scans. + +During shutdown, the service stops gRPC and waits for the scan executor. + +## Error mapping + +`PrivacyGuardError` classifies each cataloged failure as invalid input or +internal failure. + +| Error kind | gRPC status | +| --- | --- | +| `invalid_input` | `INVALID_ARGUMENT` | +| `internal` | `INTERNAL` | + +This mapping applies to `EvaluateHttpRequest`. `ValidateConfig` reports failures +in `ValidateConfigResponse` with `valid=false` and a safe reason. + +Unexpected exceptions become the content-safe +`unexpected_service_failure` error. The service does not return arbitrary +exception text. + +A gRPC error is different from a policy deny. OpenShell applies its configured +middleware failure mode when the RPC fails. A policy deny is a successful RPC +whose result explicitly stops the request. + +## Server lifecycle + +`MiddlewareServer` is the high-level API for custom scanners. It constructs: + +```text +Scanner + -> RequestProcessor + -> PrivacyGuardMiddleware + -> gRPC server +``` + +The CLI uses the same API for built-in scanners. + +The server: + +1. creates an unstarted async gRPC server +2. binds the configured address +3. starts and waits for termination +4. stops gRPC +5. closes the middleware executor + +A bind failure becomes the stable `server_bind_failed` error. + +## Testing the boundary + +Service tests should cover: + +- protobuf/domain translation +- manifest contents +- policy validation RPCs +- phase and body-size validation +- allow, replacement, and deny serialization +- finding aggregation and encoded limits +- gRPC status mapping +- worker-slot behavior under cancellation +- startup, bind failure, and shutdown + +Keep scanner detection and processor policy cases out of service tests unless +the case requires transport translation. + +[Back to the architecture overview](index.md) diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md new file mode 100644 index 0000000..05fe35d --- /dev/null +++ b/projects/privacy-guard/AGENTS.md @@ -0,0 +1,105 @@ +# Privacy Guard + +Privacy Guard is OpenShell middleware that scans request bodies and applies +policy to detected sensitive data. + +## Development commands + +Run commands from `projects/privacy-guard/`. + +- List targets: `make help` +- Run all checks: `make check` +- Check Python 3.11: `make check-py311` +- Run focused tests: `make test PYTEST_ARGS=tests/test_processor.py` +- Run the benchmark: `make benchmark` + +Run focused tests while working and `make check` before handoff. Benchmarks are +diagnostic and have no pass/fail threshold. + +## Engineering approach + +- Do not preserve backward compatibility unless the user explicitly requests + it. Update callers, tests, examples, and docs with the change. +- Add defensive handling only for a concrete failure mode at the layer that + owns it. Avoid speculative guards, duplicate validation, broad catches, + retries, and fallbacks. + +## Project map + +- `src/privacy_guard/scanners/`: scanner contract, budgets, and built-ins +- `src/privacy_guard/request_body/`: normalization and reconstruction contracts +- `src/privacy_guard/processor.py`: request orchestration and policy +- `src/privacy_guard/service/`: gRPC lifecycle and protobuf adapter +- `src/privacy_guard/bindings/`: generated protobuf files; never hand-edit +- [`docs/architecture/`](docs/architecture/index.md): symlink to the canonical + site sources under `../../docs/documentation/privacy-guard/architecture/` +- `tests/`: tests that mirror source boundaries +- `tests/test_hardening.py`: cross-cutting security tests +- `examples/`: copyable deployments + +Keep each example's policy, configuration, commands, names, and tests in sync. +Before changing `processor.py`, `scanners/`, `request_body/`, or `service/`, +read the architecture overview and the matching topic page. +Architecture documentation changes follow +[`docs/development/index.md`](../../docs/development/index.md) and require its +checks. + +## Design boundaries + +- Keep scanners separate from `RequestProcessor` and `service/`. A scanner + inspects one text block and returns block-relative findings. Do not make + scanners depend on request-body formats, policy, gRPC, or generated bindings. +- `RequestProcessor` orchestrates format handlers and scanners, applies policy, + attaches text-block paths, and enforces aggregate limits. +- Outside generated `bindings/`, only `service/` may import gRPC or generated + bindings. +- Public scanner and format-handler methods validate extension output. + `RequestProcessor` validates finding identities and offsets. Do not repeat + these checks elsewhere. +- `ScannerConfig.entity_types` lists every entity the scanner can emit; it is + independent of policy. Apply policy after scanning. +- Scanner and format-handler instances serve concurrent requests. Do not store + request content or mutable per-request state on them. +- Format handlers define path syntax. Treat paths as opaque everywhere else. + +## Extension pattern + +Define a concrete config type and implement `_scan`. Do not override `scan`. + +```python +from pydantic import Field + +from privacy_guard.scanners import Finding, ScanBudget, Scanner, ScannerConfig + + +class KeywordScannerConfig(ScannerConfig): + keyword: str = Field(min_length=1) + + +class KeywordScanner(Scanner[KeywordScannerConfig]): + def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: + start = text_block.find(self.config.keyword) + if start < 0: + return () + return ( + Finding( + entity="keyword", + scanner_name=self.scanner_name, + start_offset=start, + end_offset=start + len(self.config.keyword), + ), + ) +``` + +Scanners that need custom initialization logic should implement `_initialize` +instead of `__init__`. The base constructor calls it last, after setting +validated configuration and scanner metadata. + +## Change limits + +- Add or update tests at the layer that owns the behavior. +- Ask before adding dependencies or changing the protobuf contract, stable error + codes, protocol limits, or fail-closed defaults. +- Do not remove or weaken tests to pass checks. +- Do not add casts, explicit `Any`, blanket ignores, or broad type suppressions + to handwritten code. `tests/test_typing_policy.py` enforces this. diff --git a/projects/privacy-guard/CONTRIBUTING.md b/projects/privacy-guard/CONTRIBUTING.md deleted file mode 100644 index 4b4e1f3..0000000 --- a/projects/privacy-guard/CONTRIBUTING.md +++ /dev/null @@ -1,55 +0,0 @@ -# Contributing to Privacy Guard - -Run `scripts/validate.sh` from this directory before review. Run -`scripts/validate.sh --python 3.11` when changing interpreter-sensitive code. -The gate owns full tests, Ruff formatting and lint, curated `ty` diagnostics, -the import smoke test, and the package build. - -Trust-boundary validation has one owner: - -| Boundary | Owner | -| --- | --- | -| Scalar fields and record shape | strict frozen Pydantic models | -| Action selection and action-specific fields | `PolicyConfig.on_finding` discriminated union keyed by `action` | -| Scanner metadata and scanner output shape | `Scanner` public wrapper | -| Format metadata and normalized/reconstructed output shape | `FormatHandler` public wrappers | -| Scanner/block identity, offsets, aggregate limits, and original bytes | `RequestProcessor` | -| Recursive JSON values | cached Pydantic adapter in `JsonHandler` | -| Protobuf conversion | service adapter | - -Custom scanners and format handlers subclass the nominal ABCs, call the base -constructor, and decorate protected hook implementations with `@override`. -`RequestProcessor` accepts a scanner sequence. Scanners return block-relative -`Finding` models; the processor attaches the opaque text-block path by composing -them into `RequestBodyFinding` models. - -Scanner configuration must not depend on policy. Every `ScannerConfig` declares -the scanner's complete `entity_types` catalog, and concrete subtypes own any -additional detection behavior. Observe, block, and redact action configs own -the finding criteria applied after scanning. - -Within modules and classes, place the public API before private implementation -details whenever dependency ordering allows it. Keep private constants, helper -records, functions, and methods toward the bottom of their containing scope. - -The handwritten typing policy is AST-enforced by -`tests/test_typing_policy.py`. Generated bindings under -`src/privacy_guard/bindings` are excluded. Keep generics parameterized and -suppressions narrow and rule-specific. Production and test overrides are -explicit; example scripts omit `@override` to keep their public API surface -minimal and copyable. - -When changing performance-sensitive processing code, run the optional -diagnostic benchmark with: - -```bash -uv run --frozen python scripts/benchmark_privacy_guard.py --suite full -``` - -Use the default quick suite during iteration. Both suites verify findings and -reconstruction outcomes before reporting median wall time and peak traced -allocation. The benchmark is not part of the contributor gate and has no -pass/fail thresholds; compare results only on a controlled environment. It uses -a synthetic scanner and the synchronous processor path, so it does not measure -real scanner cost, service concurrency, queuing, gRPC adaptation, or process -RSS. diff --git a/projects/privacy-guard/Makefile b/projects/privacy-guard/Makefile new file mode 100644 index 0000000..b4df0d8 --- /dev/null +++ b/projects/privacy-guard/Makefile @@ -0,0 +1,82 @@ +.DEFAULT_GOAL := help + +UV ?= uv +UV_RUN := $(UV) run --frozen +PYTEST_ARGS ?= +PROFILE ?= /tmp/privacy-guard.prof + +.PHONY: \ + help sync lock lock-check \ + test test-examples test-hardening \ + format format-check lint lint-fix typecheck fix \ + import-check build check check-py311 \ + benchmark benchmark-full benchmark-profile \ + clean + +help: ## Show available targets and configurable variables. + @awk 'BEGIN {FS = ":.*## "}; /^[a-zA-Z0-9_.-]+:.*## / {printf " %-28s %s\n", $$1, $$2}' $(MAKEFILE_LIST) + @printf "\nVariables:\n" + @printf " PYTEST_ARGS=... Extra pytest paths or flags\n" + @printf " PROFILE=... Benchmark profile path (default: %s)\n" "$(PROFILE)" + +sync: ## Install locked runtime and development dependencies. + $(UV) sync --frozen + +lock: ## Refresh uv.lock from pyproject.toml. + $(UV) lock + +lock-check: ## Verify uv.lock matches pyproject.toml. + $(UV) lock --check + +test: ## Run tests; pass paths or flags with PYTEST_ARGS. + $(UV_RUN) pytest -q $(PYTEST_ARGS) + +test-examples: ## Run example integration tests. + $(UV_RUN) pytest -q tests/examples + +test-hardening: ## Run security and typing-policy tests. + $(UV_RUN) pytest -q tests/test_hardening.py tests/test_typing_policy.py + +format: ## Format handwritten Python. + $(UV_RUN) ruff format . + +format-check: ## Check formatting without changing files. + $(UV_RUN) ruff format --check . + +lint: ## Run Ruff lint checks. + $(UV_RUN) ruff check . + +lint-fix: ## Apply safe Ruff fixes. + $(UV_RUN) ruff check --fix . + +typecheck: ## Run curated ty checks. + $(UV_RUN) ty check + +fix: ## Apply Ruff fixes, then format. + $(UV_RUN) ruff check --fix . + $(UV_RUN) ruff format . + +import-check: ## Smoke-test the installed package import. + $(UV_RUN) python -c "import privacy_guard" + +build: ## Build the source distribution and wheel. + $(UV) build + +check: ## Run the authoritative full project checks. + scripts/check.sh + +check-py311: ## Run full checks on the minimum Python version. + scripts/check.sh --python 3.11 + +benchmark: ## Run the quick processing benchmark. + $(UV_RUN) python scripts/benchmark_privacy_guard.py + +benchmark-full: ## Run the full processing benchmark suite. + $(UV_RUN) python scripts/benchmark_privacy_guard.py --suite full + +benchmark-profile: ## Run the quick benchmark and write PROFILE. + $(UV_RUN) python scripts/benchmark_privacy_guard.py --profile "$(PROFILE)" + +clean: ## Remove build, test, lint, and Python cache artifacts. + rm -rf build dist .pytest_cache .ruff_cache + find src tests examples scripts -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 2a90c11..2c69da8 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -15,9 +15,9 @@ original. The self-contained [built-in regex scanner walkthrough](examples/regex-scanner/README.md) provides a scanner configuration, gateway registration, sandbox policy, and manual -Claude Code workflow for a typical deployment. The separate -[email scanner example](examples/email-scanner/README.md) demonstrates how to -implement a custom scanner. +Claude Code and custom-endpoint Pi workflows for a typical deployment. The +separate [email scanner example](examples/email-scanner/README.md) demonstrates +how to implement a custom scanner. ## Request flow @@ -114,10 +114,10 @@ in general finding metadata and is reported as `entity/pattern-name` by the service. Policy filtering remains at entity level. A scanner is a nominal extension: declare its strict configuration type and -implement `_scan`. Scanners that need derived, reusable state may also override -`_initialize`; the base constructor calls it after validating and retaining the -configuration. The public `scan` wrapper validates the returned tuple and each -`Finding`. +implement `_scan`. Use `_initialize` for any custom initialization logic rather +than defining `__init__`; the base constructor runs the hook at the end of +initialization, after validating and retaining the configuration. The public +`scan` wrapper validates the returned tuple and each `Finding`. ```python from privacy_guard.scanners import Finding, ScanBudget, Scanner, ScannerConfig @@ -191,6 +191,36 @@ release its scanner slot until its synchronous worker really exits. | `service` | High-level `MiddlewareServer`, gRPC lifecycle, and servicer adapter | | `bindings` | generated protobuf stubs — do not edit by hand | +## Updating the OpenShell protocol + +Privacy Guard uses +[`middleware-kit`](../middleware-kit/README.md) to keep its checked-in +OpenShell protocol and generated Python bindings aligned with a released +OpenShell version. Install the repository's local copy as the `mkit` tool: + +```bash +uv tool install --force ../middleware-kit +``` + +From this directory, update to the latest OpenShell release with: + +```bash +mkit update +``` + +For a reproducible update to a specific release, pin the version: + +```bash +mkit update --openshell-version v0.0.90 +``` + +The updater works on a validated temporary copy and replaces only +`proto/supervisor_middleware.proto`, `src/privacy_guard/bindings/`, `uv.lock`, +and `middleware-dev-manifest.json`. The manifest records the selected OpenShell +release, protocol source and checksum, and `middleware-kit` version. Review all +generated changes, then run `make check`; never edit generated bindings by +hand. + ## Notes for implementers - **Scanner metadata.** Applications pass an explicit `ScannerConfig` subtype to @@ -252,18 +282,31 @@ release its scanner slot until its synchronous worker really exits. arbitrary tracebacks. Cataloged errors and gRPC status details are content-safe, and caught collaborator exception chains must never be logged. -## Development validation +## Development checks -Run the complete local gate from this directory: +The project Makefile provides development shortcuts: ```bash -scripts/validate.sh +make help +make test PYTEST_ARGS="tests/test_processor.py -k redact" +make fix +make check ``` +`make check` delegates to the authoritative complete local check: + +```bash +scripts/check.sh +``` + +The top-level +`.github/workflows/privacy-guard.yml` workflow runs the same check in isolated +uv environments on the minimum and latest supported Python versions. + To exercise the package with the minimum supported interpreter, run -`scripts/validate.sh --python 3.11`. This project has no dedicated CI workflow, -so this committed entry point is the authoritative contributor gate. It runs the -full tests, formatting, lint, curated `ty` rules, import smoke, and package build. +`scripts/check.sh --python 3.11`. This committed script is the authoritative +project check used locally and in CI. It runs the full tests, formatting, lint, +curated `ty` rules, import smoke, and package build. The AST policy test rejects cast operations and explicit dynamic typing in handwritten `src`, `tests`, and `examples`; only generated protobuf/gRPC bindings are excluded. @@ -277,7 +320,7 @@ uv run --frozen python scripts/benchmark_privacy_guard.py ``` Use it manually when changing performance-sensitive processing code; it is not -part of `scripts/validate.sh` and does not enforce regression thresholds. For +part of `scripts/check.sh` and does not enforce regression thresholds. For the broader scenario set and seven samples per median, pass `--suite full`. Pass `--profile profile.out` to either suite to record a `cProfile` artifact. The harness reports median wall time and median peak traced allocation for the diff --git a/projects/privacy-guard/docs/architecture b/projects/privacy-guard/docs/architecture new file mode 120000 index 0000000..d63f714 --- /dev/null +++ b/projects/privacy-guard/docs/architecture @@ -0,0 +1 @@ +../../../docs/documentation/privacy-guard/architecture \ No newline at end of file diff --git a/projects/privacy-guard/docs/index.md b/projects/privacy-guard/docs/index.md new file mode 100644 index 0000000..f006a10 --- /dev/null +++ b/projects/privacy-guard/docs/index.md @@ -0,0 +1,6 @@ +# Privacy Guard documentation + +Technical documentation for people and coding agents working on Privacy Guard. + +- [Architecture](architecture/index.md): component boundaries, request flow, + extension interfaces, concurrency, failures, and limits diff --git a/projects/privacy-guard/examples/email-scanner/README.md b/projects/privacy-guard/examples/email-scanner/README.md index 5710c3e..b4a9ce7 100644 --- a/projects/privacy-guard/examples/email-scanner/README.md +++ b/projects/privacy-guard/examples/email-scanner/README.md @@ -10,11 +10,11 @@ CLI. The `EmailScanner` class in `middleware_server.py` demonstrates how to implement the scanner extension contract directly. It detects email-shaped text in Claude -Code request bodies. The policy replaces matches with `[email]` before Anthropic -receives the request. The implementation is intentionally small and is not -intended as comprehensive production PII detection. +Code request bodies. The policy replaces matches with `[email]` before the +remote model receives the request. The implementation is intentionally small +and is not intended as comprehensive production PII detection. -## Prerequisites +## Before you start - macOS with Docker Desktop running - Python 3 and `uv` @@ -24,6 +24,10 @@ intended as comprehensive production PII detection. curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh ``` +The foreground gateway commands below target the Homebrew installation on +macOS. The scanner, gateway config, and sandbox policy also work on Linux, but +use the service and TLS paths from that OpenShell installation. + Run all commands in this walkthrough from the example directory: ```bash @@ -33,17 +37,24 @@ cd projects/privacy-guard/examples/email-scanner `uv` automatically discovers the Privacy Guard project in the parent directories, so the commands do not need a `--project` option. +Use terminal 1 for Privacy Guard, terminal 2 for the foreground gateway, +terminal 3 for Claude Code, and another terminal for finite log checks. + ## 1. Generate the local gateway config -Generate a local config containing a host address that both the gateway and -sandboxes can reach: +Enter the IPv4 address of the host's physical Ethernet or Wi-Fi interface after +`YOUR_HOST_IP=`, then run both lines: ```bash -uv run python generate_gateway_config.py +YOUR_HOST_IP= +sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml +grep grpc_endpoint gateway.local.toml ``` -This writes the ignored `gateway.local.toml`. The checked-in `gateway.toml` -remains a reusable template. +Do not use `127.0.0.1`, a VPN address, or `host.openshell.internal`. The address +must be reachable from both the host gateway and sandbox supervisor. +The final command lets you verify the generated endpoint before starting +anything. ## 2. Start Privacy Guard @@ -74,11 +85,11 @@ openshell-gateway --config "$PWD/gateway.local.toml" The first command stops the background service so the foreground gateway can use the standard port. The second command reuses the credentials and state created by the recommended macOS installation, but loads this example's generated -`gateway.local.toml`. It should stay in the foreground; `Server listening` means -it is ready. +`gateway.local.toml`. The gateway should stay in the foreground. TLS startup +followed by `Server listening` means it is ready. -Middleware registration is static. After regenerating `gateway.local.toml`, stop -this foreground process with `Ctrl-C` and run the second command again. +Middleware registration is static. After regenerating `gateway.local.toml`, +stop this foreground process with `Ctrl-C` and run the second command again. ## 4. Create the sandbox and run Claude @@ -93,11 +104,11 @@ openshell gateway add \ openshell status openshell sandbox create \ - --name privacy-guard-lab \ + --name privacy-guard-email \ --from base \ --no-auto-providers \ --policy "$PWD/policy.yaml" \ - -- claude + -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude ``` `gateway add` saves the endpoint as the CLI's active gateway and refreshes its @@ -105,52 +116,79 @@ package-managed mTLS credentials. The registration continues to target the normal local gateway after this walkthrough. Choose Claude Code's subscription-account login and complete authentication. + +OpenShell can start a sandbox in a degraded state when its supervisor cannot +reach Privacy Guard. Before sending a prompt, use another terminal to inspect +the startup logs: + +```bash +openshell logs privacy-guard-email -n 100 +``` + +Do not continue if they contain `Middleware connect failed` or +`CONFIG:DEGRADED`; correct the host address and recreate the sandbox first. + Then enter: ```text Tell me something that rhymes with my email wendy@gmail.com ``` -Privacy Guard should replace the address with `[email]` before Claude sees it. -Model output is nondeterministic, so use OpenShell's logs as the authoritative -check: +Privacy Guard should replace the address with `[email]` before Anthropic +receives it. The reply must not be used to verify redaction because the model +may refer to the placeholder or invent an example address. + +In another terminal, inspect the finite recent log history: ```bash -openshell logs privacy-guard-lab --tail +openshell logs privacy-guard-email -n 100 ``` Look for the `api.anthropic.com/v1/messages` request with `transformed:true` and an email finding. +## Troubleshooting + +- If gateway startup reports `middleware registration failed`, confirm Privacy + Guard is still running and that `gateway.local.toml` contains the host's + physical Ethernet or Wi-Fi IPv4 address. +- If a prompt returns `403 "middleware_failed"`, inspect the sandbox logs. A + `Middleware connect failed` or `binding_not_described` entry means the sandbox + supervisor could not reach the configured address. Regenerate + `gateway.local.toml`, restart the foreground gateway, and recreate the sandbox. +- If logs report `middleware_timeout`, make sure the config does not contain a + VPN address and that the host firewall permits sandbox traffic to port 50051. + ## Change the behavior Edit `policy.yaml` and change `on_finding.action` to `observe`, `block`, or `redact`, then apply it without recreating the sandbox: ```bash -openshell policy set privacy-guard-lab \ +openshell policy set privacy-guard-email \ --policy "$PWD/policy.yaml" \ --wait ``` -- `redact` sends `[email]` to Claude. +- `redact` sends `[email]` to the remote model provider. - `observe` records the finding but sends the original email. - `block` denies the request. To reconnect later: ```bash -openshell sandbox connect privacy-guard-lab +openshell sandbox connect privacy-guard-email ``` -Then run `claude` inside the sandbox. +Then run `env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude` inside the +sandbox. ## Cleanup Exit Claude and the sandbox, then delete it: ```bash -openshell sandbox delete privacy-guard-lab +openshell sandbox delete privacy-guard-email ``` Stop the foreground gateway with `Ctrl-C` in terminal 2, then restore the normal diff --git a/projects/privacy-guard/examples/email-scanner/generate_gateway_config.py b/projects/privacy-guard/examples/email-scanner/generate_gateway_config.py deleted file mode 100644 index 6a4081e..0000000 --- a/projects/privacy-guard/examples/email-scanner/generate_gateway_config.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Generate a gateway config that both the host and sandboxes can reach.""" - -from __future__ import annotations - -import socket -from pathlib import Path - - -def main() -> None: - example_directory = Path(__file__).parent - template = (example_directory / "gateway.toml").read_text() - placeholder = "REPLACE_WITH_HOST_IP" - if template.count(placeholder) != 1: - raise RuntimeError("gateway.toml must contain exactly one host placeholder") - - host_ip = _discover_host_ipv4() - output_path = example_directory / "gateway.local.toml" - output_path.write_text(template.replace(placeholder, host_ip)) - print(f"Wrote {output_path.name} with middleware host {host_ip}") - - -def _discover_host_ipv4() -> str: - with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe: - probe.connect(("192.0.2.1", 80)) - return str(probe.getsockname()[0]) - - -if __name__ == "__main__": - main() diff --git a/projects/privacy-guard/examples/email-scanner/policy.yaml b/projects/privacy-guard/examples/email-scanner/policy.yaml index c117f9e..b305eb5 100644 --- a/projects/privacy-guard/examples/email-scanner/policy.yaml +++ b/projects/privacy-guard/examples/email-scanner/policy.yaml @@ -24,6 +24,8 @@ network_policies: protocol: rest enforcement: enforce access: full + - host: claude.ai + port: 443 - { host: statsig.anthropic.com, port: 443 } - { host: sentry.io, port: 443 } binaries: diff --git a/projects/privacy-guard/examples/regex-scanner/.gitignore b/projects/privacy-guard/examples/regex-scanner/.gitignore index b223234..2c00d38 100644 --- a/projects/privacy-guard/examples/regex-scanner/.gitignore +++ b/projects/privacy-guard/examples/regex-scanner/.gitignore @@ -1 +1,4 @@ gateway.local.toml +auth.json +pi-models.local.json +policy.local.yaml diff --git a/projects/privacy-guard/examples/regex-scanner/README.md b/projects/privacy-guard/examples/regex-scanner/README.md index d40fb4f..224a700 100644 --- a/projects/privacy-guard/examples/regex-scanner/README.md +++ b/projects/privacy-guard/examples/regex-scanner/README.md @@ -2,7 +2,7 @@ This self-contained example runs Privacy Guard's built-in `RegexScanner`, registers it with an OpenShell gateway, and uses a sandbox policy to redact an -email address and customer ID before Claude Code sends a request to Anthropic. +email address and customer ID before the remote model provider receives them. It is intended to be worked through by hand. The example temporarily runs the installed OpenShell gateway with the config in @@ -10,7 +10,7 @@ this directory. It does not create or modify `~/.config/openshell/gateway.toml` or create a project-local state directory. It registers the standard local gateway endpoint with the OpenShell CLI. -## Prerequisites +## Before you start - macOS with Docker Desktop running - Python 3 and `uv` @@ -20,6 +20,10 @@ gateway endpoint with the OpenShell CLI. curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh ``` +The foreground gateway commands below target the Homebrew installation on +macOS. The scanner, gateway config, and sandbox policy also work on Linux, but +use the service and TLS paths from that OpenShell installation. + Run all commands in this walkthrough from the example directory: ```bash @@ -29,6 +33,9 @@ cd projects/privacy-guard/examples/regex-scanner `uv` automatically discovers the Privacy Guard project in the parent directories, so the commands do not need a `--project` option. +Use terminal 1 for Privacy Guard, terminal 2 for the foreground gateway, +terminal 3 for the agent, and another terminal for finite log checks. + ## 1. Review the scanner configuration `regex-scanner.yaml` configures two entities: @@ -41,15 +48,19 @@ starting point, not comprehensive production detection. ## 2. Generate the local gateway config -Generate a local config containing a host address that both the gateway and -sandboxes can reach: +Enter the IPv4 address of the host's physical Ethernet or Wi-Fi interface after +`YOUR_HOST_IP=`, then run both lines: ```bash -uv run python generate_gateway_config.py +YOUR_HOST_IP= +sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml +grep grpc_endpoint gateway.local.toml ``` -This writes the ignored `gateway.local.toml`. The checked-in `gateway.toml` -remains a reusable template. +Do not use `127.0.0.1`, a VPN address, or `host.openshell.internal`. The address +must be reachable from both the host gateway and sandbox supervisor. +The final command lets you verify the generated endpoint before starting +anything. ## 3. Start Privacy Guard @@ -68,33 +79,6 @@ uv run privacy-guard regex \ Leave it running. Privacy Guard loads and compiles the scanner configuration before it binds the port. -To investigate request latency without logging request content, restart Privacy -Guard with `--debug` before the `regex` command: - -```bash -uv run privacy-guard --debug regex \ - --config regex-scanner.yaml \ - --listen 0.0.0.0:50051 -``` - -Debug output reports body size, text-block count, scanned character count, and -normalize, scan, and reconstruction timings. It never reports body content, -matched values, JSON paths, headers, or scanner configuration. - -To compare the complete body received by Privacy Guard with the body it returns -to OpenShell, use the explicit content-logging mode instead: - -```bash -uv run privacy-guard --debug-log-content regex \ - --config regex-scanner.yaml \ - --listen 0.0.0.0:50051 -``` - -Look for `stage=received` and `stage=forwarded` entries with the same -`request_id`. This mode also enables phase timings. It logs secrets by design, -so use it only for local debugging and disable it after capturing the failing -request. - ## 4. Run the installed gateway with the example config In terminal 2: @@ -109,13 +93,13 @@ openshell-gateway --config "$PWD/gateway.local.toml" The first command stops the background service so the foreground gateway can use the standard port. The second command reuses the credentials and state created by the recommended macOS installation, but loads this example's generated -`gateway.local.toml`. Keep it in the foreground; `Server listening` means it is -ready. +`gateway.local.toml`. Keep the gateway in the foreground. TLS startup followed +by `Server listening` means it is ready. -Middleware registration is static. After regenerating `gateway.local.toml`, stop -this foreground process with `Ctrl-C` and run the second command again. +Middleware registration is static. After regenerating `gateway.local.toml`, +stop this foreground process with `Ctrl-C` and run the second command again. -## 5. Create the sandbox and run Claude +## 5. Create the sandbox and run an agent In terminal 3: @@ -126,53 +110,186 @@ openshell gateway add \ --name openshell openshell status +``` +`gateway add` saves the endpoint as the CLI's active gateway and refreshes its +package-managed mTLS credentials. The registration continues to target the +normal local gateway after this walkthrough. + +Choose either of the following paths. Both use the same sandbox name, so run +only one create command. + +### Path A: Claude Code + +```bash openshell sandbox create \ - --name privacy-guard-regex-lab \ + --name privacy-guard-regex \ --from base \ --no-auto-providers \ --policy "$PWD/policy.yaml" \ -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude ``` -`gateway add` saves the endpoint as the CLI's active gateway and refreshes its -package-managed mTLS credentials. The registration continues to target the -normal local gateway after this walkthrough. - Disabling nonessential Claude Code traffic keeps telemetry and error-reporting batches out of this focused redaction exercise. Privacy Guard still processes the model requests sent to `api.anthropic.com`. Choose Claude Code's subscription-account login and complete authentication. -Then enter: + +### Path B: Pi + +This path supports a user-supplied OpenAI-compatible HTTPS endpoint on port 443. +Enter your endpoint URL, model ID, and API key after the three `=` signs: + +```bash +PI_MODEL_ENDPOINT= +PI_MODEL_ID= +export PI_MODEL_API_KEY= +``` + +Keep a suffix such as `/v1` in `PI_MODEL_ENDPOINT` when the provider requires +it. The setup below derives the policy hostname from that URL. + +Generate ignored local model and policy files: + +```bash +: "${PI_MODEL_ENDPOINT:?Set PI_MODEL_ENDPOINT above}" +: "${PI_MODEL_ID:?Set PI_MODEL_ID above}" +: "${PI_MODEL_API_KEY:?Set PI_MODEL_API_KEY above}" + +PI_MODEL_HOST=${PI_MODEL_ENDPOINT#*://} +PI_MODEL_HOST=${PI_MODEL_HOST%%/*} + +sed \ + -e "s|REPLACE_WITH_MODEL_ENDPOINT|$PI_MODEL_ENDPOINT|" \ + -e "s|REPLACE_WITH_MODEL_ID|$PI_MODEL_ID|g" \ + pi-models.template.json > pi-models.local.json + +sed "s/REPLACE_WITH_MODEL_HOST/$PI_MODEL_HOST/g" \ + policy.pi.template.yaml > policy.local.yaml +``` + +Create an OpenShell credential provider once. Passing the bare environment +variable name reads the value from the host without putting the key on the +command line: + +```bash +openshell provider create \ + --name privacy-guard-model \ + --type generic \ + --credential PI_MODEL_API_KEY +``` + +If `privacy-guard-model` already exists, skip `provider create` and confirm it +with `openshell provider get privacy-guard-model`. + +Create the sandbox with the generated model configuration and policy: + +```bash +openshell sandbox create \ + --name privacy-guard-regex \ + --from pi \ + --provider privacy-guard-model \ + --no-auto-providers \ + --policy "$PWD/policy.local.yaml" \ + --no-git-ignore \ + --upload pi-models.local.json:/sandbox/.pi/agent/models.json \ + -- pi \ + --provider custom \ + --model "$PI_MODEL_ID" +``` + +The `pi` community sandbox contains the Pi coding agent in addition to the base +sandbox tools. These options open its interactive TUI with the custom model +already selected. OpenShell injects an opaque credential placeholder into the +sandbox and substitutes the real key only in the TLS-terminated request to +the configured endpoint. `privacy-guard-model` is the OpenShell credential +provider; `custom` is the separate provider name in Pi's model configuration. + +### Confirm middleware startup + +OpenShell can start a sandbox in a degraded state when its supervisor cannot +reach Privacy Guard. Before sending a prompt, use another terminal to inspect +the startup logs: + +```bash +openshell logs privacy-guard-regex -n 100 +``` + +Do not continue if they contain `Middleware connect failed` or +`CONFIG:DEGRADED`; correct the host address and recreate the sandbox first. + +### Exercise the scanner + +In the agent you chose, enter: ```text Tell me something that rhymes with my email wendy@gmail.com and remember that my customer ID is CUST-12345678. ``` Privacy Guard should replace the values with `[email]` and `[customer-id]` -before Claude sees them. Model output is nondeterministic, so use OpenShell's -logs as the authoritative check: +before the remote model provider receives the request. The reply must not be +used to verify redaction: models may refer to the placeholders or invent +example values. + +In another terminal, inspect the finite recent log history: ```bash -openshell logs privacy-guard-regex-lab --tail +openshell logs privacy-guard-regex -n 100 ``` -Look for the `api.anthropic.com/v1/messages` request with `transformed:true` and -findings for both configured entities. +For Claude Code, look for the `api.anthropic.com/v1/messages` request. For Pi, +look for a request to the hostname assigned to `PI_MODEL_HOST`. The request +should have `transformed:true` and findings for both configured entities. + +## Troubleshooting + +- If gateway startup reports `middleware registration failed`, confirm Privacy + Guard is still running and that `gateway.local.toml` contains the host's + physical Ethernet or Wi-Fi IPv4 address. +- If a prompt returns `403 "middleware_failed"`, inspect the sandbox logs. A + `Middleware connect failed` or `binding_not_described` entry means the sandbox + supervisor could not reach the configured address. Regenerate + `gateway.local.toml`, restart the foreground gateway, and recreate the sandbox. +- If logs report `middleware_timeout`, make sure the config does not contain a + VPN address and that the host firewall permits sandbox traffic to port 50051. + +To investigate latency without logging request content, restart Privacy Guard +in terminal 1 with `--debug`: + +```bash +uv run privacy-guard --debug regex \ + --config regex-scanner.yaml \ + --listen 0.0.0.0:50051 +``` + +To compare the body received by Privacy Guard with the body it returns, use +`--debug-log-content` instead. This mode logs secrets by design: + +```bash +uv run privacy-guard --debug-log-content regex \ + --config regex-scanner.yaml \ + --listen 0.0.0.0:50051 +``` + +Match `stage=received` and `stage=forwarded` entries by `request_id`, and disable +content logging after capturing the failing request. ## Change the behavior -To change enforcement, edit `policy.yaml` and set `on_finding.action` to -`observe`, `block`, or `redact`, then apply it without recreating the sandbox: +To change enforcement, edit `policy.yaml` for Claude Code or +`policy.pi.template.yaml` for Pi and set `on_finding.action` to `observe`, +`block`, or `redact`. For Pi, regenerate `policy.local.yaml` with the command in +Path B. Then apply the selected policy without recreating the sandbox: ```bash -openshell policy set privacy-guard-regex-lab \ - --policy "$PWD/policy.yaml" \ +POLICY_FILE=policy.yaml # Use policy.local.yaml for Pi. +openshell policy set privacy-guard-regex \ + --policy "$PWD/$POLICY_FILE" \ --wait ``` -- `redact` sends `[email]` and `[customer-id]` to Claude. +- `redact` sends `[email]` and `[customer-id]` to the remote model provider. - `observe` records findings but sends the original values. - `block` denies a request containing either configured entity. @@ -183,17 +300,26 @@ startup. To reconnect later: ```bash -openshell sandbox connect privacy-guard-regex-lab +openshell sandbox connect privacy-guard-regex ``` -Then run `claude` inside the sandbox. +Then run the same launch command you chose earlier: + +```bash +env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude +# or, in a Pi sandbox: +PI_MODEL_ID= +pi \ + --provider custom \ + --model "$PI_MODEL_ID" +``` ## Cleanup -Exit Claude and the sandbox, then delete it: +Exit the agent and the sandbox, then delete it: ```bash -openshell sandbox delete privacy-guard-regex-lab +openshell sandbox delete privacy-guard-regex ``` Stop the foreground gateway with `Ctrl-C` in terminal 2, then restore the normal @@ -206,6 +332,7 @@ brew services start openshell Stop Privacy Guard with `Ctrl-C` in terminal 1. The example-specific gateway configuration was never installed as the default configuration. -This example uses Claude Code because subscription prompts are sent in -inspectable HTTP request bodies. ChatGPT-subscription Codex currently sends -prompts in WebSocket frames, which this HTTP middleware cannot inspect. +This example uses Claude Code with Anthropic or Pi with an OpenAI-compatible +endpoint because their prompts are sent in inspectable HTTP request bodies. +ChatGPT-subscription Codex currently sends prompts in WebSocket frames, which +this HTTP middleware cannot inspect. diff --git a/projects/privacy-guard/examples/regex-scanner/generate_gateway_config.py b/projects/privacy-guard/examples/regex-scanner/generate_gateway_config.py deleted file mode 100644 index 6a4081e..0000000 --- a/projects/privacy-guard/examples/regex-scanner/generate_gateway_config.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Generate a gateway config that both the host and sandboxes can reach.""" - -from __future__ import annotations - -import socket -from pathlib import Path - - -def main() -> None: - example_directory = Path(__file__).parent - template = (example_directory / "gateway.toml").read_text() - placeholder = "REPLACE_WITH_HOST_IP" - if template.count(placeholder) != 1: - raise RuntimeError("gateway.toml must contain exactly one host placeholder") - - host_ip = _discover_host_ipv4() - output_path = example_directory / "gateway.local.toml" - output_path.write_text(template.replace(placeholder, host_ip)) - print(f"Wrote {output_path.name} with middleware host {host_ip}") - - -def _discover_host_ipv4() -> str: - with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe: - probe.connect(("192.0.2.1", 80)) - return str(probe.getsockname()[0]) - - -if __name__ == "__main__": - main() diff --git a/projects/privacy-guard/examples/regex-scanner/pi-models.template.json b/projects/privacy-guard/examples/regex-scanner/pi-models.template.json new file mode 100644 index 0000000..dc71f19 --- /dev/null +++ b/projects/privacy-guard/examples/regex-scanner/pi-models.template.json @@ -0,0 +1,16 @@ +{ + "providers": { + "custom": { + "baseUrl": "REPLACE_WITH_MODEL_ENDPOINT", + "api": "openai-completions", + "apiKey": "$PI_MODEL_API_KEY", + "authHeader": true, + "models": [ + { + "id": "REPLACE_WITH_MODEL_ID", + "name": "REPLACE_WITH_MODEL_ID" + } + ] + } + } +} diff --git a/projects/privacy-guard/examples/regex-scanner/policy.pi.template.yaml b/projects/privacy-guard/examples/regex-scanner/policy.pi.template.yaml new file mode 100644 index 0000000..96a1d7b --- /dev/null +++ b/projects/privacy-guard/examples/regex-scanner/policy.pi.template.yaml @@ -0,0 +1,42 @@ +version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + pi: + name: Pi access to the configured model endpoint + endpoints: + - host: REPLACE_WITH_MODEL_HOST + port: 443 + protocol: rest + enforcement: enforce + access: full + - { host: pi.dev, port: 443 } + binaries: + - { path: /usr/bin/pi } + - { path: /usr/lib/node_modules/@earendil-works/pi-coding-agent/dist/cli.js } + - { path: /usr/bin/node } + +network_middlewares: + privacy_guard_redaction: + name: Redact configured email addresses and customer IDs + middleware: privacy-guard-regex-scanner + order: 0 + config: + body_format: json + on_finding: + action: redact + entity_types: [email, customer-id] + minimum_confidence: high + on_error: fail_closed + endpoints: + include: + - REPLACE_WITH_MODEL_HOST diff --git a/projects/privacy-guard/examples/regex-scanner/policy.yaml b/projects/privacy-guard/examples/regex-scanner/policy.yaml index bd7822f..4dc7675 100644 --- a/projects/privacy-guard/examples/regex-scanner/policy.yaml +++ b/projects/privacy-guard/examples/regex-scanner/policy.yaml @@ -19,6 +19,8 @@ network_policies: protocol: rest enforcement: enforce access: full + - host: claude.ai + port: 443 - host: platform.claude.com port: 443 protocol: rest diff --git a/projects/privacy-guard/middleware-dev-manifest.json b/projects/privacy-guard/middleware-dev-manifest.json index 6698ff1..74f1e59 100644 --- a/projects/privacy-guard/middleware-dev-manifest.json +++ b/projects/privacy-guard/middleware-dev-manifest.json @@ -1,7 +1,13 @@ { - "openshell_version": "v0.0.86", - "proto_source": "https://raw.githubusercontent.com/NVIDIA/OpenShell/v0.0.86/proto/supervisor_middleware.proto", + "openshell_version": "v0.0.90", + "proto_source": "https://raw.githubusercontent.com/NVIDIA/OpenShell/v0.0.90/proto/supervisor_middleware.proto", "proto_sha256": "e9d5a992ff5b50a33e9625176aaf6df8496d6774aa2ef3afe5cae7bc83c01105", - "languages": ["python"], - "python_package": "privacy_guard" + "languages": [ + "python" + ], + "python_package": "privacy_guard", + "generator": { + "name": "middleware-kit", + "version": "0.1.0" + } } diff --git a/projects/privacy-guard/scripts/validate.sh b/projects/privacy-guard/scripts/check.sh similarity index 86% rename from projects/privacy-guard/scripts/validate.sh rename to projects/privacy-guard/scripts/check.sh index 2eb9c60..93bd999 100755 --- a/projects/privacy-guard/scripts/validate.sh +++ b/projects/privacy-guard/scripts/check.sh @@ -6,7 +6,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.." uv_run=(uv run --frozen) if [[ $# -gt 0 ]]; then if [[ $1 != "--python" || $# -ne 2 ]]; then - echo "usage: scripts/validate.sh [--python VERSION]" >&2 + echo "usage: scripts/check.sh [--python VERSION]" >&2 exit 2 fi uv_run+=(--python "$2") diff --git a/projects/privacy-guard/src/privacy_guard/bindings/__init__.py b/projects/privacy-guard/src/privacy_guard/bindings/__init__.py index d168201..d206506 100644 --- a/projects/privacy-guard/src/privacy_guard/bindings/__init__.py +++ b/projects/privacy-guard/src/privacy_guard/bindings/__init__.py @@ -1 +1 @@ -"""Generated OpenShell supervisor middleware bindings.""" +"""Generated OpenShell supervisor middleware bindings. Do not edit.""" diff --git a/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2_grpc.py b/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2_grpc.py index 3800987..a4914b3 100644 --- a/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2_grpc.py +++ b/projects/privacy-guard/src/privacy_guard/bindings/supervisor_middleware_pb2_grpc.py @@ -1,31 +1,28 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" - import grpc -from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +import warnings +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 from . import supervisor_middleware_pb2 as supervisor__middleware__pb2 -GRPC_GENERATED_VERSION = "1.81.1" +GRPC_GENERATED_VERSION = '1.81.1' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower - - _version_not_supported = first_version_is_lower( - GRPC_VERSION, GRPC_GENERATED_VERSION - ) + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( - f"The grpc package installed is at version {GRPC_VERSION}," - + " but the generated code in supervisor_middleware_pb2_grpc.py depends on" - + f" grpcio>={GRPC_GENERATED_VERSION}." - + f" Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}" - + f" or downgrade your generated code using grpcio-tools<={GRPC_VERSION}." + f'The grpc package installed is at version {GRPC_VERSION},' + + ' but the generated code in supervisor_middleware_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) @@ -41,23 +38,20 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Describe = channel.unary_unary( - "/openshell.middleware.v1.SupervisorMiddleware/Describe", - request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, - response_deserializer=supervisor__middleware__pb2.MiddlewareManifest.FromString, - _registered_method=True, - ) + '/openshell.middleware.v1.SupervisorMiddleware/Describe', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=supervisor__middleware__pb2.MiddlewareManifest.FromString, + _registered_method=True) self.ValidateConfig = channel.unary_unary( - "/openshell.middleware.v1.SupervisorMiddleware/ValidateConfig", - request_serializer=supervisor__middleware__pb2.ValidateConfigRequest.SerializeToString, - response_deserializer=supervisor__middleware__pb2.ValidateConfigResponse.FromString, - _registered_method=True, - ) + '/openshell.middleware.v1.SupervisorMiddleware/ValidateConfig', + request_serializer=supervisor__middleware__pb2.ValidateConfigRequest.SerializeToString, + response_deserializer=supervisor__middleware__pb2.ValidateConfigResponse.FromString, + _registered_method=True) self.EvaluateHttpRequest = channel.unary_unary( - "/openshell.middleware.v1.SupervisorMiddleware/EvaluateHttpRequest", - request_serializer=supervisor__middleware__pb2.HttpRequestEvaluation.SerializeToString, - response_deserializer=supervisor__middleware__pb2.HttpRequestResult.FromString, - _registered_method=True, - ) + '/openshell.middleware.v1.SupervisorMiddleware/EvaluateHttpRequest', + request_serializer=supervisor__middleware__pb2.HttpRequestEvaluation.SerializeToString, + response_deserializer=supervisor__middleware__pb2.HttpRequestResult.FromString, + _registered_method=True) class SupervisorMiddlewareServicer: @@ -66,76 +60,73 @@ class SupervisorMiddlewareServicer: """ def Describe(self, request, context): - """Describe returns the service manifest and declared bindings.""" + """Describe returns the service manifest and declared bindings. + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details("Method not implemented!") - raise NotImplementedError("Method not implemented!") + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') def ValidateConfig(self, request, context): - """ValidateConfig checks service-specific configuration for one binding.""" + """ValidateConfig checks service-specific configuration for one binding. + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details("Method not implemented!") - raise NotImplementedError("Method not implemented!") + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') def EvaluateHttpRequest(self, request, context): """EvaluateHttpRequest returns an allow, deny, or mutation decision for one buffered HTTP request. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details("Method not implemented!") - raise NotImplementedError("Method not implemented!") + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') def add_SupervisorMiddlewareServicer_to_server(servicer, server): rpc_method_handlers = { - "Describe": grpc.unary_unary_rpc_method_handler( - servicer.Describe, - request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, - response_serializer=supervisor__middleware__pb2.MiddlewareManifest.SerializeToString, - ), - "ValidateConfig": grpc.unary_unary_rpc_method_handler( - servicer.ValidateConfig, - request_deserializer=supervisor__middleware__pb2.ValidateConfigRequest.FromString, - response_serializer=supervisor__middleware__pb2.ValidateConfigResponse.SerializeToString, - ), - "EvaluateHttpRequest": grpc.unary_unary_rpc_method_handler( - servicer.EvaluateHttpRequest, - request_deserializer=supervisor__middleware__pb2.HttpRequestEvaluation.FromString, - response_serializer=supervisor__middleware__pb2.HttpRequestResult.SerializeToString, - ), + 'Describe': grpc.unary_unary_rpc_method_handler( + servicer.Describe, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=supervisor__middleware__pb2.MiddlewareManifest.SerializeToString, + ), + 'ValidateConfig': grpc.unary_unary_rpc_method_handler( + servicer.ValidateConfig, + request_deserializer=supervisor__middleware__pb2.ValidateConfigRequest.FromString, + response_serializer=supervisor__middleware__pb2.ValidateConfigResponse.SerializeToString, + ), + 'EvaluateHttpRequest': grpc.unary_unary_rpc_method_handler( + servicer.EvaluateHttpRequest, + request_deserializer=supervisor__middleware__pb2.HttpRequestEvaluation.FromString, + response_serializer=supervisor__middleware__pb2.HttpRequestResult.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( - "openshell.middleware.v1.SupervisorMiddleware", rpc_method_handlers - ) + 'openshell.middleware.v1.SupervisorMiddleware', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) - server.add_registered_method_handlers( - "openshell.middleware.v1.SupervisorMiddleware", rpc_method_handlers - ) + server.add_registered_method_handlers('openshell.middleware.v1.SupervisorMiddleware', rpc_method_handlers) -# This class is part of an EXPERIMENTAL API. + # This class is part of an EXPERIMENTAL API. class SupervisorMiddleware: """SupervisorMiddleware lets an operator-run service inspect and transform sandbox HTTP egress before OpenShell injects credentials. """ @staticmethod - def Describe( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): + def Describe(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): return grpc.experimental.unary_unary( request, target, - "/openshell.middleware.v1.SupervisorMiddleware/Describe", + '/openshell.middleware.v1.SupervisorMiddleware/Describe', google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, supervisor__middleware__pb2.MiddlewareManifest.FromString, options, @@ -146,26 +137,23 @@ def Describe( wait_for_ready, timeout, metadata, - _registered_method=True, - ) + _registered_method=True) @staticmethod - def ValidateConfig( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): + def ValidateConfig(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): return grpc.experimental.unary_unary( request, target, - "/openshell.middleware.v1.SupervisorMiddleware/ValidateConfig", + '/openshell.middleware.v1.SupervisorMiddleware/ValidateConfig', supervisor__middleware__pb2.ValidateConfigRequest.SerializeToString, supervisor__middleware__pb2.ValidateConfigResponse.FromString, options, @@ -176,26 +164,23 @@ def ValidateConfig( wait_for_ready, timeout, metadata, - _registered_method=True, - ) + _registered_method=True) @staticmethod - def EvaluateHttpRequest( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): + def EvaluateHttpRequest(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): return grpc.experimental.unary_unary( request, target, - "/openshell.middleware.v1.SupervisorMiddleware/EvaluateHttpRequest", + '/openshell.middleware.v1.SupervisorMiddleware/EvaluateHttpRequest', supervisor__middleware__pb2.HttpRequestEvaluation.SerializeToString, supervisor__middleware__pb2.HttpRequestResult.FromString, options, @@ -206,5 +191,4 @@ def EvaluateHttpRequest( wait_for_ready, timeout, metadata, - _registered_method=True, - ) + _registered_method=True) diff --git a/projects/privacy-guard/tests/examples/test_email_scanner.py b/projects/privacy-guard/tests/examples/test_email_scanner.py index 3fa0ae9..a9aa6a2 100644 --- a/projects/privacy-guard/tests/examples/test_email_scanner.py +++ b/projects/privacy-guard/tests/examples/test_email_scanner.py @@ -34,15 +34,24 @@ def test_example_configuration_targets_its_local_middleware() -> None: assert 'name = "privacy-guard-email-scanner"' in gateway assert 'grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051"' in gateway assert "gateway.local.toml" in gitignore - assert (EXAMPLE_DIRECTORY / "generate_gateway_config.py").is_file() assert "action: redact" in policy assert "entity_types: [email]" in policy assert "cd projects/privacy-guard/examples/email-scanner" in readme assert "uv run python middleware_server.py" in readme - assert "uv run python generate_gateway_config.py" in readme + assert "YOUR_HOST_IP=" in readme + assert ( + 'sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml' + ) in readme + assert "grep grpc_endpoint gateway.local.toml" in readme assert "--listen 0.0.0.0:50051" in readme assert 'openshell-gateway --config "$PWD/gateway.local.toml"' in readme assert "openshell gateway add" in readme assert "https://127.0.0.1:17670" in readme assert "--name openshell" in readme + assert "--name privacy-guard-email" in readme + assert "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude" in readme + assert "host: claude.ai" in policy + assert "Middleware connect failed" in readme + assert '403 "middleware_failed"' in readme + assert "--tail" not in readme assert "uv run --project" not in readme diff --git a/projects/privacy-guard/tests/examples/test_regex_scanner.py b/projects/privacy-guard/tests/examples/test_regex_scanner.py index bde574b..9fb8dc4 100644 --- a/projects/privacy-guard/tests/examples/test_regex_scanner.py +++ b/projects/privacy-guard/tests/examples/test_regex_scanner.py @@ -1,5 +1,9 @@ +import json +import re from pathlib import Path +import yaml + from privacy_guard.constants import PATTERN_NAME_METADATA_KEY from privacy_guard.scanners import RegexScanner @@ -22,28 +26,91 @@ def test_example_scanner_detects_email_and_customer_id() -> None: ] +def test_pi_templates_render_for_an_openai_compatible_endpoint() -> None: + model_template = (EXAMPLE_DIRECTORY / "pi-models.template.json").read_text() + policy_template = (EXAMPLE_DIRECTORY / "policy.pi.template.yaml").read_text() + + models = json.loads( + model_template.replace( + "REPLACE_WITH_MODEL_ENDPOINT", "https://api.example.com/v1" + ).replace("REPLACE_WITH_MODEL_ID", "example/model") + ) + policy = yaml.safe_load( + policy_template.replace("REPLACE_WITH_MODEL_HOST", "api.example.com") + ) + + assert models["providers"]["custom"]["baseUrl"] == "https://api.example.com/v1" + assert models["providers"]["custom"]["models"][0]["id"] == "example/model" + assert policy["network_policies"]["pi"]["endpoints"][0]["host"] == "api.example.com" + assert policy["network_middlewares"]["privacy_guard_redaction"]["endpoints"][ + "include" + ] == ["api.example.com"] + + def test_example_configuration_and_walkthrough_are_aligned() -> None: policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() gitignore = (EXAMPLE_DIRECTORY / ".gitignore").read_text() readme = (EXAMPLE_DIRECTORY / "README.md").read_text() + pi_policy = (EXAMPLE_DIRECTORY / "policy.pi.template.yaml").read_text() + pi_models = json.loads((EXAMPLE_DIRECTORY / "pi-models.template.json").read_text()) + pi_provider = pi_models["providers"]["custom"] assert "middleware: privacy-guard-regex-scanner" in policy assert 'name = "privacy-guard-regex-scanner"' in gateway assert 'grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051"' in gateway assert "gateway.local.toml" in gitignore - assert (EXAMPLE_DIRECTORY / "generate_gateway_config.py").is_file() assert "action: redact" in policy assert "entity_types: [email, customer-id]" in policy assert "cd projects/privacy-guard/examples/regex-scanner" in readme assert "privacy-guard regex" in readme assert "--config regex-scanner.yaml" in readme - assert "uv run python generate_gateway_config.py" in readme + assert "YOUR_HOST_IP=" in readme + assert ( + 'sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml' + ) in readme + assert "grep grpc_endpoint gateway.local.toml" in readme assert "--listen 0.0.0.0:50051" in readme assert 'openshell-gateway --config "$PWD/gateway.local.toml"' in readme assert "openshell gateway add" in readme assert "https://127.0.0.1:17670" in readme assert "--name openshell" in readme assert "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude" in readme - assert "privacy-guard-regex-lab" in readme + assert "--from pi" in readme + assert "PI_MODEL_ENDPOINT=\nPI_MODEL_ID=\nexport PI_MODEL_API_KEY=" in readme + assert "PI_MODEL_HOST=${PI_MODEL_ENDPOINT#*://}" in readme + assert "--provider privacy-guard-model" in readme + assert "--credential PI_MODEL_API_KEY" in readme + assert "--no-git-ignore" in readme + assert "--upload pi-models.local.json:/sandbox/.pi/agent/models.json" in readme + assert "--provider custom" in readme + assert '--model "$PI_MODEL_ID"' in readme + assert "interactive TUI" in readme + assert "/usr/bin/pi" in pi_policy + assert "/usr/local/bin/pi" not in pi_policy + assert "host: pi.dev" in pi_policy + assert "host: REPLACE_WITH_MODEL_HOST" in pi_policy + assert "REPLACE_WITH_MODEL_HOST" not in policy + assert pi_provider["baseUrl"] == "REPLACE_WITH_MODEL_ENDPOINT" + assert pi_provider["api"] == "openai-completions" + assert pi_provider["apiKey"] == "$PI_MODEL_API_KEY" + assert pi_provider["models"] == [ + { + "id": "REPLACE_WITH_MODEL_ID", + "name": "REPLACE_WITH_MODEL_ID", + } + ] + assert "pi-models.local.json" in gitignore + assert "policy.local.yaml" in gitignore + assert "auth.json" in gitignore + assert "Middleware connect failed" in readme + assert '403 "middleware_failed"' in readme + assert "--tail" not in readme + assert "privacy-guard-regex" in readme + sandbox_names = re.findall( + r"openshell sandbox create \\\n\s+--name ([a-z0-9-]+)", readme + ) + assert sandbox_names + assert all(name.startswith("privacy-guard-") for name in sandbox_names) + assert all(len(name) <= 19 for name in sandbox_names) assert "uv run --project" not in readme diff --git a/zensical.toml b/zensical.toml index ebff6b9..5f5bf4a 100644 --- a/zensical.toml +++ b/zensical.toml @@ -21,7 +21,17 @@ nav = [ ]} ]}, {"Documentation" = [ - "documentation/index.md" + "documentation/index.md", + {"Privacy Guard" = [ + {"Architecture" = [ + "documentation/privacy-guard/architecture/index.md", + {"Request lifecycle" = "documentation/privacy-guard/architecture/request-lifecycle.md"}, + {"Scanners" = "documentation/privacy-guard/architecture/scanners.md"}, + {"Format handlers" = "documentation/privacy-guard/architecture/format-handlers.md"}, + {"Service boundary" = "documentation/privacy-guard/architecture/service-boundary.md"}, + {"Safety and limits" = "documentation/privacy-guard/architecture/safety-and-limits.md"} + ]} + ]} ]} ] From 82898dafc5602355f62c33c0c014df216aadf72d Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 16:19:54 -0400 Subject: [PATCH 24/82] Fix Privacy Guard CI runner paths --- .github/workflows/privacy-guard.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/privacy-guard.yml b/.github/workflows/privacy-guard.yml index fa36cc2..07a9c3b 100644 --- a/.github/workflows/privacy-guard.yml +++ b/.github/workflows/privacy-guard.yml @@ -27,9 +27,6 @@ jobs: defaults: run: working-directory: projects/privacy-guard - env: - UV_CACHE_DIR: ${{ runner.temp }}/privacy-guard-uv-cache - UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/privacy-guard-venv steps: - name: Checkout uses: actions/checkout@v7 @@ -42,6 +39,11 @@ jobs: version: "0.11.31" python-version: ${{ matrix.python-version }} + - name: Configure isolated uv paths + run: | + echo "UV_CACHE_DIR=$RUNNER_TEMP/privacy-guard-uv-cache" >> "$GITHUB_ENV" + echo "UV_PROJECT_ENVIRONMENT=$RUNNER_TEMP/privacy-guard-venv" >> "$GITHUB_ENV" + - name: Install locked dependencies run: uv sync --frozen From be6af1295c00a3ee7c769ebb332a31814b491bb8 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 23 Jul 2026 16:24:34 -0400 Subject: [PATCH 25/82] Make Privacy Guard CLI tests ANSI-safe --- .../tests/service/test_server.py | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index b72e91b..5d82909 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -1,4 +1,5 @@ import logging +import re from collections.abc import Sequence from pathlib import Path @@ -6,7 +7,7 @@ import pytest from google.protobuf import empty_pb2, message_factory from google.protobuf.message import Message -from typer.testing import CliRunner +from typer.testing import CliRunner, Result from typing_extensions import override from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 @@ -21,6 +22,8 @@ from ..scanner_helpers import DeterministicEmailScanner +_ANSI_STYLE_PATTERN = re.compile(r"\x1b\[[0-9;]*m") + class LifecycleServerFake(grpc.aio.Server): """Nominal gRPC server fake for lifecycle-only server tests.""" @@ -64,6 +67,10 @@ def _middleware() -> PrivacyGuardMiddleware: return PrivacyGuardMiddleware(RequestProcessor([scanner])) +def _plain_output(result: Result) -> str: + return _ANSI_STYLE_PATTERN.sub("", result.output) + + def test_middleware_server_wires_scanner_and_has_a_default_listen_address( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -240,26 +247,28 @@ def test_cli_always_requires_scanner_configuration() -> None: result = CliRunner().invoke(app, ["regex"]) assert result.exit_code == 2 - assert "--config" in result.output + assert "--config" in _plain_output(result) def test_cli_exposes_root_and_builtin_scanner_help() -> None: root_help = CliRunner().invoke(app, ["--help"]) assert root_help.exit_code == 0 - assert "privacy-guard" in root_help.output - assert "regex" in root_help.output - assert "--debug" in root_help.output - assert "--debug-log-content" in root_help.output + plain_root_help = _plain_output(root_help) + assert "privacy-guard" in plain_root_help + assert "regex" in plain_root_help + assert "--debug" in plain_root_help + assert "--debug-log-content" in plain_root_help scanner_help = CliRunner().invoke(app, ["regex", "--help"]) assert scanner_help.exit_code == 0 - assert "built-in RegexScanner" in scanner_help.output - assert "--config" in scanner_help.output - assert "--listen" in scanner_help.output - assert "--profile" in scanner_help.output - assert "--scanner-name" in scanner_help.output + plain_scanner_help = _plain_output(scanner_help) + assert "built-in RegexScanner" in plain_scanner_help + assert "--config" in plain_scanner_help + assert "--listen" in plain_scanner_help + assert "--profile" in plain_scanner_help + assert "--scanner-name" in plain_scanner_help def test_cli_enables_explicit_request_content_logging( @@ -319,7 +328,7 @@ def test_cli_rejects_builtin_options_before_subcommand() -> None: ) assert result.exit_code == 2 - assert "--profile" in result.output + assert "--profile" in _plain_output(result) @pytest.mark.asyncio From 5e4924f2490a65d0d6f5cd21ed71b16af538eeb3 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 24 Jul 2026 09:51:22 -0400 Subject: [PATCH 26/82] Update Privacy Guard middleware manifest --- ...on => .openshell-middleware-manifest.json} | 2 +- projects/privacy-guard/README.md | 21 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) rename projects/privacy-guard/{middleware-dev-manifest.json => .openshell-middleware-manifest.json} (89%) diff --git a/projects/privacy-guard/middleware-dev-manifest.json b/projects/privacy-guard/.openshell-middleware-manifest.json similarity index 89% rename from projects/privacy-guard/middleware-dev-manifest.json rename to projects/privacy-guard/.openshell-middleware-manifest.json index 74f1e59..d596981 100644 --- a/projects/privacy-guard/middleware-dev-manifest.json +++ b/projects/privacy-guard/.openshell-middleware-manifest.json @@ -7,7 +7,7 @@ ], "python_package": "privacy_guard", "generator": { - "name": "middleware-kit", + "name": "openshell-middleware-kit", "version": "0.1.0" } } diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 2c69da8..529f84a 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -194,32 +194,33 @@ release its scanner slot until its synchronous worker really exits. ## Updating the OpenShell protocol Privacy Guard uses -[`middleware-kit`](../middleware-kit/README.md) to keep its checked-in -OpenShell protocol and generated Python bindings aligned with a released -OpenShell version. Install the repository's local copy as the `mkit` tool: +[`openshell-middleware-kit`](../openshell-middleware-kit/README.md) to keep its +checked-in OpenShell protocol and generated Python bindings aligned with a +released OpenShell version. Install the repository's local copy as the `omkit` +tool: ```bash -uv tool install --force ../middleware-kit +uv tool install --force ../openshell-middleware-kit ``` From this directory, update to the latest OpenShell release with: ```bash -mkit update +omkit update ``` For a reproducible update to a specific release, pin the version: ```bash -mkit update --openshell-version v0.0.90 +omkit update --openshell-version v0.0.90 ``` The updater works on a validated temporary copy and replaces only `proto/supervisor_middleware.proto`, `src/privacy_guard/bindings/`, `uv.lock`, -and `middleware-dev-manifest.json`. The manifest records the selected OpenShell -release, protocol source and checksum, and `middleware-kit` version. Review all -generated changes, then run `make check`; never edit generated bindings by -hand. +and `.openshell-middleware-manifest.json`. The manifest records the selected +OpenShell release, protocol source and checksum, and +`openshell-middleware-kit` version. Review all generated changes, then run +`make check`; never edit generated bindings by hand. ## Notes for implementers From ce3a8017c1a22c9d62b21d2fd1da540413b3e7a9 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 24 Jul 2026 19:32:53 +0000 Subject: [PATCH 27/82] Refactor Privacy Guard around entity processing engines --- projects/privacy-guard/Makefile | 21 +- projects/privacy-guard/pyproject.toml | 4 +- .../scripts/benchmark_privacy_guard.py | 319 ------- .../privacy-guard/src/privacy_guard/base.py | 18 + .../privacy-guard/src/privacy_guard/config.py | 277 +++--- .../src/privacy_guard/constants.py | 34 +- .../src/privacy_guard/engine_registry.py | 259 ++++++ .../src/privacy_guard/engines/__init__.py | 53 ++ .../src/privacy_guard/engines/base.py | 339 ++++++++ .../src/privacy_guard/engines/regex.py | 460 ++++++++++ .../privacy-guard/src/privacy_guard/errors.py | 114 +-- .../src/privacy_guard/payloads/__init__.py | 8 - .../src/privacy_guard/payloads/request.py | 32 - .../src/privacy_guard/payloads/result.py | 26 - .../src/privacy_guard/processor.py | 517 ------------ .../privacy_guard/request_body/__init__.py | 45 - .../src/privacy_guard/request_body/base.py | 117 --- .../src/privacy_guard/request_body/json.py | 300 ------- .../src/privacy_guard/request_processor.py | 222 +++++ .../src/privacy_guard/scanners/__init__.py | 39 - .../src/privacy_guard/scanners/base.py | 225 ----- .../src/privacy_guard/scanners/regex.py | 437 ---------- .../src/privacy_guard/service/__init__.py | 7 +- .../src/privacy_guard/service/server.py | 124 ++- .../src/privacy_guard/service/servicer.py | 296 ++++--- .../src/privacy_guard/string_validators.py | 44 + .../src/privacy_guard/timeout.py | 49 ++ .../src/privacy_guard/validation.py | 58 -- .../privacy-guard/tests/engines/test_base.py | 208 +++++ .../privacy-guard/tests/engines/test_regex.py | 286 +++++++ .../tests/examples/test_email_scanner.py | 57 -- .../tests/examples/test_regex_scanner.py | 116 --- .../tests/request_body/test_base.py | 122 --- .../tests/request_body/test_json.py | 580 ------------- .../privacy-guard/tests/scanner_helpers.py | 34 - .../tests/scanners/test_regex.py | 277 ------ .../tests/scanners/test_scanner_models.py | 175 ---- .../tests/service/test_server.py | 654 ++++----------- .../tests/service/test_servicer.py | 570 +++---------- projects/privacy-guard/tests/test_config.py | 495 +++++------ .../tests/test_engine_registry.py | 186 +++++ projects/privacy-guard/tests/test_errors.py | 2 +- .../privacy-guard/tests/test_hardening.py | 600 ------------- projects/privacy-guard/tests/test_payloads.py | 118 --- .../privacy-guard/tests/test_processor.py | 786 ------------------ .../tests/test_request_processor.py | 109 +++ .../tests/test_single_block_composition.py | 47 -- projects/privacy-guard/uv.lock | 153 ++-- 48 files changed, 3237 insertions(+), 6782 deletions(-) delete mode 100755 projects/privacy-guard/scripts/benchmark_privacy_guard.py create mode 100644 projects/privacy-guard/src/privacy_guard/base.py create mode 100644 projects/privacy-guard/src/privacy_guard/engine_registry.py create mode 100644 projects/privacy-guard/src/privacy_guard/engines/__init__.py create mode 100644 projects/privacy-guard/src/privacy_guard/engines/base.py create mode 100644 projects/privacy-guard/src/privacy_guard/engines/regex.py delete mode 100644 projects/privacy-guard/src/privacy_guard/payloads/__init__.py delete mode 100644 projects/privacy-guard/src/privacy_guard/payloads/request.py delete mode 100644 projects/privacy-guard/src/privacy_guard/payloads/result.py delete mode 100644 projects/privacy-guard/src/privacy_guard/processor.py delete mode 100644 projects/privacy-guard/src/privacy_guard/request_body/__init__.py delete mode 100644 projects/privacy-guard/src/privacy_guard/request_body/base.py delete mode 100644 projects/privacy-guard/src/privacy_guard/request_body/json.py create mode 100644 projects/privacy-guard/src/privacy_guard/request_processor.py delete mode 100644 projects/privacy-guard/src/privacy_guard/scanners/__init__.py delete mode 100644 projects/privacy-guard/src/privacy_guard/scanners/base.py delete mode 100644 projects/privacy-guard/src/privacy_guard/scanners/regex.py create mode 100644 projects/privacy-guard/src/privacy_guard/string_validators.py create mode 100644 projects/privacy-guard/src/privacy_guard/timeout.py delete mode 100644 projects/privacy-guard/src/privacy_guard/validation.py create mode 100644 projects/privacy-guard/tests/engines/test_base.py create mode 100644 projects/privacy-guard/tests/engines/test_regex.py delete mode 100644 projects/privacy-guard/tests/examples/test_email_scanner.py delete mode 100644 projects/privacy-guard/tests/examples/test_regex_scanner.py delete mode 100644 projects/privacy-guard/tests/request_body/test_base.py delete mode 100644 projects/privacy-guard/tests/request_body/test_json.py delete mode 100644 projects/privacy-guard/tests/scanner_helpers.py delete mode 100644 projects/privacy-guard/tests/scanners/test_regex.py delete mode 100644 projects/privacy-guard/tests/scanners/test_scanner_models.py create mode 100644 projects/privacy-guard/tests/test_engine_registry.py delete mode 100644 projects/privacy-guard/tests/test_hardening.py delete mode 100644 projects/privacy-guard/tests/test_payloads.py delete mode 100644 projects/privacy-guard/tests/test_processor.py create mode 100644 projects/privacy-guard/tests/test_request_processor.py delete mode 100644 projects/privacy-guard/tests/test_single_block_composition.py diff --git a/projects/privacy-guard/Makefile b/projects/privacy-guard/Makefile index b4df0d8..95223f1 100644 --- a/projects/privacy-guard/Makefile +++ b/projects/privacy-guard/Makefile @@ -3,21 +3,17 @@ UV ?= uv UV_RUN := $(UV) run --frozen PYTEST_ARGS ?= -PROFILE ?= /tmp/privacy-guard.prof - .PHONY: \ help sync lock lock-check \ - test test-examples test-hardening \ + test \ format format-check lint lint-fix typecheck fix \ import-check build check check-py311 \ - benchmark benchmark-full benchmark-profile \ clean help: ## Show available targets and configurable variables. @awk 'BEGIN {FS = ":.*## "}; /^[a-zA-Z0-9_.-]+:.*## / {printf " %-28s %s\n", $$1, $$2}' $(MAKEFILE_LIST) @printf "\nVariables:\n" @printf " PYTEST_ARGS=... Extra pytest paths or flags\n" - @printf " PROFILE=... Benchmark profile path (default: %s)\n" "$(PROFILE)" sync: ## Install locked runtime and development dependencies. $(UV) sync --frozen @@ -31,12 +27,6 @@ lock-check: ## Verify uv.lock matches pyproject.toml. test: ## Run tests; pass paths or flags with PYTEST_ARGS. $(UV_RUN) pytest -q $(PYTEST_ARGS) -test-examples: ## Run example integration tests. - $(UV_RUN) pytest -q tests/examples - -test-hardening: ## Run security and typing-policy tests. - $(UV_RUN) pytest -q tests/test_hardening.py tests/test_typing_policy.py - format: ## Format handwritten Python. $(UV_RUN) ruff format . @@ -68,15 +58,6 @@ check: ## Run the authoritative full project checks. check-py311: ## Run full checks on the minimum Python version. scripts/check.sh --python 3.11 -benchmark: ## Run the quick processing benchmark. - $(UV_RUN) python scripts/benchmark_privacy_guard.py - -benchmark-full: ## Run the full processing benchmark suite. - $(UV_RUN) python scripts/benchmark_privacy_guard.py --suite full - -benchmark-profile: ## Run the quick benchmark and write PROFILE. - $(UV_RUN) python scripts/benchmark_privacy_guard.py --profile "$(PROFILE)" - clean: ## Remove build, test, lint, and Python cache artifacts. rm -rf build dist .pytest_cache .ruff_cache find src tests examples scripts -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/projects/privacy-guard/pyproject.toml b/projects/privacy-guard/pyproject.toml index b7449bc..48f03fe 100644 --- a/projects/privacy-guard/pyproject.toml +++ b/projects/privacy-guard/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "grpcio>=1.81.1,<2", "protobuf>=6.33.5,<7", "pydantic>=2.11,<3", - "pyyaml>=6.0.2,<7", + "regex>=2026.7.19,<2027", "typer>=0.16,<1", "typing-extensions>=4.12,<5", ] @@ -52,7 +52,7 @@ exclude = ["src/privacy_guard/bindings"] [tool.ty.rules] blanket-ignore-comment = "error" -missing-override-decorator = "error" +missing-override-decorator = "ignore" missing-type-argument = "error" redundant-cast = "error" unresolved-attribute = "error" diff --git a/projects/privacy-guard/scripts/benchmark_privacy_guard.py b/projects/privacy-guard/scripts/benchmark_privacy_guard.py deleted file mode 100755 index d922553..0000000 --- a/projects/privacy-guard/scripts/benchmark_privacy_guard.py +++ /dev/null @@ -1,319 +0,0 @@ -#!/usr/bin/env python3 -"""Benchmark complete Privacy Guard normalize/scan/validate/reconstruct paths.""" - -from __future__ import annotations - -import argparse -import cProfile -import gc -import platform -import statistics -import time -import tracemalloc -from dataclasses import dataclass - -from pydantic import Field -from typing_extensions import override - -from privacy_guard.config import ( - ActionConfig, - ObserveActionConfig, - PolicyConfig, - RedactActionConfig, -) -from privacy_guard.constants import ( - MAX_BODY_BYTES, - MAX_FINDINGS_PER_BLOCK, - MAX_FINDINGS_PER_REQUEST, -) -from privacy_guard.payloads import InterceptedRequest, ProcessingDecision -from privacy_guard.processor import RequestProcessor -from privacy_guard.scanners import Finding, ScanBudget, Scanner, ScannerConfig - -_KIB = 1024 -_MIB = 1024 * 1024 -_MARKER_CHARACTER = "x" -_SCANNER_COUNT_MULTIPLE = 4 -_MAXIMUM_WIDTH_NUMERIC_ELEMENT_COUNT = 2_097_151 -_MAXIMUM_WIDTH_NUMERIC_BYTES = 4_194_303 - - -class _BenchmarkScannerConfig(ScannerConfig): - findings_per_block: int = Field(ge=0, le=MAX_FINDINGS_PER_BLOCK) - - -class _BenchmarkScanner(Scanner[_BenchmarkScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - if not text_block.startswith( - _MARKER_CHARACTER * self.config.findings_per_block - ): - return () - return tuple( - Finding( - entity="benchmark", - scanner_name=self.scanner_name, - start_offset=offset, - end_offset=offset + 1, - ) - for offset in range(self.config.findings_per_block) - ) - - -@dataclass(frozen=True) -class _Scenario: - name: str - target_bytes: int - finding_load: str - scanner_count: int - reconstruction: str - body_shape: str = "text-blocks" - - -@dataclass(frozen=True) -class _PreparedScenario: - definition: _Scenario - processor: RequestProcessor - request: InterceptedRequest - expected_findings: int - - -@dataclass(frozen=True) -class _Measurement: - definition: _Scenario - actual_bytes: int - finding_count: int - median_wall_ms: float - median_peak_bytes: int - - -_QUICK_SCENARIOS = ( - _Scenario("1KiB-zero-one-noop", _KIB, "zero", 1, "noop"), - _Scenario( - "1KiB-typical-multiple-replacement", - _KIB, - "typical", - _SCANNER_COUNT_MULTIPLE, - "replacement", - ), - _Scenario("1MiB-typical-one-noop", _MIB, "typical", 1, "noop"), - _Scenario("1MiB-max-one-replacement", _MIB, "max", 1, "replacement"), - _Scenario( - "4MiB-zero-multiple-noop", - MAX_BODY_BYTES, - "zero", - _SCANNER_COUNT_MULTIPLE, - "noop", - ), - _Scenario( - "4MiB-max-multiple-replacement", - MAX_BODY_BYTES, - "max", - _SCANNER_COUNT_MULTIPLE, - "replacement", - ), -) - -_FULL_SCENARIOS = _QUICK_SCENARIOS + ( - _Scenario("1KiB-typical-one-noop", _KIB, "typical", 1, "noop"), - _Scenario("1MiB-zero-one-noop", _MIB, "zero", 1, "noop"), - _Scenario( - "1MiB-typical-multiple-replacement", - _MIB, - "typical", - _SCANNER_COUNT_MULTIPLE, - "replacement", - ), - _Scenario( - "1MiB-max-multiple-noop", - _MIB, - "max", - _SCANNER_COUNT_MULTIPLE, - "noop", - ), - _Scenario( - "4MiB-typical-one-replacement", MAX_BODY_BYTES, "typical", 1, "replacement" - ), - _Scenario( - "4MiB-typical-multiple-noop", - MAX_BODY_BYTES, - "typical", - _SCANNER_COUNT_MULTIPLE, - "noop", - ), - _Scenario("4MiB-max-one-noop", MAX_BODY_BYTES, "max", 1, "noop"), - _Scenario( - "4194303B-2097151-elements-wide-numeric-zero-one-noop", - _MAXIMUM_WIDTH_NUMERIC_BYTES, - "zero", - 1, - "noop", - "wide-numeric", - ), -) - - -def _build_json_body(target_bytes: int, block_count: int) -> bytes: - minimum_text_length = MAX_FINDINGS_PER_BLOCK - values = [_MARKER_CHARACTER * minimum_text_length for _ in range(block_count)] - encoded = ('{"blocks":["' + '","'.join(values) + '"]}').encode() - if len(encoded) > target_bytes: - raise ValueError("target body is too small for the requested finding load") - values[0] += _MARKER_CHARACTER * (target_bytes - len(encoded)) - body = ('{"blocks":["' + '","'.join(values) + '"]}').encode() - if len(body) != target_bytes: - raise AssertionError("benchmark body construction is not exact") - return body - - -def _build_maximum_width_numeric_body() -> bytes: - body = b"[" + (b"0," * (_MAXIMUM_WIDTH_NUMERIC_ELEMENT_COUNT - 1)) + b"0]" - if len(body) != _MAXIMUM_WIDTH_NUMERIC_BYTES: - raise AssertionError("maximum-width body construction is not exact") - return body - - -def _prepare_scenario(definition: _Scenario) -> _PreparedScenario: - if definition.finding_load == "zero": - total_findings_per_block = 0 - block_count = 1 - elif definition.finding_load == "typical": - total_findings_per_block = 8 - block_count = 1 - elif definition.finding_load == "max": - total_findings_per_block = MAX_FINDINGS_PER_BLOCK - block_count = MAX_FINDINGS_PER_REQUEST // MAX_FINDINGS_PER_BLOCK - else: - raise ValueError("unknown finding load") - - findings_per_scanner = total_findings_per_block // definition.scanner_count - if findings_per_scanner * definition.scanner_count != total_findings_per_block: - raise AssertionError("finding load must divide evenly across scanners") - scanners = [ - _BenchmarkScanner( - _BenchmarkScannerConfig( - name=f"benchmark-{index}", - entity_types=frozenset({"benchmark"}), - findings_per_block=findings_per_scanner, - ) - ) - for index in range(definition.scanner_count) - ] - action: ActionConfig = ( - RedactActionConfig(template="") - if definition.reconstruction == "replacement" - else ObserveActionConfig() - ) - policy = PolicyConfig(on_finding=action) - expected_findings = block_count * total_findings_per_block - if definition.body_shape == "text-blocks": - body = _build_json_body(definition.target_bytes, block_count) - elif definition.body_shape == "wide-numeric": - if block_count != 1 or expected_findings != 0: - raise AssertionError("wide numeric scenario must have zero findings") - body = _build_maximum_width_numeric_body() - else: - raise ValueError("unknown body shape") - request = InterceptedRequest(raw_body=body, policy_config=policy) - return _PreparedScenario( - definition=definition, - processor=RequestProcessor(scanners), - request=request, - expected_findings=expected_findings, - ) - - -def _run_and_verify(prepared: _PreparedScenario) -> None: - result = prepared.processor.process(prepared.request) - if result.decision is not ProcessingDecision.ALLOW: - raise AssertionError(f"{prepared.definition.name} unexpectedly denied") - if len(result.findings) != prepared.expected_findings: - raise AssertionError(f"{prepared.definition.name} finding count changed") - has_replacement = result.replacement_body is not None - if has_replacement != (prepared.definition.reconstruction == "replacement"): - raise AssertionError(f"{prepared.definition.name} reconstruction changed") - - -def _measure(prepared: _PreparedScenario, samples: int) -> _Measurement: - _run_and_verify(prepared) - wall_samples: list[float] = [] - for _ in range(samples): - started = time.perf_counter_ns() - _run_and_verify(prepared) - wall_samples.append((time.perf_counter_ns() - started) / 1_000_000) - - peak_samples: list[int] = [] - for _ in range(samples): - gc.collect() - tracemalloc.start() - try: - _run_and_verify(prepared) - _, peak = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - peak_samples.append(peak) - - return _Measurement( - definition=prepared.definition, - actual_bytes=len(prepared.request.raw_body), - finding_count=prepared.expected_findings, - median_wall_ms=statistics.median(wall_samples), - median_peak_bytes=int(statistics.median(peak_samples)), - ) - - -def _print_results(measurements: list[_Measurement], samples: int) -> None: - print(f"Python: {platform.python_version()} ({platform.platform()})") - print(f"Samples per median: {samples}") - print( - "| Scenario | Input bytes | Findings | Scanners | Reconstruction | " - "Median wall (ms) | Median peak traced allocation (bytes) |" - ) - print("| --- | ---: | ---: | ---: | --- | ---: | ---: |") - for measurement in measurements: - definition = measurement.definition - print( - f"| {definition.name} | {measurement.actual_bytes} | " - f"{measurement.finding_count} | {definition.scanner_count} | " - f"{definition.reconstruction} | {measurement.median_wall_ms:.3f} | " - f"{measurement.median_peak_bytes} |" - ) - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--suite", choices=("quick", "full"), default="quick") - parser.add_argument("--samples", type=int) - parser.add_argument( - "--profile", - metavar="PATH", - help="write cProfile data for one verified pass through every scenario", - ) - return parser.parse_args() - - -def main() -> int: - arguments = _parse_args() - scenarios = _QUICK_SCENARIOS if arguments.suite == "quick" else _FULL_SCENARIOS - samples = ( - arguments.samples - if arguments.samples is not None - else (3 if arguments.suite == "quick" else 7) - ) - if samples < 3 or samples % 2 == 0: - raise SystemExit("--samples must be an odd integer of at least 3") - prepared = [_prepare_scenario(scenario) for scenario in scenarios] - if arguments.profile is not None: - profiler = cProfile.Profile() - profiler.enable() - for item in prepared: - _run_and_verify(item) - profiler.disable() - profiler.dump_stats(arguments.profile) - measurements = [_measure(item, samples) for item in prepared] - _print_results(measurements, samples) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/projects/privacy-guard/src/privacy_guard/base.py b/projects/privacy-guard/src/privacy_guard/base.py new file mode 100644 index 0000000..6977d08 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/base.py @@ -0,0 +1,18 @@ +"""Package-wide base class for validated, immutable domain objects.""" + +from pydantic import BaseModel, ConfigDict + + +class StrictDomainModel(BaseModel): + """Base for immutable domain values parsed without implicit coercion.""" + + model_config = ConfigDict( + strict=True, + frozen=True, + extra="forbid", + hide_input_in_errors=True, + validate_default=True, + ) + + +__all__ = ["StrictDomainModel"] diff --git a/projects/privacy-guard/src/privacy_guard/config.py b/projects/privacy-guard/src/privacy_guard/config.py index b032b02..76a5a19 100644 --- a/projects/privacy-guard/src/privacy_guard/config.py +++ b/projects/privacy-guard/src/privacy_guard/config.py @@ -1,135 +1,200 @@ -"""Strict middleware policy configuration at the untrusted config boundary.""" +"""Strict entity-processing policy configuration. + +The concrete model accepted at the policy boundary is finalized by +``EngineRegistry``. Its stage ``config`` field is a Pydantic discriminated +union containing the exact config model registered by every engine. +""" from __future__ import annotations -from collections.abc import Mapping +import json +from collections.abc import Mapping, Sequence from enum import StrEnum -from string import Formatter -from typing import Literal, Self, TypeAlias - -from pydantic import Field, field_validator +from functools import reduce +from hashlib import sha256 +from operator import or_ +from typing import Annotated, Generic, Self, TypeAlias, TypeVar + +from pydantic import ( + Field, + TypeAdapter, + ValidationError, + field_validator, + model_validator, +) +from privacy_guard.base import StrictDomainModel +from privacy_guard.engines import EngineConfig from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.scanners import Confidence -from privacy_guard.validation import ( +from privacy_guard.string_validators import ( BoundedMetadataString, - NonEmptyScalarString, - ScalarString, - StrictSensitiveModel, - parse_scalar_string, + validate_scalar_string, ) class PolicyAction(StrEnum): - """Supported actions for scanner findings.""" + """User-facing disposition applied after all configured stages run.""" - OBSERVE = "observe" - REDACT = "redact" + DETECT = "detect" BLOCK = "block" + REPLACE = "replace" -class ActionConfig(StrictSensitiveModel): - """Finding criteria shared by every policy action.""" +class OnDetection(StrictDomainModel): + """Required policy disposition for detected entities.""" action: PolicyAction - entity_types: frozenset[BoundedMetadataString] | None = Field( - default=None, repr=False - ) - minimum_confidence: Confidence | None = None - - -class ObserveActionConfig(ActionConfig): - """Observe selected findings without changing or blocking the request.""" - - action: Literal[PolicyAction.OBSERVE] = PolicyAction.OBSERVE + @field_validator("action", mode="before") + @classmethod + def _parse_action(cls, value: object) -> PolicyAction: + if isinstance(value, PolicyAction): + return value + return PolicyAction(validate_scalar_string(value)) -class BlockActionConfig(ActionConfig): - """Block the request when at least one selected finding is present.""" - action: Literal[PolicyAction.BLOCK] = PolicyAction.BLOCK +_EngineConfigT = TypeVar( + "_EngineConfigT", + bound=EngineConfig[StrictDomainModel], +) -class RedactActionConfig(ActionConfig): - """Replace selected finding spans using a bounded text template.""" +class EntityProcessingStage( + StrictDomainModel, + Generic[_EngineConfigT], +): + """One ordered invocation of an engine with an optional diagnostic name.""" - action: Literal[PolicyAction.REDACT] = PolicyAction.REDACT - template: ScalarString = Field(default="[{entity}]", repr=False) + name: BoundedMetadataString | None = None + config: _EngineConfigT = Field(repr=False) - @field_validator("template") - @classmethod - def _validate_template(cls, value: str) -> str: - """Allow static text and the finding entity, but no formatting features.""" - try: - parsed_fields = Formatter().parse(value) - for _, field_name, format_spec, conversion in parsed_fields: - if field_name is not None and field_name != "entity": - raise ValueError("template field is unsupported") - if format_spec or conversion is not None: - raise ValueError("template formatting options are unsupported") - except ValueError: - raise ValueError("redaction template syntax is invalid") from None - return value - - -PolicyActionConfig: TypeAlias = ( - ObserveActionConfig | BlockActionConfig | RedactActionConfig -) + def diagnostic_name(self, stage_number: int) -> str: + """Return the explicit name or a deterministic one-based source label.""" + if self.name is not None: + return self.name + if isinstance(stage_number, bool) or stage_number < 1: + raise ValueError("stage number must be a positive integer") + engine = getattr(self.config, "engine", None) + if not isinstance(engine, str): + raise ValueError("stage config has no engine discriminator") + return f"{engine}[{stage_number}]" -class PolicyConfig(StrictSensitiveModel): - """Strict, immutable policy parsed from the supervisor's request config.""" +class EntityProcessing( + StrictDomainModel, + Generic[_EngineConfigT], +): + """The ordered entity-processing stages for one policy.""" - body_format: NonEmptyScalarString = Field(default="json", repr=False) - on_finding: PolicyActionConfig = Field( - default_factory=RedactActionConfig, - discriminator="action", - repr=False, - ) + stages: tuple[EntityProcessingStage[_EngineConfigT], ...] = Field(repr=False) + @field_validator("stages", mode="before") @classmethod - def from_mapping(cls, values: object) -> Self: - """Parse untrusted values while discarding Pydantic error content.""" - try: - if not isinstance(values, Mapping): - raise TypeError("configuration input must be a mapping") - prepared = dict(values) - if "on_finding" in prepared: - prepared["on_finding"] = _prepare_action_config(prepared["on_finding"]) - return cls.model_validate(prepared) - except (TypeError, ValueError): - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None - - -def _parse_policy_action(value: object) -> PolicyAction: - return PolicyAction(parse_scalar_string(value)) - - -def _parse_confidence(value: object) -> Confidence: - return Confidence(parse_scalar_string(value)) - - -def _prepare_action_config(value: object) -> dict[str, object]: - if not isinstance(value, Mapping): - raise ValueError("action configuration must be a mapping") - prepared = dict(value) - if "action" in prepared: - prepared["action"] = _parse_policy_action(prepared["action"]) - if prepared.get("minimum_confidence") is not None: - prepared["minimum_confidence"] = _parse_confidence( - prepared["minimum_confidence"] - ) - if "entity_types" in prepared: - prepared["entity_types"] = _parse_entity_type_list(prepared["entity_types"]) - return prepared - - -def _parse_entity_type_list(value: object) -> frozenset[str] | None: - if value is None: - return None - if not isinstance(value, list): - raise ValueError("entity types must be a list of strings or null") - parsed = tuple(parse_scalar_string(item) for item in value) - if len(set(parsed)) != len(parsed): - raise ValueError("entity types must be unique") - return frozenset(parsed) + def _parse_stages(cls, value: object) -> object: + if not isinstance(value, list | tuple) or not value: + raise ValueError("stages must be a non-empty list") + return tuple(value) + + @model_validator(mode="after") + def _diagnostic_names_are_unique(self) -> Self: + names = [ + stage.diagnostic_name(index) + for index, stage in enumerate(self.stages, start=1) + ] + if len(names) != len(set(names)): + raise ValueError("stage diagnostic names must be unique") + return self + + +class PrivacyGuardConfig( + StrictDomainModel, + Generic[_EngineConfigT], +): + """Complete validated Privacy Guard behavior for one OpenShell policy.""" + + entity_processing: EntityProcessing[_EngineConfigT] = Field(repr=False) + on_detection: OnDetection = Field(repr=False) + + +FinalizedPrivacyGuardConfig: TypeAlias = PrivacyGuardConfig[ + EngineConfig[StrictDomainModel] +] +FinalizedPrivacyGuardConfigType = type[FinalizedPrivacyGuardConfig] + + +def build_privacy_guard_config_type( + config_types: Sequence[type[EngineConfig[StrictDomainModel]]], +) -> FinalizedPrivacyGuardConfigType: + """Build the exact registry-dependent discriminated policy model.""" + if not config_types: + raise ValueError("at least one engine config type must be registered") + registered_union = reduce(or_, config_types) + registered_config = Annotated[ + registered_union, # ty: ignore[invalid-type-form] + Field(discriminator="engine"), + ] + config_type = PrivacyGuardConfig.__class_getitem__( + registered_config # ty: ignore[invalid-argument-type] + ) + if not isinstance(config_type, type) or not issubclass( + config_type, PrivacyGuardConfig + ): + raise TypeError("Pydantic did not construct a policy config type") + return config_type # ty: ignore[invalid-return-type] + + +def build_privacy_guard_config_adapter( + config_types: Sequence[type[EngineConfig[StrictDomainModel]]], +) -> TypeAdapter[FinalizedPrivacyGuardConfig]: + """Build the registry-dependent adapter used for validation and schemas.""" + config_type = build_privacy_guard_config_type(config_types) + return TypeAdapter(config_type) + + +def parse_privacy_guard_config( + adapter: TypeAdapter[FinalizedPrivacyGuardConfig], + values: object, +) -> FinalizedPrivacyGuardConfig: + """Parse an expanded mapping without exposing rejected values in errors.""" + if not isinstance(values, Mapping): + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) + try: + return adapter.validate_python(dict(values)) + except (TypeError, ValueError, ValidationError): + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None + + +def canonical_config_json( + config: PrivacyGuardConfig[EngineConfig[StrictDomainModel]], +) -> bytes: + """Serialize every concrete engine field deterministically for hashing.""" + return json.dumps( + config.model_dump(mode="json"), + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + +def configuration_fingerprint( + config: PrivacyGuardConfig[EngineConfig[StrictDomainModel]], +) -> str: + """Return the canonical SHA-256 fingerprint of an expanded policy config.""" + return sha256(canonical_config_json(config)).hexdigest() + + +__all__ = [ + "EntityProcessing", + "EntityProcessingStage", + "FinalizedPrivacyGuardConfig", + "FinalizedPrivacyGuardConfigType", + "OnDetection", + "PolicyAction", + "PrivacyGuardConfig", + "build_privacy_guard_config_adapter", + "build_privacy_guard_config_type", + "canonical_config_json", + "configuration_fingerprint", + "parse_privacy_guard_config", +] diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index 8dee5ca..d89f3df 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -16,41 +16,29 @@ BLOCK_REASON_CODE = "privacy_guard_blocked" LIMIT_REASON = "Privacy Guard denied a result that exceeded a safety limit" LIMIT_REASON_CODE = "privacy_guard_limit_exceeded" -PATTERN_NAME_METADATA_KEY = "pattern_name" - -# Request-body and JSON shape limits. +# Text input limits. MAX_BODY_BYTES = 4 * 1024 * 1024 -MAX_JSON_NESTING = 64 -MAX_TEXT_BLOCKS = 4096 MAX_SCANNED_CHARACTERS = 4 * 1024 * 1024 -# Scanner and result limits. -MAX_FINDINGS_PER_BLOCK = 256 -MAX_FINDINGS_PER_REQUEST = 4096 -MAX_SCANNER_METADATA_BYTES = 1024 +# Engine and result limits. +MAX_DETECTIONS_PER_STAGE = 256 +MAX_DETECTIONS_PER_REQUEST = 4096 +MAX_DIAGNOSTIC_TEXT_BYTES = 1024 MAX_FINDING_METADATA_ENTRIES = 32 MAX_PROTO_FINDING_GROUPS = 32 MAX_PROTO_FINDING_BYTES = 4 * 1024 -# Regex scanner configuration and execution limits. Catalog limits are independent -# of request result limits: sparse requests can use large, focused rule sets. -MAX_SCANNER_CONFIG_BYTES = 16 * 1024 * 1024 -MAX_SCANNER_CONFIG_NESTING = 16 -MAX_SCANNER_CONFIG_NODES = 250_000 -MAX_SCANNER_CONFIG_SCALAR_BYTES = 16 * 1024 +# Engine configuration and regex execution limits. MAX_REGEX_NAME_BYTES = 128 -MAX_REGEX_PROFILES = 32 -MAX_REGEX_ENTITIES_PER_PROFILE = 2_000 -MAX_REGEX_PATTERNS_PER_PROFILE = 10_000 -MAX_REGEX_ENTITIES_TOTAL = 10_000 -MAX_REGEX_PATTERNS_TOTAL = 50_000 +MAX_REGEX_ENTITIES_PER_CATALOG = 2_000 +MAX_REGEX_PATTERNS_PER_CATALOG = 10_000 MAX_REGEX_PATTERN_BYTES = 16 * 1024 MAX_MATCHES_PER_PATTERN = 256 -DEFAULT_SCAN_TIMEOUT_SECONDS = 1.0 -MAX_SCAN_TIMEOUT_SECONDS = 30.0 +DEFAULT_TIMEOUT_SECONDS = 1.0 +MAX_TIMEOUT_SECONDS = 30.0 # Service concurrency and transport limits. -MAX_CONCURRENT_SCANS = 4 +MAX_CONCURRENT_PROCESSING = 4 MAX_CONCURRENT_RPCS = 16 PROTOBUF_ENVELOPE_ALLOWANCE_BYTES = 1024 * 1024 MAX_RECEIVE_MESSAGE_BYTES = MAX_BODY_BYTES + PROTOBUF_ENVELOPE_ALLOWANCE_BYTES diff --git a/projects/privacy-guard/src/privacy_guard/engine_registry.py b/projects/privacy-guard/src/privacy_guard/engine_registry.py new file mode 100644 index 0000000..4dd39f0 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/engine_registry.py @@ -0,0 +1,259 @@ +"""Engine registration and finalized policy-schema construction.""" + +from __future__ import annotations + +import inspect +import re +from dataclasses import dataclass +from types import NoneType +from typing import Literal, get_args, get_origin + +from pydantic import TypeAdapter +from pydantic_core import PydanticUndefined + +from privacy_guard.base import StrictDomainModel +from privacy_guard.config import ( + FinalizedPrivacyGuardConfig, + FinalizedPrivacyGuardConfigType, + PolicyAction, + build_privacy_guard_config_type, + parse_privacy_guard_config, +) +from privacy_guard.engines import ( + EngineConfig, + EngineConfigurationError, + EntityProcessingEngine, + EntityProcessingStrategy, +) +from privacy_guard.errors import ErrorCode, PrivacyGuardError + + +class EngineRegistryError(Exception): + """A content-safe engine registration or registry lifecycle failure.""" + + +@dataclass(frozen=True) +class EngineDescription: + """Safe discovery metadata for one registered engine.""" + + engine: str + description: str + supported_strategy: EntityProcessingStrategy + configuration_schema: dict[str, object] + + +class EngineRegistry: + """Register engine implementations and finalize their exact policy union.""" + + def __init__(self) -> None: + self._registrations: dict[str, _Registration] = {} + self._config_type: FinalizedPrivacyGuardConfigType | None = None + self._config_adapter: TypeAdapter[FinalizedPrivacyGuardConfig] | None = None + + @property + def is_finalized(self) -> bool: + return self._config_adapter is not None + + @property + def engine_names(self) -> tuple[str, ...]: + return tuple(self._registrations) + + @property + def config_type(self) -> FinalizedPrivacyGuardConfigType: + if self._config_type is None: + raise EngineRegistryError("engine registry is not finalized") + return self._config_type + + @property + def config_adapter(self) -> TypeAdapter[FinalizedPrivacyGuardConfig]: + if self._config_adapter is None: + raise EngineRegistryError("engine registry is not finalized") + return self._config_adapter + + def register( + self, + engine_type: type[object], + *, + resources: object = None, + ) -> None: + """Register one engine implementation and its operator-owned resources.""" + if self.is_finalized: + raise EngineRegistryError("cannot register after finalization") + if not isinstance(engine_type, type) or not issubclass( + engine_type, EntityProcessingEngine + ): + raise EngineRegistryError("registered engine type is invalid") + + try: + config_type = engine_type.get_config_type() + resources_type = engine_type.get_resources_type() + except (AttributeError, TypeError): + raise EngineRegistryError("engine generic declaration is invalid") from None + if not isinstance(config_type, type) or not issubclass( + config_type, EngineConfig + ): + raise EngineRegistryError("engine config type is invalid") + resources_runtime_type = get_origin(resources_type) or resources_type + if not isinstance(resources_runtime_type, type): + raise EngineRegistryError("engine resources type is invalid") + + engine_name = _engine_discriminator(config_type) + if engine_name in self._registrations: + raise EngineRegistryError("engine discriminator is already registered") + if any( + registration.config_type is config_type + for registration in self._registrations.values() + ): + raise EngineRegistryError("engine config type is already registered") + + supported_strategy = getattr(engine_type, "supported_strategy", None) + if not isinstance(supported_strategy, EntityProcessingStrategy): + raise EngineRegistryError("engine supported strategy is invalid") + if resources_runtime_type is NoneType: + if resources is not None: + raise EngineRegistryError("resource-free engine received resources") + elif resources is None or not isinstance(resources, resources_runtime_type): + raise EngineRegistryError( + "engine resources do not match their declared type" + ) + + self._registrations[engine_name] = _Registration( + engine_type=engine_type, + config_type=config_type, + resources_type=resources_type, + resources=resources, + supported_strategy=supported_strategy, + ) + + def finalize(self) -> FinalizedPrivacyGuardConfigType: + """Freeze registrations and build the Pydantic discriminated union.""" + if self.is_finalized: + return self.config_type + try: + config_type = build_privacy_guard_config_type( + tuple( + registration.config_type + for registration in self._registrations.values() + ) + ) + except ValueError: + raise EngineRegistryError( + "cannot finalize an empty engine registry" + ) from None + self._config_type = config_type + self._config_adapter = TypeAdapter(config_type) + return config_type + + def validate_config(self, values: object) -> FinalizedPrivacyGuardConfig: + """Purely parse and validate an expanded Privacy Guard configuration.""" + config = parse_privacy_guard_config(self.config_adapter, values) + for stage in config.entity_processing.stages: + registration = self._resolve_registration(stage.config) + engine_type = registration.engine_type + if not issubclass(engine_type, EntityProcessingEngine): + raise EngineRegistryError("registered engine type is invalid") + try: + validate_config = getattr(engine_type, "validate_config") + validate_config( + stage.config, + registration.resources, + ) + except EngineConfigurationError: + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None + if config.on_detection.action is PolicyAction.REPLACE: + if ( + registration.supported_strategy + is not EntityProcessingStrategy.REPLACE + or stage.config.replacement is None + ): + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) + return config + + def create_engine( + self, + config: EngineConfig[StrictDomainModel], + ) -> EntityProcessingEngine[EngineConfig[StrictDomainModel], object]: + """Construct an initialized engine from its exact validated config.""" + registration = self._resolve_registration(config) + if type(config) is not registration.config_type: + raise EngineRegistryError("engine config concrete type is invalid") + return registration.engine_type(config, registration.resources) + + def configuration_json_schema(self) -> dict[str, object]: + """Return the finalized complete policy JSON Schema.""" + return self.config_adapter.json_schema() + + def describe_engines(self) -> tuple[EngineDescription, ...]: + """Return safe engine metadata without constructing runtime engines.""" + return tuple( + EngineDescription( + engine=engine, + description=_engine_description(registration.engine_type), + supported_strategy=registration.supported_strategy, + configuration_schema=registration.config_type.model_json_schema(), + ) + for engine, registration in self._registrations.items() + ) + + def _resolve_registration( + self, + config: EngineConfig[StrictDomainModel], + ) -> _Registration: + if not self.is_finalized: + raise EngineRegistryError("engine registry is not finalized") + try: + engine_name = getattr(config, "engine") + if not isinstance(engine_name, str): + raise AttributeError + registration = self._registrations[engine_name] + except (AttributeError, KeyError): + raise EngineRegistryError("engine config is not registered") from None + return registration + + +@dataclass(frozen=True) +class _Registration: + engine_type: type[object] + config_type: type[EngineConfig[StrictDomainModel]] + resources_type: object + resources: object + supported_strategy: EntityProcessingStrategy + + +def _engine_discriminator( + config_type: type[EngineConfig[StrictDomainModel]], +) -> str: + field = config_type.model_fields.get("engine") + if field is None: + raise EngineRegistryError("engine config lacks an engine discriminator") + if get_origin(field.annotation) is not Literal: + raise EngineRegistryError("engine discriminator must be one string Literal") + values = get_args(field.annotation) + if len(values) != 1 or not isinstance(values[0], str): + raise EngineRegistryError("engine discriminator must be one string Literal") + engine = values[0] + if _ENGINE_NAME.fullmatch(engine) is None or len(engine.encode("ascii")) > 128: + raise EngineRegistryError("engine discriminator is invalid") + if field.default is not PydanticUndefined and field.default != engine: + raise EngineRegistryError("engine discriminator default is inconsistent") + return engine + + +def _engine_description( + engine_type: type[object], +) -> str: + description = inspect.getdoc(engine_type) or "" + first_line = description.splitlines()[0] if description else "" + if len(first_line.encode("utf-8")) > 1024: + return "" + return first_line + + +_ENGINE_NAME = re.compile(r"[a-z][a-z0-9-]{0,127}\Z") + + +__all__ = [ + "EngineDescription", + "EngineRegistry", + "EngineRegistryError", +] diff --git a/projects/privacy-guard/src/privacy_guard/engines/__init__.py b/projects/privacy-guard/src/privacy_guard/engines/__init__.py new file mode 100644 index 0000000..ba8bc18 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/engines/__init__.py @@ -0,0 +1,53 @@ +"""Supported entity-processing extension and built-in regex engine surface.""" + +from __future__ import annotations + +from privacy_guard.engines.base import ( + BoundedMetadata, + ConfidenceLevel, + DetectionConfidence, + EngineConfig, + EngineConfigurationError, + EngineContractError, + EngineExecutionError, + EngineLimitExceeded, + EntityDetection, + EntityName, + EntityProcessingEngine, + EntityProcessingError, + EntityProcessingStrategy, + TextProcessingResult, + UnitInterval, +) +from privacy_guard.engines.regex import ( + RegexEngine, + RegexEngineConfig, + RegexEntity, + RegexPattern, + RegexPatternCatalog, + RegexReplacement, +) + +__all__ = [ + "BoundedMetadata", + "ConfidenceLevel", + "DetectionConfidence", + "EngineConfig", + "EngineConfigurationError", + "EngineContractError", + "EngineExecutionError", + "EngineLimitExceeded", + "EntityDetection", + "EntityName", + "EntityProcessingEngine", + "EntityProcessingError", + "EntityProcessingStrategy", + "RegexEngine", + "RegexEngineConfig", + "RegexEntity", + "RegexPattern", + "RegexPatternCatalog", + "RegexReplacement", + "TextProcessingResult", + "UnitInterval", +] diff --git a/projects/privacy-guard/src/privacy_guard/engines/base.py b/projects/privacy-guard/src/privacy_guard/engines/base.py new file mode 100644 index 0000000..f9e21ef --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/engines/base.py @@ -0,0 +1,339 @@ +"""Core entity-processing engine extension contract.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Mapping +from enum import StrEnum +from types import MappingProxyType +from typing import ( + Annotated, + ClassVar, + Generic, + TypeAlias, + TypeVar, + get_args, + get_origin, +) + +from pydantic import ( + BeforeValidator, + Field, + ValidationError, + field_validator, + model_validator, +) + +from privacy_guard.base import StrictDomainModel +from privacy_guard.constants import ( + MAX_BODY_BYTES, + MAX_DETECTIONS_PER_STAGE, + MAX_FINDING_METADATA_ENTRIES, +) +from privacy_guard.string_validators import ( + ScalarString, + validate_bounded_metadata_string, + validate_scalar_string, +) +from privacy_guard.timeout import Timeout + + +class EntityProcessingStrategy(StrEnum): + """Select whether one engine invocation detects or replaces entities.""" + + DETECT = "detect" + REPLACE = "replace" + + +class ConfidenceLevel(StrEnum): + """Categorical certainty reported by an entity-processing engine.""" + + LOW = "low" + MEDIUM = "medium" + HIGH = "high" + + +def _parse_unit_interval(value: object) -> float: + if isinstance(value, bool) or not isinstance(value, int | float): + raise ValueError("confidence must be a number from zero through one") + result = float(value) + if not 0.0 <= result <= 1.0: + raise ValueError("confidence must be a number from zero through one") + return result + + +UnitInterval = Annotated[float, BeforeValidator(_parse_unit_interval)] +DetectionConfidence: TypeAlias = ConfidenceLevel | UnitInterval +EntityName = Annotated[str, BeforeValidator(validate_bounded_metadata_string)] +MetadataString = Annotated[str, BeforeValidator(validate_bounded_metadata_string)] +BoundedMetadata: TypeAlias = Mapping[MetadataString, MetadataString] + + +class EntityDetection(StrictDomainModel): + """One sensitive entity occurrence in the engine's input text.""" + + entity: EntityName + start: int = Field(ge=0) + end: int + confidence: DetectionConfidence | None = None + metadata: BoundedMetadata = Field(default_factory=dict, repr=False) + + @field_validator("confidence", mode="before") + @classmethod + def _parse_confidence(cls, value: object) -> object: + if isinstance(value, str): + return ConfidenceLevel(validate_scalar_string(value)) + return value + + @field_validator("metadata") + @classmethod + def _copy_bounded_metadata(cls, value: Mapping[str, str]) -> Mapping[str, str]: + if len(value) > MAX_FINDING_METADATA_ENTRIES: + raise ValueError("detection metadata has too many entries") + copied: dict[str, str] = {} + for key, item in value.items(): + copied[validate_bounded_metadata_string(key)] = ( + validate_bounded_metadata_string(item) + ) + return MappingProxyType(copied) + + @model_validator(mode="after") + def _span_is_non_empty(self) -> EntityDetection: + if self.end <= self.start: + raise ValueError("detection span must be non-empty") + return self + + +class TextProcessingResult(StrictDomainModel): + """The authoritative text and detections returned by one engine run.""" + + text: ScalarString = Field(repr=False) + detections: tuple[EntityDetection, ...] + + @field_validator("detections", mode="before") + @classmethod + def _detections_are_a_tuple(cls, value: object) -> object: + if not isinstance(value, tuple): + raise ValueError("detections must be a tuple") + return value + + +_ReplacementConfigT = TypeVar( + "_ReplacementConfigT", + bound=StrictDomainModel, + covariant=True, +) + + +class EngineConfig(StrictDomainModel, Generic[_ReplacementConfigT]): + """Common typed location for an engine-specific replacement recipe.""" + + replacement: _ReplacementConfigT | None = None + + +class EntityProcessingError(Exception): + """Base for content-safe entity-processing failures.""" + + +class EngineConfigurationError(EntityProcessingError): + """An engine class or configured instance is invalid.""" + + +class EngineContractError(EntityProcessingError): + """An engine invocation or returned result violated the public contract.""" + + +class EngineExecutionError(EntityProcessingError): + """An engine's configured runtime failed to complete one text input.""" + + +class EngineLimitExceeded(EntityProcessingError): + """An engine exceeded a bounded configuration or output limit.""" + + +_ConfigT = TypeVar("_ConfigT", bound=EngineConfig[StrictDomainModel]) +_ResourcesT = TypeVar("_ResourcesT") + + +class EntityProcessingEngine(ABC, Generic[_ConfigT, _ResourcesT]): + """Nominal, typed extension point for processing one text string.""" + + supported_strategy: ClassVar[EntityProcessingStrategy] + + def __init__(self, config: _ConfigT, resources: _ResourcesT) -> None: + """Validate typed configuration/resources and initialize reusable state.""" + self.validate_config(config, resources) + self.__config = config + self.__resources = resources + self._initialize() + + @classmethod + def validate_config( + cls, + config: _ConfigT, + resources: _ResourcesT, + ) -> None: + """Purely validate one exact config and its registered resources.""" + cls._validate_class_contract() + config_type = cls.get_config_type() + try: + if not isinstance(config, config_type): + raise ValueError + config_type.model_validate(config) + except (ValidationError, ValueError): + raise EngineConfigurationError("engine configuration is invalid") from None + resources_type = cls.get_resources_type() + if not _is_valid_resources(resources, resources_type): + raise EngineConfigurationError("engine resources are invalid") + cls._validate_config(config, resources) + + @classmethod + def _validate_config( + cls, + config: _ConfigT, + resources: _ResourcesT, + ) -> None: + """Optionally validate resource-backed config without side effects.""" + + @classmethod + def get_config_type(cls) -> type[EngineConfig[StrictDomainModel]]: + """Return the concrete ``EngineConfig`` generic argument.""" + config_type, _ = _declared_engine_types(cls) + return config_type + + @classmethod + def get_resources_type(cls) -> object: + """Return the concrete runtime-resources generic argument.""" + _, resources_type = _declared_engine_types(cls) + return resources_type + + @classmethod + def _validate_class_contract(cls) -> None: + if not isinstance( + getattr(cls, "supported_strategy", None), EntityProcessingStrategy + ): + raise EngineConfigurationError("engine supported strategy is invalid") + cls.get_config_type() + cls.get_resources_type() + + @property + def config(self) -> _ConfigT: + """Return the immutable, concrete engine configuration.""" + return self.__config + + @property + def resources(self) -> _ResourcesT: + """Return the validated, injected runtime resources.""" + return self.__resources + + def _initialize(self) -> None: + """Optionally initialize reusable state from config and resources.""" + + def run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + """Process one text value and validate the complete collaborator result.""" + try: + validated_text = validate_scalar_string(text) + except ValueError: + raise EngineContractError("engine input text is invalid") from None + if not isinstance(strategy, EntityProcessingStrategy): + raise EngineContractError("engine processing strategy is invalid") + if not isinstance(timeout, Timeout): + raise EngineContractError("engine timeout is invalid") + if ( + strategy is EntityProcessingStrategy.REPLACE + and self.supported_strategy is EntityProcessingStrategy.DETECT + ): + raise EngineContractError("engine does not support replacement") + timeout.raise_if_expired() + result: object = self._run( + validated_text, + strategy=strategy, + timeout=timeout, + ) + timeout.raise_if_expired() + return _validate_result(validated_text, result, strategy=strategy) + + @abstractmethod + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + """Return processed text and every detected entity occurrence.""" + raise NotImplementedError + + +def _declared_engine_types( + engine_type: type[object], +) -> tuple[type[EngineConfig[StrictDomainModel]], object]: + for candidate in engine_type.__mro__: + for base in getattr(candidate, "__orig_bases__", ()): + if get_origin(base) is not EntityProcessingEngine: + continue + arguments = get_args(base) + if len(arguments) != 2: + break + config_type, resources_type = arguments + if isinstance(config_type, type) and issubclass(config_type, EngineConfig): + return config_type, resources_type + raise EngineConfigurationError( + "engine must declare concrete configuration and resource types" + ) + + +def _is_valid_resources(resources: object, resources_type: object) -> bool: + if resources_type in (None, type(None)): + return resources is None + origin = get_origin(resources_type) + if origin is not None: + resources_type = origin + return isinstance(resources_type, type) and isinstance(resources, resources_type) + + +def _validate_result( + input_text: str, + result: object, + *, + strategy: EntityProcessingStrategy, +) -> TextProcessingResult: + if not isinstance(result, TextProcessingResult): + raise EngineContractError("engine output is invalid") + if len(result.detections) > MAX_DETECTIONS_PER_STAGE: + raise EngineLimitExceeded("engine returned too many detections") + if len(result.text.encode("utf-8")) > MAX_BODY_BYTES: + raise EngineLimitExceeded("engine output text exceeds the size limit") + for detection in result.detections: + if detection.end > len(input_text): + raise EngineContractError("engine detection span is invalid") + if strategy is EntityProcessingStrategy.DETECT and result.text != input_text: + raise EngineContractError("detection-only engine output changed text") + if result.text != input_text and not result.detections: + raise EngineContractError("engine changed text without a detection") + return result + + +__all__ = [ + "BoundedMetadata", + "ConfidenceLevel", + "DetectionConfidence", + "EngineConfig", + "EngineConfigurationError", + "EngineContractError", + "EngineExecutionError", + "EngineLimitExceeded", + "EntityDetection", + "EntityName", + "EntityProcessingEngine", + "EntityProcessingError", + "EntityProcessingStrategy", + "TextProcessingResult", + "UnitInterval", +] diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py new file mode 100644 index 0000000..36c6127 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -0,0 +1,460 @@ +"""Bounded regular-expression entity detection and replacement.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from string import Formatter +from typing import Literal, Protocol, Self + +import regex +from pydantic import Field, field_validator, model_validator + +from privacy_guard.base import StrictDomainModel +from privacy_guard.constants import ( + CONFIDENCE_RANK, + MAX_BODY_BYTES, + MAX_DETECTIONS_PER_STAGE, + MAX_DIAGNOSTIC_TEXT_BYTES, + MAX_MATCHES_PER_PATTERN, + MAX_REGEX_ENTITIES_PER_CATALOG, + MAX_REGEX_NAME_BYTES, + MAX_REGEX_PATTERN_BYTES, + MAX_REGEX_PATTERNS_PER_CATALOG, +) +from privacy_guard.engines.base import ( + ConfidenceLevel, + EngineConfig, + EngineConfigurationError, + EngineContractError, + EngineLimitExceeded, + EntityDetection, + EntityProcessingEngine, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.string_validators import ScalarString, validate_scalar_string +from privacy_guard.timeout import Timeout, TimeoutExpired + + +class RegexPattern(StrictDomainModel): + """One optional diagnostic identity, pattern string, and explicit flags.""" + + name: str | None = None + pattern: ScalarString = Field(repr=False) + confidence: ConfidenceLevel + ignore_case: bool = False + multiline: bool = False + dot_all: bool = False + ascii: bool = False + + @field_validator("name") + @classmethod + def _name_is_safe(cls, value: str | None) -> str | None: + return None if value is None else _validate_name(value) + + @field_validator("pattern") + @classmethod + def _pattern_is_bounded(cls, value: str) -> str: + if not value: + raise ValueError("pattern must be non-empty") + if len(value.encode("utf-8")) > MAX_REGEX_PATTERN_BYTES: + raise ValueError("pattern exceeds the size limit") + return value + + @field_validator("confidence", mode="before") + @classmethod + def _parse_confidence(cls, value: object) -> ConfidenceLevel: + if not isinstance(value, str): + raise ValueError("confidence must be a string") + return ConfidenceLevel(validate_scalar_string(value)) + + +class RegexEntity(StrictDomainModel): + """One entity name and its ordered, non-empty regex patterns.""" + + name: str + patterns: tuple[RegexPattern, ...] = Field(repr=False) + + @field_validator("name") + @classmethod + def _name_is_safe(cls, value: str) -> str: + return _validate_name(value) + + @field_validator("patterns", mode="before") + @classmethod + def _patterns_are_non_empty(cls, value: object) -> object: + if not isinstance(value, list | tuple) or not value: + raise ValueError("patterns must be a non-empty list") + return tuple(value) + + @model_validator(mode="after") + def _supplied_pattern_names_are_unique(self) -> Self: + supplied_names = [ + pattern.name for pattern in self.patterns if pattern.name is not None + ] + if len(supplied_names) != len(set(supplied_names)): + raise ValueError("supplied pattern names must be unique within an entity") + return self + + +class RegexPatternCatalog(StrictDomainModel): + """The complete ordered entity catalog for one RegexEngine stage.""" + + entities: tuple[RegexEntity, ...] = Field(repr=False) + + @field_validator("entities", mode="before") + @classmethod + def _entities_are_non_empty(cls, value: object) -> object: + if not isinstance(value, list | tuple) or not value: + raise ValueError("entities must be a non-empty list") + return tuple(value) + + @model_validator(mode="after") + def _catalog_is_bounded_and_unambiguous(self) -> Self: + names = [entity.name for entity in self.entities] + if len(names) != len(set(names)): + raise ValueError("entity names must be unique") + if len(self.entities) > MAX_REGEX_ENTITIES_PER_CATALOG: + raise ValueError("entity catalog exceeds the size limit") + if ( + sum(len(entity.patterns) for entity in self.entities) + > MAX_REGEX_PATTERNS_PER_CATALOG + ): + raise ValueError("pattern catalog exceeds the size limit") + return self + + +class RegexReplacement(StrictDomainModel): + """A constrained template replacement recipe.""" + + strategy: Literal["template"] = "template" + template: ScalarString = Field(default="[{entity}]", repr=False) + + @field_validator("template") + @classmethod + def _template_is_safe_and_bounded(cls, value: str) -> str: + if len(value.encode("utf-8")) > MAX_DIAGNOSTIC_TEXT_BYTES: + raise ValueError("replacement template exceeds the size limit") + try: + for _, field_name, format_spec, conversion in Formatter().parse(value): + if field_name is not None and field_name != "entity": + raise ValueError + if format_spec or conversion is not None: + raise ValueError + except ValueError: + raise ValueError("replacement template syntax is invalid") from None + return value + + +class RegexEngineConfig(EngineConfig[RegexReplacement]): + """Exact policy configuration owned by ``RegexEngine``.""" + + engine: Literal["regex"] = "regex" + pattern_catalog: RegexPatternCatalog = Field(repr=False) + + @model_validator(mode="after") + def _patterns_are_valid(self) -> Self: + try: + for global_index, (entity, pattern_index, pattern) in enumerate( + _iter_catalog_patterns(self.pattern_catalog) + ): + _compile_rule( + entity, + pattern, + catalog_index=global_index, + entity_pattern_index=pattern_index, + ) + except (RecursionError, ValueError, regex.error): + raise ValueError("regex pattern catalog is invalid") from None + return self + + +class RegexEngine(EntityProcessingEngine[RegexEngineConfig, None]): + """Detect overlapping regex matches and optionally replace deterministic winners.""" + + supported_strategy = EntityProcessingStrategy.REPLACE + + def _initialize(self) -> None: + try: + self._rules = tuple( + _compile_rule( + entity, + pattern, + catalog_index=global_index, + entity_pattern_index=pattern_index, + ) + for global_index, (entity, pattern_index, pattern) in enumerate( + _iter_catalog_patterns(self.config.pattern_catalog) + ) + ) + except (RecursionError, ValueError, regex.error): + raise EngineConfigurationError( + "regex engine configuration is invalid" + ) from None + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + detections_with_identity: list[tuple[EntityDetection, str]] = [] + try: + for rule in self._rules: + match_count = 0 + next_position = 0 + while next_position <= len(text): + match = rule.compiled.search( + text, + next_position, + timeout=timeout.remaining_seconds(), + ) + timeout.raise_if_expired() + if match is None: + break + start, end = match.span() + if start == end or match.span(rule.marker) != (end, end): + raise EngineConfigurationError( + "regex engine configuration is invalid" + ) + match_count += 1 + if match_count > MAX_MATCHES_PER_PATTERN: + raise EngineLimitExceeded("regex match count exceeds the limit") + detection = EntityDetection( + entity=rule.entity, + start=start, + end=end, + confidence=rule.confidence, + metadata={_PATTERN_METADATA_KEY: rule.pattern_identity}, + ) + detections_with_identity.append((detection, rule.pattern_identity)) + if len(detections_with_identity) > MAX_DETECTIONS_PER_STAGE: + raise EngineLimitExceeded( + "regex detection count exceeds the limit" + ) + next_position = start + 1 + except TimeoutError: + raise TimeoutExpired from None + + detections_with_identity.sort( + key=lambda item: ( + item[0].start, + item[0].end, + item[0].entity, + item[1], + ) + ) + detections = tuple(item[0] for item in detections_with_identity) + output_text = text + if strategy is EntityProcessingStrategy.REPLACE and detections: + replacement = self.config.replacement + if replacement is None: + raise EngineConfigurationError( + "regex replacement configuration is required" + ) + winners = _resolve_overlaps(detections_with_identity) + output_text = _render_bounded_replacement( + text, + winners, + replacement.template, + ) + return TextProcessingResult(text=output_text, detections=detections) + + +@dataclass(frozen=True) +class _CompiledRule: + entity: str + pattern_identity: str + confidence: ConfidenceLevel + marker: str + compiled: _CompiledPattern + + +class _RegexMatch(Protocol): + def span(self, group: int | str = 0) -> tuple[int, int]: + """Return the matched span for a numbered or named group.""" + ... + + +class _CompiledPattern(Protocol): + @property + def groupindex(self) -> Mapping[str, int]: + """Return the pattern's named capture groups.""" + ... + + def search( + self, + string: str, + pos: int = 0, + *, + timeout: float | None = None, + ) -> _RegexMatch | None: + """Search from a code-point offset with a bounded timeout.""" + ... + + +def _validate_name(value: str) -> str: + if ( + not isinstance(value, str) + or _NAME_PATTERN.fullmatch(value) is None + or len(value.encode("ascii")) > MAX_REGEX_NAME_BYTES + ): + raise ValueError("name is invalid") + return value + + +def _compile_rule( + entity: RegexEntity, + pattern: RegexPattern, + catalog_index: int, + entity_pattern_index: int, +) -> _CompiledRule: + flags = 0 + if pattern.ignore_case: + flags |= regex.IGNORECASE + if pattern.multiline: + flags |= regex.MULTILINE + if pattern.dot_all: + flags |= regex.DOTALL + if pattern.ascii: + flags |= regex.ASCII + if _contains_inline_flags(pattern.pattern): + raise ValueError("inline flags are unsupported") + unmarked = regex.compile(pattern.pattern, flags) + if unmarked.groupindex: + raise ValueError("named groups are reserved") + if unmarked.search("") is not None: + raise ValueError("pattern must not match empty input") + marker = f"_pg_pattern_{catalog_index:06d}" + compiled = regex.compile(f"(?:{pattern.pattern})(?P<{marker}>)", flags) + if marker not in compiled.groupindex: + raise ValueError("internal marker is missing") + pattern_identity = pattern.name or f"{entity.name}.patterns[{entity_pattern_index}]" + return _CompiledRule( + entity=entity.name, + pattern_identity=pattern_identity, + confidence=pattern.confidence, + marker=marker, + compiled=compiled, + ) + + +def _iter_catalog_patterns( + catalog: RegexPatternCatalog, +) -> tuple[tuple[RegexEntity, int, RegexPattern], ...]: + return tuple( + (entity, pattern_index, pattern) + for entity in catalog.entities + for pattern_index, pattern in enumerate(entity.patterns) + ) + + +def _contains_inline_flags(pattern: str) -> bool: + escaped = False + in_character_class = False + index = 0 + while index < len(pattern): + character = pattern[index] + if escaped: + escaped = False + elif character == "\\": + escaped = True + elif character == "[": + in_character_class = True + elif character == "]" and in_character_class: + in_character_class = False + elif not in_character_class and pattern.startswith("(?", index): + suffix = pattern[index + 2 :] + if _INLINE_FLAG_PATTERN.match(suffix) is not None: + return True + index += 1 + return False + + +def _resolve_overlaps( + detections: list[tuple[EntityDetection, str]], +) -> tuple[EntityDetection, ...]: + winners: list[EntityDetection] = [] + ranked = sorted( + detections, + key=lambda item: ( + -_categorical_confidence_rank(item[0].confidence), + -(item[0].end - item[0].start), + item[0].start, + item[0].end, + item[0].entity, + item[1], + ), + ) + for candidate, _ in ranked: + if all( + candidate.end <= winner.start or candidate.start >= winner.end + for winner in winners + ): + winners.append(candidate) + return tuple( + sorted( + winners, + key=lambda item: (item.start, item.end, item.entity), + ) + ) + + +def _categorical_confidence_rank(confidence: object) -> int: + if not isinstance(confidence, ConfidenceLevel): + raise EngineContractError("regex detection confidence is invalid") + return CONFIDENCE_RANK[confidence.value] + + +def _render_bounded_replacement( + text: str, + detections: tuple[EntityDetection, ...], + template: str, +) -> str: + projected_size = 0 + cursor = 0 + for detection in detections: + projected_size += len(text[cursor : detection.start].encode("utf-8")) + projected_size += _rendered_template_size(template, detection.entity) + if projected_size > MAX_BODY_BYTES: + raise EngineLimitExceeded("regex replacement exceeds the size limit") + cursor = detection.end + projected_size += len(text[cursor:].encode("utf-8")) + if projected_size > MAX_BODY_BYTES: + raise EngineLimitExceeded("regex replacement exceeds the size limit") + + parts: list[str] = [] + cursor = 0 + for detection in detections: + parts.append(text[cursor : detection.start]) + parts.append(template.format(entity=detection.entity)) + cursor = detection.end + parts.append(text[cursor:]) + return "".join(parts) + + +def _rendered_template_size(template: str, entity: str) -> int: + size = 0 + entity_size = len(entity.encode("utf-8")) + for literal, field_name, _, _ in Formatter().parse(template): + size += len(literal.encode("utf-8")) + if field_name is not None: + size += entity_size + return size + + +_NAME_PATTERN = regex.compile(r"[A-Za-z_][A-Za-z0-9_-]*\Z") +_INLINE_FLAG_PATTERN = regex.compile(r"[A-Za-z0-9-]+(?=[:)])") +_PATTERN_METADATA_KEY = "pattern" + + +__all__ = [ + "RegexEngine", + "RegexEngineConfig", + "RegexEntity", + "RegexPattern", + "RegexPatternCatalog", + "RegexReplacement", +] diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py index 1bd0643..b062339 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -5,8 +5,6 @@ from dataclasses import dataclass from enum import StrEnum -from typing_extensions import override - class ErrorKind(StrEnum): """Whether a failure is attributable to input or middleware internals.""" @@ -19,8 +17,7 @@ class ErrorComponent(StrEnum): """The Privacy Guard component responsible for a failure.""" CONFIG = "config" - FORMAT_HANDLER = "format_handler" - SCANNER = "scanner" + ENGINE = "engine" PROCESSOR = "processor" SERVICE = "service" SERVER = "server" @@ -32,17 +29,9 @@ class ErrorCode(StrEnum): CONFIG_INVALID = "config_invalid" REQUEST_PHASE_INVALID = "request_phase_invalid" REQUEST_BODY_TOO_LARGE = "request_body_too_large" - REQUEST_SHAPE_LIMIT_EXCEEDED = "request_shape_limit_exceeded" - BODY_FORMAT_UNSUPPORTED = "body_format_unsupported" BODY_ENCODING_INVALID = "body_encoding_invalid" - BODY_JSON_INVALID = "body_json_invalid" - BODY_RECONSTRUCTION_INVALID = "body_reconstruction_invalid" - FORMAT_HANDLER_OUTPUT_INVALID = "format_handler_output_invalid" - FORMAT_HANDLER_EXECUTION_FAILED = "format_handler_execution_failed" - SCANNER_OUTPUT_INVALID = "scanner_output_invalid" - SCANNER_EXECUTION_FAILED = "scanner_execution_failed" - SCANNER_CONFIG_INVALID = "scanner_config_invalid" - FINDING_LIMIT_EXCEEDED = "finding_limit_exceeded" + ENGINE_OUTPUT_INVALID = "engine_output_invalid" + ENGINE_EXECUTION_FAILED = "engine_execution_failed" RESULT_LIMIT_EXCEEDED = "result_limit_exceeded" SERVER_BIND_FAILED = "server_bind_failed" UNEXPECTED_SERVICE_FAILURE = "unexpected_service_failure" @@ -87,7 +76,6 @@ def summary(self) -> str: def hint(self) -> str: return self._spec.hint - @override def __str__(self) -> str: return ( f"[{self.code.value}] {self.component.value}.{self.operation}: " @@ -95,31 +83,14 @@ def __str__(self) -> str: ) -class InternalPrivacyGuardError(PrivacyGuardError): - """A cataloged failure reclassified as internal for its runtime context.""" - - @property - @override - def kind(self) -> ErrorKind: - return ErrorKind.INTERNAL - - _ERROR_SPECS: dict[ErrorCode, ErrorSpec] = { ErrorCode.CONFIG_INVALID: ErrorSpec( ErrorKind.INVALID_INPUT, ErrorComponent.CONFIG, "parse", "Policy configuration is invalid.", - "Check allowed fields, strict string types, finding action, confidence, " - "entity filters, and redact template syntax.", - ), - ErrorCode.SCANNER_CONFIG_INVALID: ErrorSpec( - ErrorKind.INVALID_INPUT, - ErrorComponent.SCANNER, - "configure", - "Scanner configuration is invalid.", - "Check the YAML shape, selected profile, catalog limits, names, and " - "regular expressions.", + "Check entity-processing stages, engine configuration, pattern catalogs, " + "replacement recipes, and the on-detection action.", ), ErrorCode.REQUEST_PHASE_INVALID: ErrorSpec( ErrorKind.INVALID_INPUT, @@ -135,84 +106,33 @@ def kind(self) -> ErrorKind: "Request body exceeds the advertised size limit.", "Reduce the request body to the maximum size in the middleware manifest.", ), - ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED: ErrorSpec( - ErrorKind.INVALID_INPUT, - ErrorComponent.FORMAT_HANDLER, - "normalize", - "Request body exceeds a structural scanning limit.", - "Reduce JSON nesting, text fields, or total text content.", - ), - ErrorCode.BODY_FORMAT_UNSUPPORTED: ErrorSpec( - ErrorKind.INVALID_INPUT, - ErrorComponent.FORMAT_HANDLER, - "select", - "Request body format is unsupported.", - "Register or select a supported body format.", - ), ErrorCode.BODY_ENCODING_INVALID: ErrorSpec( ErrorKind.INVALID_INPUT, - ErrorComponent.FORMAT_HANDLER, - "normalize", + ErrorComponent.SERVICE, + "decode_text", "Request body encoding is invalid.", "Supply a valid UTF-8 request body.", ), - ErrorCode.BODY_JSON_INVALID: ErrorSpec( - ErrorKind.INVALID_INPUT, - ErrorComponent.FORMAT_HANDLER, - "normalize", - "Request body is not valid JSON.", - "Check UTF-8 JSON syntax, duplicate keys, non-finite numbers, Unicode " - "scalars, and the configured body format.", - ), - ErrorCode.BODY_RECONSTRUCTION_INVALID: ErrorSpec( - ErrorKind.INTERNAL, - ErrorComponent.FORMAT_HANDLER, - "reconstruct", - "Request body reconstruction failed validation.", - "Check processor-generated paths and replacement text.", - ), - ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID: ErrorSpec( - ErrorKind.INTERNAL, - ErrorComponent.PROCESSOR, - "validate_handler", - "Format handler output is invalid.", - "Check the FormatHandler ABC and normalized-body contract.", - ), - ErrorCode.FORMAT_HANDLER_EXECUTION_FAILED: ErrorSpec( - ErrorKind.INTERNAL, - ErrorComponent.FORMAT_HANDLER, - "call", - "Format handler execution failed.", - "Run handler unit tests for normalization and reconstruction.", - ), - ErrorCode.SCANNER_OUTPUT_INVALID: ErrorSpec( + ErrorCode.ENGINE_OUTPUT_INVALID: ErrorSpec( ErrorKind.INTERNAL, ErrorComponent.PROCESSOR, - "validate_scanner", - "Scanner output is invalid.", - "Check scanner identity, supported entities, result tuple, labels, spans, " - "confidence, and paths.", + "validate_engine", + "An entity-processing engine returned an invalid result.", + "Check the engine run contract, result model, spans, strategy, and limits.", ), - ErrorCode.SCANNER_EXECUTION_FAILED: ErrorSpec( + ErrorCode.ENGINE_EXECUTION_FAILED: ErrorSpec( ErrorKind.INTERNAL, - ErrorComponent.SCANNER, - "scan", - "Scanner execution failed.", - "Run the scanner single-block unit tests.", - ), - ErrorCode.FINDING_LIMIT_EXCEEDED: ErrorSpec( - ErrorKind.INTERNAL, - ErrorComponent.PROCESSOR, - "scan", - "Scanner finding limit was exceeded.", - "Tune scanner cardinality or the policy before enabling traffic.", + ErrorComponent.ENGINE, + "run", + "An entity-processing engine failed.", + "Run the engine's focused configuration and single-text tests.", ), ErrorCode.RESULT_LIMIT_EXCEEDED: ErrorSpec( ErrorKind.INTERNAL, ErrorComponent.SERVICE, "serialize_result", "A safe middleware result could not be represented.", - "Tune redaction size or scanner finding cardinality.", + "Tune replacement size or entity-detection cardinality.", ), ErrorCode.SERVER_BIND_FAILED: ErrorSpec( ErrorKind.INTERNAL, diff --git a/projects/privacy-guard/src/privacy_guard/payloads/__init__.py b/projects/privacy-guard/src/privacy_guard/payloads/__init__.py deleted file mode 100644 index 18cee29..0000000 --- a/projects/privacy-guard/src/privacy_guard/payloads/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Data records carrying one request and its processing result.""" - -from __future__ import annotations - -from privacy_guard.payloads.request import InterceptedRequest -from privacy_guard.payloads.result import ProcessingDecision, ProcessingResult - -__all__ = ["InterceptedRequest", "ProcessingDecision", "ProcessingResult"] diff --git a/projects/privacy-guard/src/privacy_guard/payloads/request.py b/projects/privacy-guard/src/privacy_guard/payloads/request.py deleted file mode 100644 index 9bdafbe..0000000 --- a/projects/privacy-guard/src/privacy_guard/payloads/request.py +++ /dev/null @@ -1,32 +0,0 @@ -"""The provider-bound request captured pre-credentials, free of proto types.""" - -from __future__ import annotations - -from pydantic import Field, field_validator - -from privacy_guard.config import PolicyConfig -from privacy_guard.validation import StrictSensitiveModel - - -class InterceptedRequest(StrictSensitiveModel): - """A captured HTTP request, decoupled from the proto ``HttpRequestEvaluation``. - - The servicer builds this from the proto message so that nothing below the - service layer depends on ``bindings/``. ``raw_body`` is kept out of ``repr`` - so routine object representations do not include the sensitive payload. - """ - - raw_body: bytes = Field(repr=False) - # Retained as normalized request context for future format negotiation; the - # JSON handler is selected explicitly by policy and does not inspect it. - content_type: str = "application/json" - policy_config: PolicyConfig = Field(default_factory=PolicyConfig, repr=False) - request_id: str = "" - - @field_validator("policy_config", mode="before") - @classmethod - def require_parsed_policy_config(cls, value: object) -> object: - """Require policy input to cross its content-safe parser first.""" - if not isinstance(value, PolicyConfig): - raise ValueError("policy config must already be parsed") - return value diff --git a/projects/privacy-guard/src/privacy_guard/payloads/result.py b/projects/privacy-guard/src/privacy_guard/payloads/result.py deleted file mode 100644 index 48b1ed3..0000000 --- a/projects/privacy-guard/src/privacy_guard/payloads/result.py +++ /dev/null @@ -1,26 +0,0 @@ -"""The proto-free outcome of processing one intercepted request.""" - -from __future__ import annotations - -from enum import StrEnum - -from pydantic import Field - -from privacy_guard.scanners import RequestBodyFinding -from privacy_guard.validation import StrictSensitiveModel - - -class ProcessingDecision(StrEnum): - """Whether the supervisor should continue or stop the request.""" - - ALLOW = "allow" - DENY = "deny" - - -class ProcessingResult(StrictSensitiveModel): - """A decision, safe findings, and bytes that replace the body when present.""" - - decision: ProcessingDecision - replacement_body: bytes | None = Field(default=None, repr=False) - findings: tuple[RequestBodyFinding, ...] = () - reason_code: str | None = None diff --git a/projects/privacy-guard/src/privacy_guard/processor.py b/projects/privacy-guard/src/privacy_guard/processor.py deleted file mode 100644 index 133465a..0000000 --- a/projects/privacy-guard/src/privacy_guard/processor.py +++ /dev/null @@ -1,517 +0,0 @@ -"""Proto-free processing for one complete intercepted request.""" - -from __future__ import annotations - -import logging -import math -import time -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from string import Formatter - -from privacy_guard.config import ( - ActionConfig, - BlockActionConfig, - PolicyConfig, - RedactActionConfig, -) -from privacy_guard.constants import ( - BLOCK_REASON_CODE, - CONFIDENCE_RANK, - DEFAULT_SCAN_TIMEOUT_SECONDS, - LIMIT_REASON_CODE, - MAX_BODY_BYTES, - MAX_FINDINGS_PER_BLOCK, - MAX_FINDINGS_PER_REQUEST, - MAX_SCAN_TIMEOUT_SECONDS, - MAX_SCANNED_CHARACTERS, - MAX_TEXT_BLOCKS, - PATTERN_NAME_METADATA_KEY, -) -from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.payloads import ( - InterceptedRequest, - ProcessingDecision, - ProcessingResult, -) -from privacy_guard.request_body import ( - DEFAULT_FORMAT_HANDLERS, - FormatHandler, - FormatHandlerContractError, - RequestBody, - TextBlock, -) -from privacy_guard.scanners import ( - Finding, - RequestBodyFinding, - ScanBudget, - ScanBudgetExceeded, - Scanner, - ScannerConfig, - ScannerContractError, - ScannerFindingLimitExceeded, -) - - -class RequestProcessor: - """Coordinate normalization, scanners, policy, and reconstruction per request. - - Processor instances may be invoked concurrently by the service. Every configured - scanner must therefore make ``scan`` thread-safe and retain no request content. - """ - - def __init__( - self, - scanners: Sequence[Scanner[ScannerConfig]], - format_handlers: Mapping[str, FormatHandler] = DEFAULT_FORMAT_HANDLERS, - *, - scan_timeout_seconds: float = DEFAULT_SCAN_TIMEOUT_SECONDS, - log_request_content: bool = False, - ) -> None: - if ( - isinstance(scan_timeout_seconds, bool) - or not isinstance(scan_timeout_seconds, int | float) - or not math.isfinite(scan_timeout_seconds) - or scan_timeout_seconds <= 0 - or scan_timeout_seconds > MAX_SCAN_TIMEOUT_SECONDS - ): - raise ValueError("scan timeout must be finite, positive, and bounded") - scanner_tuple = tuple(scanners) - if not scanner_tuple: - raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) - names: set[str] = set() - supported_entities: set[str] = set() - for item in scanner_tuple: - if not isinstance(item, Scanner): - raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) - scanner_name = item.scanner_name - if scanner_name in names: - raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) - names.add(scanner_name) - supported_entities.update(item.supported_entity_types) - self._scanners: tuple[Scanner[ScannerConfig], ...] = scanner_tuple - self._supported_entities = frozenset(supported_entities) - self._scan_timeout_seconds = float(scan_timeout_seconds) - self._log_request_content = log_request_content - self._format_handlers: dict[str, FormatHandler] = {} - for format_name, handler in format_handlers.items(): - if not isinstance(handler, FormatHandler): - raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID) - if handler.format_name != format_name: - raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID) - self._format_handlers[format_name] = handler - - def validate_policy_config(self, policy_config: PolicyConfig) -> None: - self._select_handler(policy_config.body_format) - self._validate_entity_filter(policy_config.on_finding) - - def process(self, request: InterceptedRequest) -> ProcessingResult: - if self._log_request_content: - _LOGGER.debug( - "privacy_guard_request_content request_id=%s stage=received body=%r", - request.request_id, - request.raw_body, - ) - _LOGGER.debug( - "privacy_guard_processing_started request_id=%s body_bytes=%d", - request.request_id, - len(request.raw_body), - ) - policy = request.policy_config - action = policy.on_finding - handler = self._select_handler(policy.body_format) - self._validate_entity_filter(action) - if request.raw_body == b"": - _LOGGER.debug( - "privacy_guard_processing_bodyless request_id=%s", - request.request_id, - ) - if self._log_request_content: - _LOGGER.debug( - "privacy_guard_request_content request_id=%s " - "stage=forwarded body=%r", - request.request_id, - request.raw_body, - ) - return ProcessingResult(decision=ProcessingDecision.ALLOW) - - phase_started = time.monotonic() - _LOGGER.debug( - "privacy_guard_processing_phase_started request_id=%s phase=normalize", - request.request_id, - ) - request_body = _normalize_request_body(handler, request) - verified_body = _validate_normalized_body(request_body, request.raw_body) - scanned_characters = sum( - len(text_block.text) for text_block in verified_body.text_blocks - ) - _LOGGER.debug( - "privacy_guard_processing_phase_completed request_id=%s " - "phase=normalize duration_ms=%.3f text_block_count=%d " - "scanned_characters=%d", - request.request_id, - (time.monotonic() - phase_started) * 1000, - len(verified_body.text_blocks), - scanned_characters, - ) - budget = ScanBudget.from_timeout(self._scan_timeout_seconds) - phase_started = time.monotonic() - _LOGGER.debug( - "privacy_guard_processing_phase_started request_id=%s phase=scan", - request.request_id, - ) - scan_result = self._scan_text_blocks(verified_body.text_blocks, action, budget) - _LOGGER.debug( - "privacy_guard_processing_phase_completed request_id=%s phase=scan " - "duration_ms=%.3f finding_count=%d limit_exceeded=%s", - request.request_id, - (time.monotonic() - phase_started) * 1000, - len(scan_result.findings), - scan_result.limit_exceeded, - ) - if scan_result.limit_exceeded: - return ProcessingResult( - decision=ProcessingDecision.DENY, reason_code=LIMIT_REASON_CODE - ) - if isinstance(action, BlockActionConfig) and scan_result.findings: - return ProcessingResult( - decision=ProcessingDecision.DENY, - findings=scan_result.findings, - reason_code=BLOCK_REASON_CODE, - ) - if isinstance(action, RedactActionConfig) and any( - not block.replaceable - and scan_result.findings_by_text_block_path[block.path] - for block in verified_body.text_blocks - ): - # JSON keys are scanned, but never rewritten: key mutation could collide. - return ProcessingResult( - decision=ProcessingDecision.DENY, - findings=scan_result.findings, - reason_code=BLOCK_REASON_CODE, - ) - - replacements: dict[str, str] = {} - redacted_text_bytes = 0 - if isinstance(action, RedactActionConfig): - for block in verified_body.text_blocks: - findings = scan_result.findings_by_text_block_path[block.path] - if findings: - redaction = _build_bounded_redaction( - block.text, - findings, - action.template, - MAX_BODY_BYTES - redacted_text_bytes, - ) - if redaction is None: - return ProcessingResult( - decision=ProcessingDecision.DENY, - reason_code=LIMIT_REASON_CODE, - ) - replacement, projected_size = redaction - redacted_text_bytes += projected_size - replacements[block.path] = replacement - phase_started = time.monotonic() - _LOGGER.debug( - "privacy_guard_processing_phase_started request_id=%s phase=reconstruct", - request.request_id, - ) - reconstructed = _reconstruct_body(handler, verified_body.source, replacements) - _LOGGER.debug( - "privacy_guard_processing_phase_completed request_id=%s " - "phase=reconstruct duration_ms=%.3f output_bytes=%d transformed=%s", - request.request_id, - (time.monotonic() - phase_started) * 1000, - len(reconstructed), - reconstructed != request.raw_body, - ) - if len(reconstructed) > MAX_BODY_BYTES: - return ProcessingResult( - decision=ProcessingDecision.DENY, reason_code=LIMIT_REASON_CODE - ) - if self._log_request_content: - _LOGGER.debug( - "privacy_guard_request_content request_id=%s stage=forwarded body=%r", - request.request_id, - reconstructed, - ) - return ProcessingResult( - decision=ProcessingDecision.ALLOW, - replacement_body=reconstructed - if reconstructed != request.raw_body - else None, - findings=scan_result.findings, - ) - - def _select_handler(self, requested_name: str) -> FormatHandler: - try: - handler = self._format_handlers[requested_name] - except KeyError: - raise PrivacyGuardError(ErrorCode.BODY_FORMAT_UNSUPPORTED) from None - return handler - - def _validate_entity_filter(self, action: ActionConfig) -> None: - if not action.entity_types: - return - if not action.entity_types.issubset(self._supported_entities): - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) - - def _scan_and_validate( - self, - scanner: Scanner[ScannerConfig], - text_block: TextBlock, - budget: ScanBudget, - ) -> tuple[Finding, ...]: - try: - result = scanner.scan(text_block.text, budget=budget) - except ScanBudgetExceeded: - raise - except ScannerFindingLimitExceeded: - raise PrivacyGuardError(ErrorCode.FINDING_LIMIT_EXCEEDED) from None - except ScannerContractError: - raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) from None - except PrivacyGuardError as error: - if error.code is ErrorCode.SCANNER_CONFIG_INVALID: - raise - raise PrivacyGuardError(ErrorCode.SCANNER_EXECUTION_FAILED) from None - except Exception: - raise PrivacyGuardError(ErrorCode.SCANNER_EXECUTION_FAILED) from None - for finding in result: - if ( - finding.scanner_name != scanner.scanner_name - or finding.entity not in scanner.supported_entity_types - or finding.end_offset > len(text_block.text) - ): - raise PrivacyGuardError(ErrorCode.SCANNER_OUTPUT_INVALID) - return tuple( - sorted( - result, - key=lambda item: ( - item.start_offset, - item.end_offset, - item.entity, - _finding_pattern_name(item), - ), - ) - ) - - def _scan_text_blocks( - self, - text_blocks: tuple[TextBlock, ...], - action: ActionConfig, - budget: ScanBudget, - ) -> _ScanResult: - by_path: dict[str, tuple[Finding, ...]] = {} - all_findings: list[RequestBodyFinding] = [] - enabled = action.entity_types - threshold = ( - None - if action.minimum_confidence is None - else CONFIDENCE_RANK[action.minimum_confidence] - ) - for block in text_blocks: - block_findings: list[Finding] = [] - try: - for scanner in self._scanners: - for finding in self._scan_and_validate(scanner, block, budget): - if (enabled is None or finding.entity in enabled) and ( - threshold is None - or CONFIDENCE_RANK[finding.confidence] >= threshold - ): - block_findings.append(finding) - if ( - len(block_findings) > MAX_FINDINGS_PER_BLOCK - or len(all_findings) + len(block_findings) - > MAX_FINDINGS_PER_REQUEST - ): - return _ScanResult((), {}, True) - except ScanBudgetExceeded: - return _ScanResult((), {}, True) - except PrivacyGuardError as error: - if error.code is ErrorCode.FINDING_LIMIT_EXCEEDED: - return _ScanResult((), {}, True) - raise - ordered = tuple( - sorted( - block_findings, - key=lambda item: ( - item.start_offset, - item.end_offset, - item.scanner_name, - item.entity, - _finding_pattern_name(item), - ), - ) - ) - by_path[block.path] = ordered - all_findings.extend( - RequestBodyFinding(finding=finding, text_block_path=block.path) - for finding in ordered - ) - return _ScanResult(tuple(all_findings), by_path) - - -@dataclass(frozen=True) -class _ScanResult: - findings: tuple[RequestBodyFinding, ...] - findings_by_text_block_path: Mapping[str, tuple[Finding, ...]] - limit_exceeded: bool = False - - -@dataclass(frozen=True) -class _VerifiedRequestBody: - """A body paired with the processor-validated request-relative view.""" - - source: RequestBody - text_blocks: tuple[TextBlock, ...] - - -def _normalize_request_body( - handler: FormatHandler, request: InterceptedRequest -) -> RequestBody: - try: - return handler.normalize(request.raw_body, request.policy_config) - except FormatHandlerContractError: - raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID) from None - except PrivacyGuardError: - raise - except Exception: - raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_EXECUTION_FAILED) from None - - -def _validate_normalized_body( - request_body: RequestBody, original_body: bytes -) -> _VerifiedRequestBody: - if request_body.original_bytes != original_body: - raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID) - text_blocks = request_body.text_blocks - if len(text_blocks) > MAX_TEXT_BLOCKS: - raise PrivacyGuardError(ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED) - - seen_paths: set[str] = set() - total_characters = 0 - for text_block in text_blocks: - if text_block.path in seen_paths: - raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID) - seen_paths.add(text_block.path) - total_characters += len(text_block.text) - if total_characters > MAX_SCANNED_CHARACTERS: - raise PrivacyGuardError(ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED) - return _VerifiedRequestBody(source=request_body, text_blocks=text_blocks) - - -def _resolve_overlaps(findings: tuple[Finding, ...]) -> tuple[Finding, ...]: - """Choose deterministic redaction winners; observation retains all findings.""" - winners: list[Finding] = [] - ranked = sorted( - findings, - key=lambda item: ( - -CONFIDENCE_RANK[item.confidence], - -(item.end_offset - item.start_offset), - item.start_offset, - item.end_offset, - item.scanner_name, - item.entity, - _finding_pattern_name(item), - ), - ) - for candidate in ranked: - if all( - candidate.end_offset <= winner.start_offset - or candidate.start_offset >= winner.end_offset - for winner in winners - ): - winners.append(candidate) - return tuple( - sorted( - winners, - key=lambda item: ( - item.start_offset, - item.end_offset, - item.scanner_name, - item.entity, - _finding_pattern_name(item), - ), - ) - ) - - -def _redact_text( - original_text: str, findings: tuple[Finding, ...], template: str -) -> str: - parts: list[str] = [] - cursor = 0 - for finding in findings: - parts.append(original_text[cursor : finding.start_offset]) - parts.append(template.format(entity=finding.entity)) - cursor = finding.end_offset - parts.append(original_text[cursor:]) - return "".join(parts) - - -def _formatted_template_size(template: str, entity: str) -> int: - """Return UTF-8 size without constructing a potentially large rendering.""" - size = 0 - entity_size = len(entity.encode("utf-8")) - for literal, field_name, _, _ in Formatter().parse(template): - size += len(literal.encode("utf-8")) - if field_name is not None: - size += entity_size - return size - - -def _project_redacted_text_size( - original_text: str, - findings: tuple[Finding, ...], - template: str, - remaining_bytes: int, -) -> int | None: - """Bound rendered text before allocating it; serialization is checked later.""" - size = 0 - cursor = 0 - for finding in findings: - size += len(original_text[cursor : finding.start_offset].encode("utf-8")) - size += _formatted_template_size(template, finding.entity) - if size > remaining_bytes: - return None - cursor = finding.end_offset - size += len(original_text[cursor:].encode("utf-8")) - return size if size <= remaining_bytes else None - - -def _build_bounded_redaction( - original_text: str, - findings: tuple[Finding, ...], - template: str, - remaining_bytes: int, -) -> tuple[str, int] | None: - """Resolve overlaps once, then render only bounded intermediate text.""" - resolved = _resolve_overlaps(findings) - projected_size = _project_redacted_text_size( - original_text, resolved, template, remaining_bytes - ) - if projected_size is None: - return None - return _redact_text(original_text, resolved, template), projected_size - - -def _reconstruct_body( - handler: FormatHandler, request_body: RequestBody, replacements: Mapping[str, str] -) -> bytes: - try: - return handler.reconstruct(request_body, replacements) - except FormatHandlerContractError: - raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID) from None - except PrivacyGuardError: - raise - except Exception: - raise PrivacyGuardError(ErrorCode.FORMAT_HANDLER_EXECUTION_FAILED) from None - - -def _finding_pattern_name(finding: Finding) -> str: - if finding.metadata is None: - return "" - return finding.metadata.get(PATTERN_NAME_METADATA_KEY, "") - - -_LOGGER = logging.getLogger(__name__) diff --git a/projects/privacy-guard/src/privacy_guard/request_body/__init__.py b/projects/privacy-guard/src/privacy_guard/request_body/__init__.py deleted file mode 100644 index c4b5486..0000000 --- a/projects/privacy-guard/src/privacy_guard/request_body/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Format-handler registry and format-agnostic request-body records.""" - -from __future__ import annotations - -from collections.abc import Mapping -from types import MappingProxyType - -from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.request_body.base import ( - FormatHandler, - FormatHandlerContractError, - RequestBody, - TextBlock, - parse_normalized_body, -) -from privacy_guard.request_body.json import JsonHandler - -DEFAULT_FORMAT_HANDLERS: Mapping[str, FormatHandler] = MappingProxyType( - {handler.format_name: handler for handler in (JsonHandler(),)} -) -"""Built-in handlers keyed by name; processors may receive a custom mapping.""" - - -def select_format_handler(body_format: str) -> FormatHandler: - """Return the registered FormatHandler for ``body_format`` (e.g. "json"). - - ``body_format`` comes from ``PolicyConfig.body_format``. Raise for an - unregistered format rather than silently falling back. - """ - try: - return DEFAULT_FORMAT_HANDLERS[body_format] - except KeyError: - raise PrivacyGuardError(ErrorCode.BODY_FORMAT_UNSUPPORTED) from None - - -__all__ = [ - "DEFAULT_FORMAT_HANDLERS", - "FormatHandler", - "FormatHandlerContractError", - "JsonHandler", - "RequestBody", - "TextBlock", - "parse_normalized_body", - "select_format_handler", -] diff --git a/projects/privacy-guard/src/privacy_guard/request_body/base.py b/projects/privacy-guard/src/privacy_guard/request_body/base.py deleted file mode 100644 index 2453cf0..0000000 --- a/projects/privacy-guard/src/privacy_guard/request_body/base.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Strict request-body models and the nominal format-handler contract.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from collections.abc import Mapping - -from pydantic import Field - -from privacy_guard.config import PolicyConfig -from privacy_guard.validation import ( - ScalarString, - StrictSensitiveModel, - parse_non_empty_scalar_string, -) - - -class TextBlock(StrictSensitiveModel): - """One scannable text block, addressed structurally within the body. - - ``path`` is an opaque, format-owned address that may contain sensitive - structural keys. Callers may use it as a replacement key but must not parse - it. ``replaceable=False`` exposes observable text, such as a JSON object key, - that may be scanned but must not be rewritten. - """ - - path: ScalarString = Field(repr=False) - text: ScalarString = Field(repr=False) - replaceable: bool = True - - -class RequestBody(StrictSensitiveModel): - """A normalized body plus handler-owned reconstruction state. - - ``text_blocks`` are the independently scannable values selected by the - handler. ``parsed_value`` is opaque handler-owned state and reconstruction - must not mutate it. ``original_bytes`` contains bytes equal to the exact input; - a handler returns the stored value for a no-op reconstruction. Rewritten bodies - need only preserve untouched values semantically. - """ - - text_blocks: tuple[TextBlock, ...] - parsed_value: object = Field(repr=False) - original_bytes: bytes = Field(repr=False) - - -class FormatHandlerContractError(Exception): - """A content-safe format-handler metadata or output contract failure.""" - - -class FormatHandler(ABC): - """Nominal, concurrency-safe extension point for one request-body format. - - Processor registries reuse handler instances across requests. Implementations - must therefore retain no request content or mutable per-request state. - """ - - def __init__(self, *, format_name: str) -> None: - try: - self.__format_name = parse_non_empty_scalar_string(format_name) - except ValueError: - raise FormatHandlerContractError( - "format-handler metadata is invalid" - ) from None - - @property - def format_name(self) -> str: - """Return the immutable construction-time format identity.""" - return self.__format_name - - def normalize(self, raw_body: bytes, policy_config: PolicyConfig) -> RequestBody: - """Parse request bytes and validate the implementation's body model.""" - result: object = self._normalize(raw_body, policy_config) - return parse_normalized_body(result) - - def reconstruct( - self, - request_body: RequestBody, - replacements_by_path: Mapping[str, str], - ) -> bytes: - """Rebuild a normalized body and require a bytes result.""" - result: object = self._reconstruct(request_body, replacements_by_path) - if not isinstance(result, bytes): - raise FormatHandlerContractError( - "format-handler output is invalid" - ) from None - return result - - @abstractmethod - def _normalize(self, raw_body: bytes, policy_config: PolicyConfig) -> RequestBody: - """Implement format-specific parsing without retaining request state.""" - raise NotImplementedError - - @abstractmethod - def _reconstruct( - self, - request_body: RequestBody, - replacements_by_path: Mapping[str, str], - ) -> bytes: - """Implement format-specific reconstruction without mutating the model.""" - raise NotImplementedError - - -def parse_normalized_body(result: object) -> RequestBody: - """Require handlers to return the documented body model.""" - if not isinstance(result, RequestBody): - raise FormatHandlerContractError("format-handler output is invalid") from None - return result - - -__all__ = [ - "FormatHandler", - "FormatHandlerContractError", - "RequestBody", - "TextBlock", - "parse_normalized_body", -] diff --git a/projects/privacy-guard/src/privacy_guard/request_body/json.py b/projects/privacy-guard/src/privacy_guard/request_body/json.py deleted file mode 100644 index 3906f9b..0000000 --- a/projects/privacy-guard/src/privacy_guard/request_body/json.py +++ /dev/null @@ -1,300 +0,0 @@ -"""JSON FormatHandler that addresses string leaves by JSON Pointer path.""" - -from __future__ import annotations - -import json -import math -from collections.abc import Iterator, Mapping -from copy import deepcopy -from dataclasses import dataclass, field -from typing import TYPE_CHECKING - -from pydantic import ConfigDict, TypeAdapter, ValidationError -from typing_extensions import TypeAliasType, override - -from privacy_guard.constants import ( - MAX_JSON_NESTING, - MAX_SCANNED_CHARACTERS, - MAX_TEXT_BLOCKS, -) -from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.request_body.base import FormatHandler, RequestBody, TextBlock -from privacy_guard.validation import ScalarString, parse_scalar_string - -if TYPE_CHECKING: - from privacy_guard.config import PolicyConfig - - -JsonValue = TypeAliasType( - "JsonValue", - ScalarString - | int - | float - | bool - | None - | dict[ScalarString, "JsonValue"] - | list["JsonValue"], -) - - -class JsonHandler(FormatHandler): - """Handle strict UTF-8 JSON bodies, with one TextBlock per string leaf. - - Text-block paths are JSON Pointers (RFC 6901), e.g. ``/items/0/text``, so - ``reconstruct`` can locate each replacement. Rewritten JSON preserves - untouched values semantically but may change serialization details. - """ - - def __init__(self) -> None: - super().__init__(format_name="json") - - @override - def _normalize(self, raw_body: bytes, policy_config: PolicyConfig) -> RequestBody: - """Decode UTF-8, parse JSON, and collect every string leaf as a text block.""" - try: - decoded_body = raw_body.decode("utf-8") - except UnicodeDecodeError: - raise PrivacyGuardError(ErrorCode.BODY_ENCODING_INVALID) from None - - try: - parsed_value = self._parse_json_value(decoded_body) - except _InvalidJsonError: - raise PrivacyGuardError(ErrorCode.BODY_JSON_INVALID) from None - except _InvalidJsonShapeError: - raise PrivacyGuardError(ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED) from None - - try: - text_blocks = tuple(self._iter_text_blocks(parsed_value)) - except _InvalidJsonError: - raise PrivacyGuardError(ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED) from None - return RequestBody( - text_blocks=text_blocks, - parsed_value=_JsonBodyState(parsed_value), - original_bytes=raw_body, - ) - - @override - def _reconstruct( - self, - request_body: RequestBody, - replacements_by_path: Mapping[str, str], - ) -> bytes: - """Apply JSON Pointer replacements, or return exact bytes for a no-op.""" - parsed_state = request_body.parsed_value - if not isinstance(parsed_state, _JsonBodyState): - raise PrivacyGuardError(ErrorCode.BODY_RECONSTRUCTION_INVALID) from None - if not replacements_by_path: - return request_body.original_bytes - - try: - reconstructed_value = deepcopy(parsed_state.value) - for json_pointer, replacement_text in replacements_by_path.items(): - parse_scalar_string(replacement_text) - - path_tokens = self._decode_json_pointer(json_pointer) - if not path_tokens: - if not isinstance(reconstructed_value, str): - raise _InvalidJsonError - reconstructed_value = replacement_text - continue - - parent_node = reconstructed_value - for path_token in path_tokens[:-1]: - parent_node = self._resolve_child_node(parent_node, path_token) - - final_token = path_tokens[-1] - target_value = self._resolve_child_node(parent_node, final_token) - if not isinstance(target_value, str): - raise _InvalidJsonError - - if isinstance(parent_node, dict): - parent_node[final_token] = replacement_text - elif isinstance(parent_node, list): - array_index = self._parse_array_index(final_token, len(parent_node)) - parent_node[array_index] = replacement_text - else: - raise _InvalidJsonError - except Exception: - raise PrivacyGuardError(ErrorCode.BODY_RECONSTRUCTION_INVALID) from None - - try: - return json.dumps( - reconstructed_value, - ensure_ascii=False, - separators=(",", ":"), - allow_nan=False, - ).encode("utf-8") - except Exception: - raise PrivacyGuardError(ErrorCode.BODY_RECONSTRUCTION_INVALID) from None - - @staticmethod - def _build_unique_object( - object_pairs: list[tuple[str, object]], - ) -> dict[str, object]: - object_members: dict[str, object] = {} - for member_name, member_value in object_pairs: - if member_name in object_members: - raise _InvalidJsonError - object_members[member_name] = member_value - return object_members - - @staticmethod - def _reject_non_finite_constant(_: str) -> object: - raise _InvalidJsonError - - @staticmethod - def _parse_finite_float(value: str) -> float: - parsed = float(value) - if not math.isfinite(parsed): - raise _InvalidJsonError - return parsed - - @classmethod - def _parse_json_value(cls, decoded_body: str) -> JsonValue: - """Parse duplicate-aware JSON and return its one validated typed tree.""" - try: - raw_value = json.loads( - decoded_body, - object_pairs_hook=cls._build_unique_object, - parse_constant=cls._reject_non_finite_constant, - parse_float=cls._parse_finite_float, - ) - except RecursionError: - raise _InvalidJsonShapeError from None - except (json.JSONDecodeError, _InvalidJsonError, ValueError): - raise _InvalidJsonError from None - - try: - return _JSON_VALUE_ADAPTER.validate_python(raw_value, strict=True) - except ValidationError as validation_error: - if any( - error["type"] == "recursion_loop" - for error in validation_error.errors( - include_url=False, - include_context=False, - include_input=False, - ) - ): - raise _InvalidJsonShapeError from None - raise _InvalidJsonError from None - - @classmethod - def _iter_text_blocks(cls, parsed_json: JsonValue) -> Iterator[TextBlock]: - """Walk a parsed JSON value incrementally in stable depth-first order.""" - count = 0 - total_characters = 0 - for emitted in cls._walk_text_blocks(parsed_json, "", 0, True): - # Bound built-in parsing before tuple/model materialization. The - # processor repeats aggregates for arbitrary handler outputs. - count += 1 - total_characters += len(emitted.text) - if count > MAX_TEXT_BLOCKS or total_characters > MAX_SCANNED_CHARACTERS: - raise _InvalidJsonError - yield emitted - - @classmethod - def _walk_text_blocks( - cls, - node: JsonValue, - json_pointer: str, - depth: int, - replaceable: bool, - ) -> Iterator[TextBlock]: - """Yield one branch at a time so traversal state remains O(depth).""" - if depth > MAX_JSON_NESTING: - raise _InvalidJsonError - if isinstance(node, str): - yield TextBlock(path=json_pointer, text=node, replaceable=replaceable) - return - if isinstance(node, dict): - for key, child_node in node.items(): - path_token = key.replace("~", "~0").replace("/", "~1") - child_path = f"{json_pointer}/{path_token}" - # Key paths are deliberately outside the JSON Pointer namespace. - yield from cls._walk_text_blocks( - key, f"#key:{child_path}", depth, False - ) - yield from cls._walk_text_blocks( - child_node, child_path, depth + 1, True - ) - return - if isinstance(node, list): - for index, child_node in enumerate(node): - yield from cls._walk_text_blocks( - child_node, f"{json_pointer}/{index}", depth + 1, True - ) - - @staticmethod - def _decode_json_pointer(json_pointer: str) -> list[str]: - if json_pointer == "": - return [] - if not json_pointer.startswith("/"): - raise _InvalidJsonError - - decoded_tokens: list[str] = [] - for encoded_token in json_pointer[1:].split("/"): - decoded_characters: list[str] = [] - character_index = 0 - while character_index < len(encoded_token): - character = encoded_token[character_index] - if character != "~": - decoded_characters.append(character) - character_index += 1 - continue - if character_index + 1 >= len(encoded_token) or encoded_token[ - character_index + 1 - ] not in {"0", "1"}: - raise _InvalidJsonError - decoded_characters.append( - "~" if encoded_token[character_index + 1] == "0" else "/" - ) - character_index += 2 - decoded_tokens.append("".join(decoded_characters)) - return decoded_tokens - - @classmethod - def _resolve_child_node(cls, parent_node: JsonValue, path_token: str) -> JsonValue: - if isinstance(parent_node, dict): - if path_token not in parent_node: - raise _InvalidJsonError - return parent_node[path_token] - if isinstance(parent_node, list): - array_index = cls._parse_array_index(path_token, len(parent_node)) - return parent_node[array_index] - raise _InvalidJsonError - - @staticmethod - def _parse_array_index(path_token: str, array_length: int) -> int: - if not path_token.isascii() or not path_token.isdecimal(): - raise _InvalidJsonError - if len(path_token) > 1 and path_token.startswith("0"): - raise _InvalidJsonError - array_index = int(path_token) - if array_index >= array_length: - raise _InvalidJsonError - return array_index - - -_JSON_VALUE_ADAPTER = TypeAdapter( - JsonValue, - config=ConfigDict( - strict=True, - allow_inf_nan=False, - hide_input_in_errors=True, - ), -) - - -@dataclass(frozen=True) -class _JsonBodyState: - """Typed reconstruction state assembled from an already validated value.""" - - value: JsonValue = field(repr=False) - - -class _InvalidJsonError(ValueError): - """Signal a strict-JSON violation without retaining request content.""" - - -class _InvalidJsonShapeError(ValueError): - """Signal JSON nesting that exceeds a parser or validator safe limit.""" diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py new file mode 100644 index 0000000..af6121d --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -0,0 +1,222 @@ +"""Sequential entity-processing orchestration for one text input.""" + +from __future__ import annotations + +import logging +import math +from collections import OrderedDict +from collections.abc import Sequence +from enum import StrEnum +from typing import Protocol + +from pydantic import Field + +from privacy_guard.base import StrictDomainModel +from privacy_guard.config import FinalizedPrivacyGuardConfig, PolicyAction +from privacy_guard.constants import ( + BLOCK_REASON_CODE, + DEFAULT_TIMEOUT_SECONDS, + LIMIT_REASON_CODE, + MAX_BODY_BYTES, + MAX_DETECTIONS_PER_REQUEST, + MAX_SCANNED_CHARACTERS, + MAX_TIMEOUT_SECONDS, +) +from privacy_guard.engines import ( + DetectionConfidence, + EngineContractError, + EngineLimitExceeded, + EntityProcessingError, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.string_validators import validate_scalar_string +from privacy_guard.timeout import Timeout, TimeoutExpired + + +class RequestDecision(StrEnum): + """Whether OpenShell should continue or stop the request.""" + + ALLOW = "allow" + DENY = "deny" + + +class EntityDetectionSummary(StrictDomainModel): + """One bounded aggregate suitable for user-facing audit output.""" + + entity: str + source_stage: str + confidence: DetectionConfidence | None = None + count: int = Field(ge=1) + + +class RequestProcessingResult(StrictDomainModel): + """The processor's decision, summaries, and optional replacement text.""" + + decision: RequestDecision + replacement_text: str | None = Field(default=None, repr=False) + detection_summaries: tuple[EntityDetectionSummary, ...] = () + reason_code: str | None = None + + +class RequestProcessor: + """Run configured entity-processing stages once, in policy order.""" + + def __init__( + self, + config: FinalizedPrivacyGuardConfig, + stages: Sequence[tuple[str, _RunnableEngine]], + *, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + log_request_content: bool = False, + ) -> None: + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, int | float) + or not math.isfinite(timeout_seconds) + or timeout_seconds <= 0 + or timeout_seconds > MAX_TIMEOUT_SECONDS + ): + raise ValueError("timeout must be finite, positive, and bounded") + configured_stages = tuple(stages) + if len(configured_stages) != len(config.entity_processing.stages): + raise ValueError("configured stages do not match the policy") + if not configured_stages: + raise ValueError("at least one configured stage is required") + sources = tuple(source for source, _ in configured_stages) + if any(not source for source in sources) or len(sources) != len(set(sources)): + raise ValueError("stage sources must be non-empty and unique") + self._config = config + self._stages = configured_stages + self._timeout_seconds = float(timeout_seconds) + self._log_request_content = log_request_content + + @property + def config(self) -> FinalizedPrivacyGuardConfig: + """Return the exact validated configuration retained by this processor.""" + return self._config + + def process(self, text: str) -> RequestProcessingResult: + """Process one complete request text and apply the user-facing action.""" + try: + input_text = validate_scalar_string(text) + except ValueError: + raise PrivacyGuardError(ErrorCode.BODY_ENCODING_INVALID) from None + if ( + len(input_text) > MAX_SCANNED_CHARACTERS + or len(input_text.encode("utf-8")) > MAX_BODY_BYTES + ): + raise PrivacyGuardError(ErrorCode.REQUEST_BODY_TOO_LARGE) + if self._log_request_content: + _LOGGER.debug("privacy_guard_text_input text=%r", input_text) + + action = self._config.on_detection.action + strategy = ( + EntityProcessingStrategy.REPLACE + if action is PolicyAction.REPLACE + else EntityProcessingStrategy.DETECT + ) + timeout = Timeout.from_seconds(self._timeout_seconds) + current_text = input_text + stage_results: list[tuple[str, TextProcessingResult]] = [] + try: + for source, engine in self._stages: + _LOGGER.debug( + "privacy_guard_stage_run source=%s strategy=%s", + source, + strategy.value, + ) + result = engine.run( + current_text, + strategy=strategy, + timeout=timeout, + ) + if ( + len(result.text) > MAX_SCANNED_CHARACTERS + or len(result.text.encode("utf-8")) > MAX_BODY_BYTES + ): + raise EngineLimitExceeded("intermediate text exceeds the limit") + if ( + sum(len(item.detections) for _, item in stage_results) + + len(result.detections) + > MAX_DETECTIONS_PER_REQUEST + ): + raise EngineLimitExceeded("request detections exceed the limit") + stage_results.append((source, result)) + current_text = result.text + timeout.raise_if_expired() + except (EngineLimitExceeded, TimeoutExpired): + return RequestProcessingResult( + decision=RequestDecision.DENY, + reason_code=LIMIT_REASON_CODE, + ) + except EngineContractError: + raise PrivacyGuardError(ErrorCode.ENGINE_OUTPUT_INVALID) from None + except EntityProcessingError: + raise PrivacyGuardError(ErrorCode.ENGINE_EXECUTION_FAILED) from None + except PrivacyGuardError: + raise + except Exception: + raise PrivacyGuardError(ErrorCode.ENGINE_EXECUTION_FAILED) from None + + detections = _aggregate_detections(stage_results) + if action is PolicyAction.BLOCK and detections: + return RequestProcessingResult( + decision=RequestDecision.DENY, + detection_summaries=detections, + reason_code=BLOCK_REASON_CODE, + ) + replacement_text = current_text if action is PolicyAction.REPLACE else None + if self._log_request_content: + _LOGGER.debug("privacy_guard_text_output text=%r", current_text) + return RequestProcessingResult( + decision=RequestDecision.ALLOW, + replacement_text=replacement_text, + detection_summaries=detections, + ) + + +def _aggregate_detections( + stage_results: Sequence[tuple[str, TextProcessingResult]], +) -> tuple[EntityDetectionSummary, ...]: + groups: OrderedDict[ + tuple[str, str, DetectionConfidence | None], + int, + ] = OrderedDict() + for source, result in stage_results: + for detection in result.detections: + key = (source, detection.entity, detection.confidence) + groups[key] = groups.get(key, 0) + 1 + return tuple( + EntityDetectionSummary( + source_stage=source, + entity=entity, + confidence=confidence, + count=count, + ) + for (source, entity, confidence), count in groups.items() + ) + + +class _RunnableEngine(Protocol): + """The engine behavior needed by request orchestration.""" + + def run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: ... + + +_LOGGER = logging.getLogger(__name__) + + +__all__ = [ + "EntityDetectionSummary", + "RequestDecision", + "RequestProcessingResult", + "RequestProcessor", +] diff --git a/projects/privacy-guard/src/privacy_guard/scanners/__init__.py b/projects/privacy-guard/src/privacy_guard/scanners/__init__.py deleted file mode 100644 index 67212d6..0000000 --- a/projects/privacy-guard/src/privacy_guard/scanners/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Scan individual text blocks into safe findings without copying matches.""" - -from __future__ import annotations - -from privacy_guard.scanners.base import ( - Confidence, - Finding, - RequestBodyFinding, - ScanBudget, - ScanBudgetExceeded, - Scanner, - ScannerConfig, - ScannerContractError, - ScannerFindingLimitExceeded, - parse_scanner_output, -) -from privacy_guard.scanners.regex import ( - RegexEntity, - RegexPattern, - RegexScanner, - RegexScannerConfig, -) - -__all__ = [ - "Confidence", - "Finding", - "RequestBodyFinding", - "RegexEntity", - "RegexPattern", - "RegexScanner", - "RegexScannerConfig", - "ScanBudget", - "ScanBudgetExceeded", - "Scanner", - "ScannerConfig", - "ScannerContractError", - "ScannerFindingLimitExceeded", - "parse_scanner_output", -] diff --git a/projects/privacy-guard/src/privacy_guard/scanners/base.py b/projects/privacy-guard/src/privacy_guard/scanners/base.py deleted file mode 100644 index f600c85..0000000 --- a/projects/privacy-guard/src/privacy_guard/scanners/base.py +++ /dev/null @@ -1,225 +0,0 @@ -"""Strict finding models and the nominal scanner extension contract.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from collections.abc import Mapping -from enum import StrEnum -from time import monotonic -from types import MappingProxyType -from typing import Generic, Self, TypeVar, get_args, get_origin - -from pydantic import ( - Field, - ValidationError, - field_validator, - model_validator, -) -from typing_extensions import TypeIs - -from privacy_guard.constants import ( - DEFAULT_SCAN_TIMEOUT_SECONDS, - MAX_FINDING_METADATA_ENTRIES, - MAX_FINDINGS_PER_BLOCK, - MAX_SCANNER_METADATA_BYTES, -) -from privacy_guard.validation import ScalarString, StrictDomainModel - - -class Confidence(StrEnum): - """Scanner-owned confidence assigned to a finding.""" - - LOW = "low" - MEDIUM = "medium" - HIGH = "high" - - -class Finding(StrictDomainModel): - """A scanner-owned, block-relative sensitive-data detection.""" - - entity: str = Field(..., min_length=1, max_length=MAX_SCANNER_METADATA_BYTES) - scanner_name: str = Field(..., min_length=1, max_length=MAX_SCANNER_METADATA_BYTES) - start_offset: int = Field(..., ge=0) - end_offset: int - confidence: Confidence = Confidence.HIGH - metadata: Mapping[str, str] | None = Field(default=None, repr=False) - - @field_validator("entity", "scanner_name") - @classmethod - def validate_metadata_boundary(cls, value: str) -> str: - """Require valid Unicode metadata that fits the UTF-8 byte limit.""" - if any("\ud800" <= character <= "\udfff" for character in value): - raise ValueError("finding metadata must contain valid Unicode") - if len(value.encode("utf-8")) > MAX_SCANNER_METADATA_BYTES: - raise ValueError("finding metadata exceeds the UTF-8 byte limit") - return value - - @field_validator("metadata") - @classmethod - def validate_finding_metadata( - cls, value: Mapping[str, str] | None - ) -> Mapping[str, str] | None: - """Copy valid, bounded string metadata into an immutable mapping.""" - if value is None: - return None - if len(value) > MAX_FINDING_METADATA_ENTRIES: - raise ValueError("finding metadata has too many entries") - for key, item in value.items(): - cls.validate_metadata_boundary(key) - cls.validate_metadata_boundary(item) - return MappingProxyType(dict(value)) - - @model_validator(mode="after") - def validate_non_empty_span(self) -> Self: - """Require the finding to cover at least one character.""" - if self.end_offset <= self.start_offset: - raise ValueError("finding span must be non-empty") - return self - - -class RequestBodyFinding(StrictDomainModel): - """Place a block-relative Finding at its path within a normalized RequestBody.""" - - finding: Finding - text_block_path: ScalarString = Field(repr=False) - - -class ScannerConfig(StrictDomainModel): - """Immutable scanner identity and complete entity catalog.""" - - name: str = Field(..., min_length=1) - entity_types: frozenset[str] - - -class ScannerContractError(Exception): - """A content-safe scanner configuration or output contract failure.""" - - -class ScannerFindingLimitExceeded(Exception): - """The scanner exceeded the bounded per-block finding count.""" - - -class ScanBudgetExceeded(Exception): - """A content-safe signal that scanning exhausted its shared request budget.""" - - -class ScanBudget(StrictDomainModel): - """Immutable request-scoped monotonic deadline shared by all scanner calls.""" - - deadline: float = Field(allow_inf_nan=False) - - @classmethod - def from_timeout(cls, timeout_seconds: float) -> Self: - if ( - isinstance(timeout_seconds, bool) - or not isinstance(timeout_seconds, int | float) - or timeout_seconds <= 0 - ): - raise ValueError("scan timeout must be finite and positive") - return cls(deadline=monotonic() + timeout_seconds) - - def remaining_seconds(self) -> float: - remaining = self.deadline - monotonic() - if remaining <= 0: - raise ScanBudgetExceeded - return remaining - - -def parse_scanner_output(result: object) -> tuple[Finding, ...]: - """Validate a scanner's tuple output against the Finding contract.""" - if not isinstance(result, tuple): - raise ScannerContractError("scanner output is invalid") - if len(result) > MAX_FINDINGS_PER_BLOCK: - raise ScannerFindingLimitExceeded - if not _contains_only_findings(result): - raise ScannerContractError("scanner output is invalid") - return result - - -def _contains_only_findings( - result: tuple[object, ...], -) -> TypeIs[tuple[Finding, ...]]: - return all(isinstance(finding, Finding) for finding in result) - - -# This private type variable must precede the public generic classes that use it. -_ScannerConfigT = TypeVar("_ScannerConfigT", bound=ScannerConfig, covariant=True) - - -class Scanner(ABC, Generic[_ScannerConfigT]): - """Nominal, concurrency-safe extension point for scanning one text block.""" - - def __init__(self, config: _ScannerConfigT) -> None: - """Validate and retain the scanner's declared concrete configuration.""" - try: - validated_config = self.get_config_type().model_validate(config) - except ValidationError: - raise ScannerContractError("scanner configuration is invalid") from None - self.__config = validated_config - self.__scanner_name = validated_config.name - self._initialize() - - @classmethod - def get_config_type(cls) -> type[_ScannerConfigT]: - """Return the ScannerConfig type declared by the subclass.""" - for base in getattr(cls, "__orig_bases__", ()): - for argument in get_args(base): - origin = get_origin(argument) or argument - if isinstance(origin, type) and issubclass(origin, ScannerConfig): - return origin - raise TypeError(f"{cls.__name__} must declare a ScannerConfig generic argument") - - @property - def config(self) -> _ScannerConfigT: - """Return the scanner's immutable concrete configuration.""" - return self.__config - - @property - def scanner_name(self) -> str: - """Return the immutable stable scanner name.""" - return self.__scanner_name - - @property - def supported_entity_types(self) -> frozenset[str]: - """Return the complete entity catalog declared by the scanner config.""" - return self.config.entity_types - - def _initialize(self) -> None: - """Optionally initialize scanner state after validated config is retained.""" - - def scan( - self, text_block: ScalarString, *, budget: ScanBudget | None = None - ) -> tuple[Finding, ...]: - """Scan one block using a shared or standalone request deadline. - - RequestProcessor supplies one budget to every scanner and text block in a - request. Standalone callers may omit it and receive a fresh default budget. - """ - effective_budget = budget or ScanBudget.from_timeout( - DEFAULT_SCAN_TIMEOUT_SECONDS - ) - result: object = self._scan(text_block, effective_budget) - return parse_scanner_output(result) - - @abstractmethod - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - """Return block-relative findings for one block of text. - - Implementations with potentially expensive work should call - ``budget.remaining_seconds()`` at practical interruption points. - """ - raise NotImplementedError - - -__all__ = [ - "Confidence", - "Finding", - "RequestBodyFinding", - "ScanBudget", - "ScanBudgetExceeded", - "Scanner", - "ScannerConfig", - "ScannerContractError", - "ScannerFindingLimitExceeded", - "parse_scanner_output", -] diff --git a/projects/privacy-guard/src/privacy_guard/scanners/regex.py b/projects/privacy-guard/src/privacy_guard/scanners/regex.py deleted file mode 100644 index d426165..0000000 --- a/projects/privacy-guard/src/privacy_guard/scanners/regex.py +++ /dev/null @@ -1,437 +0,0 @@ -"""Strict YAML-configured regular-expression scanner.""" - -from __future__ import annotations - -import re -from collections.abc import Mapping -from dataclasses import dataclass -from pathlib import Path -from typing import Self - -import yaml -from pydantic import Field, ValidationError, field_validator, model_validator -from typing_extensions import TypeIs, override -from yaml.events import ( - AliasEvent, - CollectionEndEvent, - CollectionStartEvent, - NodeEvent, - ScalarEvent, -) -from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode - -from privacy_guard.constants import ( - MAX_FINDINGS_PER_BLOCK, - MAX_MATCHES_PER_PATTERN, - MAX_REGEX_ENTITIES_PER_PROFILE, - MAX_REGEX_ENTITIES_TOTAL, - MAX_REGEX_NAME_BYTES, - MAX_REGEX_PATTERN_BYTES, - MAX_REGEX_PATTERNS_PER_PROFILE, - MAX_REGEX_PATTERNS_TOTAL, - MAX_REGEX_PROFILES, - MAX_SCANNER_CONFIG_BYTES, - MAX_SCANNER_CONFIG_NESTING, - MAX_SCANNER_CONFIG_NODES, - MAX_SCANNER_CONFIG_SCALAR_BYTES, - PATTERN_NAME_METADATA_KEY, -) -from privacy_guard.errors import ( - ErrorCode, - InternalPrivacyGuardError, - PrivacyGuardError, -) -from privacy_guard.scanners.base import ( - Confidence, - Finding, - ScanBudget, - Scanner, - ScannerConfig, - ScannerFindingLimitExceeded, -) -from privacy_guard.validation import StrictSensitiveModel - - -class RegexPattern(StrictSensitiveModel): - """One named expression and its explicit engine flags.""" - - name: str - regex: str = Field(repr=False) - confidence: Confidence - ignore_case: bool = False - multiline: bool = False - dot_all: bool = False - ascii: bool = False - - @field_validator("name") - @classmethod - def _name_is_safe(cls, value: str) -> str: - return _validate_name(value) - - @field_validator("regex") - @classmethod - def _regex_is_bounded(cls, value: str) -> str: - if not isinstance(value, str) or not value: - raise ValueError("regex must be a non-empty string") - if any("\ud800" <= character <= "\udfff" for character in value): - raise ValueError("regex must contain valid Unicode") - if len(value.encode("utf-8")) > MAX_REGEX_PATTERN_BYTES: - raise ValueError("regex is too long") - return value - - @field_validator("confidence", mode="before") - @classmethod - def _parse_confidence(cls, value: object) -> Confidence: - if not isinstance(value, str): - raise ValueError("confidence must be a string") - return Confidence(value) - - -class RegexEntity(StrictSensitiveModel): - """A named entity and its non-empty pattern catalog.""" - - name: str - patterns: tuple[RegexPattern, ...] = Field(repr=False) - - @field_validator("name") - @classmethod - def _name_is_safe(cls, value: str) -> str: - return _validate_name(value) - - @field_validator("patterns", mode="before") - @classmethod - def _patterns_are_a_list(cls, value: object) -> object: - if not isinstance(value, list | tuple) or not value: - raise ValueError("patterns must be a non-empty list") - return tuple(value) - - @model_validator(mode="after") - def _pattern_names_are_unique(self) -> Self: - names = [pattern.name for pattern in self.patterns] - if len(names) != len(set(names)): - raise ValueError("pattern names must be unique within an entity") - return self - - -class RegexScannerConfig(ScannerConfig): - """Selected, immutable regular-expression entity catalog.""" - - entities: tuple[RegexEntity, ...] = Field(repr=False) - - @model_validator(mode="after") - def _catalog_is_consistent(self) -> Self: - if not self.entities: - raise ValueError("entity catalog must not be empty") - names = [entity.name for entity in self.entities] - if len(names) != len(set(names)) or self.entity_types != frozenset(names): - raise ValueError("entity catalog is inconsistent") - if len(self.entities) > MAX_REGEX_ENTITIES_PER_PROFILE: - raise ValueError("entity catalog is too large") - if sum(len(entity.patterns) for entity in self.entities) > ( - MAX_REGEX_PATTERNS_PER_PROFILE - ): - raise ValueError("pattern catalog is too large") - return self - - -class RegexScanner(Scanner[RegexScannerConfig]): - """Find every configured, possibly overlapping regular-expression match.""" - - @override - def _initialize(self) -> None: - """Compile the validated catalog once for reuse across scanner calls.""" - try: - self._rules = tuple( - _compile_rule(entity.name, pattern) - for entity in self.config.entities - for pattern in entity.patterns - ) - except (RecursionError, ValueError, re.error): - raise PrivacyGuardError(ErrorCode.SCANNER_CONFIG_INVALID) from None - - @classmethod - def from_yaml( - cls, - path: str | Path, - profile: str | None = None, - *, - scanner_name: str = "regex", - ) -> Self: - """Load, validate, select, and compile a complete YAML catalog.""" - try: - raw = _read_bounded_file(path) - value = _load_yaml(raw) - catalogs = _parse_catalogs(value) - for entities in catalogs.values(): - for entity in entities: - for pattern in entity.patterns: - _compile_rule(entity.name, pattern) - entities = _select_catalog(catalogs, profile) - config = RegexScannerConfig( - name=_validate_name(scanner_name), - entity_types=frozenset(entity.name for entity in entities), - entities=entities, - ) - return cls(config) - except PrivacyGuardError: - raise - except ( - OSError, - RecursionError, - UnicodeError, - ValueError, - TypeError, - ValidationError, - yaml.YAMLError, - re.error, - ): - raise PrivacyGuardError(ErrorCode.SCANNER_CONFIG_INVALID) from None - - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - findings: list[Finding] = [] - for rule in self._rules: - match_count = 0 - next_position = 0 - while next_position <= len(text_block): - budget.remaining_seconds() - match = rule.expression.search(text_block, next_position) - budget.remaining_seconds() - if match is None: - break - start, end = match.span() - if start == end: - raise InternalPrivacyGuardError(ErrorCode.SCANNER_CONFIG_INVALID) - if match.span(rule.marker_name) == (-1, -1): - raise InternalPrivacyGuardError(ErrorCode.SCANNER_CONFIG_INVALID) - match_count += 1 - if match_count > MAX_MATCHES_PER_PATTERN: - raise ScannerFindingLimitExceeded - findings.append( - Finding( - entity=rule.entity_name, - metadata={PATTERN_NAME_METADATA_KEY: rule.pattern_name}, - scanner_name=self.scanner_name, - start_offset=start, - end_offset=end, - confidence=rule.confidence, - ) - ) - if len(findings) > MAX_FINDINGS_PER_BLOCK: - raise ScannerFindingLimitExceeded - next_position = start + 1 - return tuple(findings) - - -__all__ = ["RegexEntity", "RegexPattern", "RegexScanner", "RegexScannerConfig"] - - -@dataclass(frozen=True) -class _CompiledRule: - entity_name: str - pattern_name: str - confidence: Confidence - marker_name: str - expression: re.Pattern[str] - - -def _validate_name(value: str) -> str: - if not isinstance(value, str) or _NAME_PATTERN.fullmatch(value) is None: - raise ValueError("name is invalid") - if len(value.encode("ascii")) > MAX_REGEX_NAME_BYTES: - raise ValueError("name is too long") - return value - - -def _read_bounded_file(path: str | Path) -> bytes: - """Bound allocation before handing configuration bytes to PyYAML.""" - with Path(path).open("rb") as stream: - return stream.read(MAX_SCANNER_CONFIG_BYTES + 1) - - -def _compile_rule(entity_name: str, pattern: RegexPattern) -> _CompiledRule: - flags = 0 - if pattern.ignore_case: - flags |= re.IGNORECASE - if pattern.multiline: - flags |= re.MULTILINE - if pattern.dot_all: - flags |= re.DOTALL - if pattern.ascii: - flags |= re.ASCII - if _contains_inline_flags(pattern.regex): - raise ValueError("inline flags are unsupported") - unmarked = re.compile(pattern.regex, flags) - if unmarked.groupindex: - raise ValueError("named groups are reserved") - if unmarked.search("") is not None: - raise ValueError("regex must not match empty input") - marker_name = pattern.name.replace("-", "_") - expression = re.compile(f"(?:{pattern.regex})(?P<{marker_name}>)", flags) - return _CompiledRule( - entity_name=entity_name, - pattern_name=pattern.name, - confidence=pattern.confidence, - marker_name=marker_name, - expression=expression, - ) - - -def _contains_inline_flags(expression: str) -> bool: - escaped = False - in_character_class = False - index = 0 - while index < len(expression): - character = expression[index] - if escaped: - escaped = False - elif character == "\\": - escaped = True - elif character == "[": - in_character_class = True - elif character == "]" and in_character_class: - in_character_class = False - elif not in_character_class and expression.startswith("(?", index): - suffix = expression[index + 2 :] - flag_match = re.match(r"[A-Za-z0-9-]+(?=[:)])", suffix) - if flag_match is not None: - return True - index += 1 - return False - - -def _load_yaml(raw: bytes) -> object: - if not raw or len(raw) > MAX_SCANNER_CONFIG_BYTES: - raise ValueError("configuration size is invalid") - text = raw.decode("utf-8", errors="strict") - _validate_yaml_events(text) - root = yaml.compose(text, Loader=yaml.SafeLoader) - if root is None: - raise ValueError("configuration is empty") - _validate_yaml_node(root, depth=1, count=[0]) - return yaml.safe_load(text) - - -def _validate_yaml_events(text: str) -> None: - depth = 0 - node_count = 0 - for event in yaml.parse(text, Loader=yaml.SafeLoader): - if isinstance(event, AliasEvent) or ( - isinstance(event, NodeEvent) and event.anchor is not None - ): - raise ValueError("anchors and aliases are unsupported") - if isinstance(event, NodeEvent): - node_count += 1 - if node_count > MAX_SCANNER_CONFIG_NODES: - raise ValueError("configuration has too many nodes") - if event.tag is not None and not event.tag.startswith("tag:yaml.org,2002:"): - raise ValueError("YAML tag is unsupported") - if isinstance(event, ScalarEvent) and ( - len(event.value.encode("utf-8")) > MAX_SCANNER_CONFIG_SCALAR_BYTES - ): - raise ValueError("configuration scalar is too large") - if isinstance(event, CollectionStartEvent): - depth += 1 - if depth > MAX_SCANNER_CONFIG_NESTING: - raise ValueError("configuration nesting is too deep") - elif isinstance(event, CollectionEndEvent): - depth -= 1 - - -def _validate_yaml_node(node: Node, *, depth: int, count: list[int]) -> None: - count[0] += 1 - if depth > MAX_SCANNER_CONFIG_NESTING or count[0] > MAX_SCANNER_CONFIG_NODES: - raise ValueError("configuration structure is too large") - if isinstance(node, ScalarNode): - if len(node.value.encode("utf-8")) > MAX_SCANNER_CONFIG_SCALAR_BYTES: - raise ValueError("configuration scalar is too large") - return - if isinstance(node, SequenceNode): - for child in node.value: - _validate_yaml_node(child, depth=depth + 1, count=count) - return - if isinstance(node, MappingNode): - keys: set[tuple[str, str]] = set() - for key, value in node.value: - if not isinstance(key, ScalarNode): - raise ValueError("mapping keys must be scalars") - identity = (key.tag, key.value) - if identity in keys: - raise ValueError("mapping keys must be unique") - keys.add(identity) - _validate_yaml_node(key, depth=depth + 1, count=count) - _validate_yaml_node(value, depth=depth + 1, count=count) - return - raise ValueError("YAML node is unsupported") - - -def _parse_catalogs(value: object) -> dict[str | None, tuple[RegexEntity, ...]]: - if isinstance(value, list): - return {None: _parse_entities(value)} - if not _is_object_mapping(value) or set(value) != {"profiles"}: - raise ValueError("configuration shape is invalid") - profiles = value["profiles"] - if not isinstance(profiles, Mapping) or not profiles: - raise ValueError("profiles must be a non-empty mapping") - if len(profiles) > MAX_REGEX_PROFILES: - raise ValueError("too many profiles") - catalogs: dict[str | None, tuple[RegexEntity, ...]] = {} - for name, entities in profiles.items(): - if not isinstance(name, str): - raise ValueError("profile name must be a string") - catalogs[_validate_name(name)] = _parse_entities(entities) - _validate_document_totals(catalogs) - return catalogs - - -def _is_object_mapping(value: object) -> TypeIs[Mapping[object, object]]: - return isinstance(value, Mapping) - - -def _parse_entities(value: object) -> tuple[RegexEntity, ...]: - if not isinstance(value, list) or not value: - raise ValueError("entity catalog must be a non-empty list") - entities = tuple(RegexEntity.model_validate(item) for item in value) - names = [entity.name for entity in entities] - if len(names) != len(set(names)): - raise ValueError("entity names must be unique") - if len(entities) > MAX_REGEX_ENTITIES_PER_PROFILE: - raise ValueError("too many entities") - pattern_count = sum(len(entity.patterns) for entity in entities) - if pattern_count > MAX_REGEX_PATTERNS_PER_PROFILE: - raise ValueError("too many patterns") - return entities - - -def _validate_document_totals( - catalogs: Mapping[str | None, tuple[RegexEntity, ...]], -) -> None: - if sum(len(entities) for entities in catalogs.values()) > MAX_REGEX_ENTITIES_TOTAL: - raise ValueError("too many entities in document") - if ( - sum( - len(entity.patterns) - for entities in catalogs.values() - for entity in entities - ) - > MAX_REGEX_PATTERNS_TOTAL - ): - raise ValueError("too many patterns in document") - - -def _select_catalog( - catalogs: Mapping[str | None, tuple[RegexEntity, ...]], profile: str | None -) -> tuple[RegexEntity, ...]: - if None in catalogs: - if profile is not None: - raise ValueError("profile is invalid for single-profile configuration") - return catalogs[None] - if profile is None: - raise ValueError("profile is required") - try: - return catalogs[_validate_name(profile)] - except KeyError: - raise ValueError("profile does not exist") from None - - -_NAME_PATTERN = re.compile(r"[A-Za-z_][A-Za-z0-9_-]*\Z") diff --git a/projects/privacy-guard/src/privacy_guard/service/__init__.py b/projects/privacy-guard/src/privacy_guard/service/__init__.py index 582585e..802d49e 100644 --- a/projects/privacy-guard/src/privacy_guard/service/__init__.py +++ b/projects/privacy-guard/src/privacy_guard/service/__init__.py @@ -1,9 +1,6 @@ """gRPC transport and servicer for the Privacy Guard middleware.""" from privacy_guard.service.server import MiddlewareServer -from privacy_guard.service.servicer import ( - PrivacyGuardMiddleware, - RequestProcessorLike, -) +from privacy_guard.service.servicer import PrivacyGuardMiddleware -__all__ = ["MiddlewareServer", "PrivacyGuardMiddleware", "RequestProcessorLike"] +__all__ = ["MiddlewareServer", "PrivacyGuardMiddleware"] diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 1ff7a65..7eb84a4 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -1,15 +1,10 @@ -"""gRPC server process that hosts Privacy Guard at a configured endpoint. - -This is pure transport lifecycle -- it has no counterpart in an in-process -(built-in) middleware. It exists only because Privacy Guard runs out-of-process -and the supervisor reaches it over gRPC. The default endpoint is loopback. -""" +"""Loopback gRPC server and configuration-discovery CLI.""" from __future__ import annotations import asyncio +import json import logging -from pathlib import Path from typing import Annotated import grpc @@ -17,46 +12,48 @@ from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES +from privacy_guard.engine_registry import EngineRegistry +from privacy_guard.engines import RegexEngine from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.processor import RequestProcessor -from privacy_guard.scanners import RegexScanner, Scanner, ScannerConfig from privacy_guard.service.servicer import PrivacyGuardMiddleware app = typer.Typer( name="privacy-guard", - help="Run Privacy Guard with a built-in scanner.", + help="Run Privacy Guard or inspect its registered entity-processing engines.", no_args_is_help=True, add_completion=False, ) +def create_default_registry() -> EngineRegistry: + """Build the operator registry shipped by the base package.""" + registry = EngineRegistry() + registry.register(RegexEngine) + registry.finalize() + return registry + + class MiddlewareServer: - """High-level server that wires a scanner into the Privacy Guard service.""" + """High-level server owning registry, middleware, gRPC, and shutdown.""" def __init__( self, *, - scanner: Scanner[ScannerConfig], + registry: EngineRegistry | None = None, log_request_content: bool = False, ) -> None: self._servicer = PrivacyGuardMiddleware( - RequestProcessor( - [scanner], - log_request_content=log_request_content, - ) + registry or create_default_registry(), + log_request_content=log_request_content, ) def serve(self, listen: str = "127.0.0.1:50051") -> None: - """Serve until termination using a managed synchronous entry point.""" + """Serve until termination through a managed synchronous entry point.""" asyncio.run(serve(self._servicer, listen)) def create_server(servicer: PrivacyGuardMiddleware) -> grpc.aio.Server: - """Build an unstarted gRPC server with the servicer mounted (no port bound). - - The receive limit reserves bounded space around the advertised body maximum - for the protobuf envelope; the servicer enforces the body limit itself. - """ + """Build an unstarted bounded gRPC server.""" server = grpc.aio.server( maximum_concurrent_rpcs=MAX_CONCURRENT_RPCS, options=(("grpc.max_receive_message_length", MAX_RECEIVE_MESSAGE_BYTES),), @@ -69,7 +66,7 @@ async def serve( servicer: PrivacyGuardMiddleware, listen: str = "127.0.0.1:50051", ) -> None: - """Bind ``listen``, start the server, and serve until terminated.""" + """Bind, start, and serve until termination.""" server = create_server(servicer) try: try: @@ -90,23 +87,16 @@ def main( context: typer.Context, debug: Annotated[ bool, - typer.Option( - "--debug", - help="Log content-safe request processing and phase timings.", - ), + typer.Option(help="Enable content-safe processing diagnostics."), ] = False, debug_log_content: Annotated[ bool, typer.Option( - "--debug-log-content", - help=( - "DANGEROUS: log complete request bodies before and after " - "Privacy Guard processing." - ), + help="DANGEROUS: log complete input and processed text.", ), ] = False, ) -> None: - """Select one of Privacy Guard's built-in scanners.""" + """Configure Privacy Guard command logging.""" logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s", @@ -118,49 +108,44 @@ def main( if debug_log_content: _LOGGER.warning( "privacy_guard_request_content_logging_enabled " - "complete_request_bodies_may_contain_secrets" + "complete_request_text_may_contain_secrets" ) -@app.command("regex") -def run_regex( +@app.command("serve") +def run_server( context: typer.Context, - config: Annotated[ - Path, - typer.Option(help="Path to the RegexScanner configuration."), - ], listen: Annotated[ str, typer.Option(help="Address on which the middleware server listens."), ] = "127.0.0.1:50051", - profile: Annotated[ - str | None, - typer.Option(help="Profile required for a multi-profile configuration."), - ] = None, - scanner_name: Annotated[ - str, - typer.Option(help="Scanner identity attached to findings."), - ] = "regex", ) -> None: - """Run Privacy Guard with the built-in RegexScanner.""" - try: - scanner = RegexScanner.from_yaml( - config, - profile, - scanner_name=scanner_name, + """Run the service; entity behavior comes from prepared policy config.""" + _LOGGER.info("privacy_guard_server_starting listen=%s", listen) + MiddlewareServer(log_request_content=context.obj is True).serve(listen) + + +@app.command("schema") +def show_schema() -> None: + """Print the exact finalized policy JSON Schema.""" + typer.echo( + json.dumps( + create_default_registry().configuration_json_schema(), + indent=2, + ensure_ascii=False, + sort_keys=True, ) - _LOGGER.info( - "privacy_guard_server_starting scanner=%s listen=%s", - scanner.scanner_name, - listen, + ) + + +@app.command("engines") +def show_engines() -> None: + """List installed engines and their maximum processing strategy.""" + for description in create_default_registry().describe_engines(): + typer.echo( + f"{description.engine}\t{description.supported_strategy.value}" + f"\t{description.description}" ) - MiddlewareServer( - scanner=scanner, - log_request_content=context.obj is True, - ).serve(listen) - except PrivacyGuardError as error: - typer.echo(str(error), err=True) - raise typer.Exit(code=1) from None _LOGGER = logging.getLogger(__name__) @@ -168,3 +153,12 @@ def run_regex( if __name__ == "__main__": app() + + +__all__ = [ + "MiddlewareServer", + "app", + "create_default_registry", + "create_server", + "serve", +] diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index b6f9012..21858b6 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -1,4 +1,4 @@ -"""Thin gRPC adapter for the proto-free Privacy Guard processor.""" +"""gRPC boundary for cached entity-processing configurations.""" from __future__ import annotations @@ -6,66 +6,67 @@ import logging import time from collections import OrderedDict -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future, ThreadPoolExecutor +from threading import RLock from typing import Never, Protocol, TypedDict import grpc from google.protobuf import json_format from google.protobuf.message import Message -from pydantic import ValidationError from typing_extensions import override from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc -from privacy_guard.config import PolicyConfig +from privacy_guard.config import FinalizedPrivacyGuardConfig, configuration_fingerprint from privacy_guard.constants import ( BLOCK_REASON, BLOCK_REASON_CODE, LIMIT_REASON, LIMIT_REASON_CODE, MAX_BODY_BYTES, - MAX_CONCURRENT_SCANS, + MAX_CONCURRENT_PROCESSING, MAX_PROTO_FINDING_BYTES, MAX_PROTO_FINDING_GROUPS, - PATTERN_NAME_METADATA_KEY, REASON_CODE_PATTERN, SERVICE_NAME, SERVICE_VERSION, - UINT32_MAX, ) +from privacy_guard.engine_registry import EngineRegistry +from privacy_guard.engines import ConfidenceLevel from privacy_guard.errors import ErrorCode, ErrorKind, PrivacyGuardError -from privacy_guard.payloads import ( - InterceptedRequest, - ProcessingDecision, - ProcessingResult, +from privacy_guard.request_processor import ( + EntityDetectionSummary, + RequestDecision, + RequestProcessingResult, + RequestProcessor, ) -from privacy_guard.scanners import RequestBodyFinding - - -class RequestProcessorLike(Protocol): - """Narrow structural seam for injecting request processors into the service.""" - - def validate_policy_config(self, policy_config: PolicyConfig) -> None: ... - - def process(self, request: InterceptedRequest) -> ProcessingResult: ... class PrivacyGuardMiddleware(pb2_grpc.SupervisorMiddlewareServicer): - """Translate protobuf requests and responses at the transport boundary.""" - - def __init__(self, processor: RequestProcessorLike) -> None: - self._processor = processor - self._scan_slots = asyncio.Semaphore(MAX_CONCURRENT_SCANS) - self._scan_executor = ThreadPoolExecutor( - max_workers=MAX_CONCURRENT_SCANS, - thread_name_prefix="privacy-guard-scan", + """Validate, prepare, resolve, and run Privacy Guard policies.""" + + def __init__( + self, + registry: EngineRegistry, + *, + log_request_content: bool = False, + ) -> None: + if not registry.is_finalized: + registry.finalize() + self._registry = registry + self._processors = _RequestProcessorCache( + registry, + log_request_content=log_request_content, + ) + self._processing_slots = asyncio.Semaphore(MAX_CONCURRENT_PROCESSING) + self._processing_executor = ThreadPoolExecutor( + max_workers=MAX_CONCURRENT_PROCESSING, + thread_name_prefix="privacy-guard-processing", ) async def close(self) -> None: - """Stop accepting worker jobs and wait for active synchronous scans.""" - await asyncio.to_thread( - self._scan_executor.shutdown, wait=True, cancel_futures=True - ) + """Wait for in-flight synchronous engines during shutdown.""" + self._processing_executor.shutdown(wait=True, cancel_futures=True) @override async def Describe( @@ -73,7 +74,7 @@ async def Describe( request: object, context: grpc.aio.ServicerContext[object, pb2.MiddlewareManifest], ) -> pb2.MiddlewareManifest: - """Advertise the HTTP pre-credentials binding and maximum body size.""" + """Advertise the binding and its finalized policy schema.""" return self._describe() @override @@ -81,10 +82,11 @@ async def ValidateConfig( self, request: pb2.ValidateConfigRequest, context: grpc.aio.ServicerContext[ - pb2.ValidateConfigRequest, pb2.ValidateConfigResponse + pb2.ValidateConfigRequest, + pb2.ValidateConfigResponse, ], ) -> pb2.ValidateConfigResponse: - """Parse and validate policy configuration without processing a body.""" + """Validate expanded configuration without preparing runtime state.""" return self._validate_config(request) @override @@ -92,14 +94,14 @@ async def EvaluateHttpRequest( self, request: pb2.HttpRequestEvaluation, context: grpc.aio.ServicerContext[ - pb2.HttpRequestEvaluation, pb2.HttpRequestResult + pb2.HttpRequestEvaluation, + pb2.HttpRequestResult, ], ) -> pb2.HttpRequestResult: - """Validate transport input, delegate valid requests, and map the result.""" + """Resolve the prepared config, decode one text, and process it.""" return await self._evaluate_rpc(request, context) def _describe(self) -> pb2.MiddlewareManifest: - """Build the transport manifest without requiring an RPC context.""" return pb2.MiddlewareManifest( name=SERVICE_NAME, service_version=SERVICE_VERSION, @@ -113,12 +115,11 @@ def _describe(self) -> pb2.MiddlewareManifest: ) def _validate_config( - self, request: pb2.ValidateConfigRequest + self, + request: pb2.ValidateConfigRequest, ) -> pb2.ValidateConfigResponse: - """Validate config through the narrow processor seam.""" try: - policy_config = _policy_from_proto(request.config) - self._processor.validate_policy_config(policy_config) + self._registry.validate_config(_mapping_from_proto(request.config)) except PrivacyGuardError as error: return pb2.ValidateConfigResponse(valid=False, reason=str(error)) except Exception: @@ -127,16 +128,12 @@ def _validate_config( return pb2.ValidateConfigResponse(valid=True) async def _evaluate_rpc( - self, request: pb2.HttpRequestEvaluation, context: _AbortContext + self, + request: pb2.HttpRequestEvaluation, + context: _AbortContext, ) -> pb2.HttpRequestResult: - """Run one evaluation using only the context's abort capability.""" started = time.monotonic() request_id = request.context.request_id - _LOGGER.debug( - "privacy_guard_evaluation_received request_id=%s body_bytes=%d", - request_id, - len(request.body), - ) failure: PrivacyGuardError | None = None action = "error" finding_count = 0 @@ -167,7 +164,6 @@ async def _evaluate_rpc( log_extra["error_code"] or "none", extra=log_extra, ) - status = ( grpc.StatusCode.INVALID_ARGUMENT if failure.kind is ErrorKind.INVALID_INPUT @@ -176,32 +172,99 @@ async def _evaluate_rpc( await context.abort(status, str(failure)) async def _evaluate_http_request( - self, request: pb2.HttpRequestEvaluation + self, + request: pb2.HttpRequestEvaluation, ) -> pb2.HttpRequestResult: - """Evaluate a request through the context-free application seam.""" if request.phase != pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS: raise PrivacyGuardError(ErrorCode.REQUEST_PHASE_INVALID) if len(request.body) > MAX_BODY_BYTES: raise PrivacyGuardError(ErrorCode.REQUEST_BODY_TOO_LARGE) + processor = self._processors.resolve(_mapping_from_proto(request.config)) + if not request.body: + return pb2.HttpRequestResult(decision=pb2.DECISION_ALLOW) + try: + text = request.body.decode("utf-8", errors="strict") + except UnicodeDecodeError: + raise PrivacyGuardError(ErrorCode.BODY_ENCODING_INVALID) from None - intercepted = _request_from_proto(request) - # Cancellation cannot stop a Python worker already running. The done - # callback keeps its slot occupied until it really finishes, while the - # RPC can still observe cancellation immediately. - await self._scan_slots.acquire() + await self._processing_slots.acquire() try: - future = asyncio.get_running_loop().run_in_executor( - self._scan_executor, self._processor.process, intercepted + worker = self._processing_executor.submit( + processor.process, + text, ) + future = asyncio.create_task(_await_worker(worker)) except BaseException: - self._scan_slots.release() + self._processing_slots.release() raise - future.add_done_callback(lambda _: self._scan_slots.release()) + future.add_done_callback(lambda _: self._processing_slots.release()) result = await asyncio.shield(future) return _result_to_proto(result) -_LOGGER = logging.getLogger(__name__) +class _RequestProcessorCache: + """Bounded, recoverable cache keyed by canonical expanded configuration.""" + + def __init__( + self, + registry: EngineRegistry, + *, + log_request_content: bool, + ) -> None: + self._registry = registry + self._log_request_content = log_request_content + self._processors: OrderedDict[str, RequestProcessor] = OrderedDict() + self._lock = RLock() + + def resolve(self, values: object) -> RequestProcessor: + """Return the cached or newly prepared processor for expanded config.""" + _, processor = self._prepare(values) + return processor + + def _prepare(self, values: object) -> tuple[str, RequestProcessor]: + config = self._registry.validate_config(values) + fingerprint = configuration_fingerprint(config) + with self._lock: + cached = self._processors.get(fingerprint) + if cached is not None: + self._processors.move_to_end(fingerprint) + return fingerprint, cached + processor = self._build_processor(config) + with self._lock: + self._processors[fingerprint] = processor + self._processors.move_to_end(fingerprint) + while len(self._processors) > _MAX_CACHED_PROCESSORS: + self._processors.popitem(last=False) + return fingerprint, processor + + def _build_processor( + self, + config: FinalizedPrivacyGuardConfig, + ) -> RequestProcessor: + stages = tuple( + ( + stage.diagnostic_name(index), + self._registry.create_engine(stage.config), + ) + for index, stage in enumerate( + config.entity_processing.stages, + start=1, + ) + ) + return RequestProcessor( + config, + stages, + log_request_content=self._log_request_content, + ) + + +async def _await_worker( + worker: Future[RequestProcessingResult], +) -> RequestProcessingResult: + """Bridge a worker without relying on broken cross-thread loop wakeups.""" + while not worker.done(): + await asyncio.sleep(0.001) + return worker.result() class _AbortContext(Protocol): @@ -224,7 +287,6 @@ def _evaluation_log_extra( finding_count: int, failure: PrivacyGuardError | None, ) -> _EvaluationLogExtra: - """Build the typed operational fields attached to an evaluation log.""" return { "request_id": request_id, "duration_ms": round((time.monotonic() - started) * 1000, 3), @@ -234,55 +296,38 @@ def _evaluation_log_extra( } -def _policy_from_proto(config: Message) -> PolicyConfig: +def _mapping_from_proto(config: Message) -> dict[str, object]: try: - values = json_format.MessageToDict(config) - return PolicyConfig.from_mapping(values) + return json_format.MessageToDict(config) except Exception: raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None -def _request_from_proto(request: pb2.HttpRequestEvaluation) -> InterceptedRequest: - policy_config = _policy_from_proto(request.config) +def _result_to_proto(result: RequestProcessingResult) -> pb2.HttpRequestResult: try: - return InterceptedRequest( - raw_body=request.body, - content_type=next( - ( - header.value - for header in request.headers - if header.name == "content-type" - ), - "", - ), - request_id=request.context.request_id, - policy_config=policy_config, - ) - except ValidationError: - raise PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) from None - - -def _result_to_proto(result: ProcessingResult) -> pb2.HttpRequestResult: - if ( - result.replacement_body is not None - and len(result.replacement_body) > MAX_BODY_BYTES - ): - return _limit_deny() - try: - findings = _aggregate_findings(result.findings) + findings = [ + _detection_to_proto(detection) for detection in result.detection_summaries + ] except PrivacyGuardError as error: if error.code is ErrorCode.RESULT_LIMIT_EXCEEDED: return _limit_deny() raise - if result.decision is ProcessingDecision.ALLOW: - has_body = result.replacement_body is not None + if len(findings) > MAX_PROTO_FINDING_GROUPS: + return _limit_deny() + if result.decision is RequestDecision.ALLOW: + replacement = result.replacement_text + replacement_body = ( + replacement.encode("utf-8") if replacement is not None else b"" + ) + if len(replacement_body) > MAX_BODY_BYTES: + return _limit_deny() return pb2.HttpRequestResult( decision=pb2.DECISION_ALLOW, - body=result.replacement_body if has_body else b"", - has_body=has_body, + body=replacement_body, + has_body=replacement is not None, findings=findings, ) - if result.decision is ProcessingDecision.DENY: + if result.decision is RequestDecision.DENY: reason_code = result.reason_code or BLOCK_REASON_CODE if REASON_CODE_PATTERN.fullmatch(reason_code) is None: return _limit_deny() @@ -290,51 +335,40 @@ def _result_to_proto(result: ProcessingResult) -> pb2.HttpRequestResult: decision=pb2.DECISION_DENY, reason=LIMIT_REASON if reason_code == LIMIT_REASON_CODE else BLOCK_REASON, reason_code=reason_code, - has_body=False, findings=findings, ) raise PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) +def _detection_to_proto(detection: EntityDetectionSummary) -> pb2.Finding: + confidence = detection.confidence + if isinstance(confidence, ConfidenceLevel): + confidence_text = confidence.value + elif confidence is None: + confidence_text = "" + else: + confidence_text = format(confidence, ".12g") + result = pb2.Finding( + type="detected_entity", + label=f"{detection.entity} ({detection.source_stage})", + confidence=confidence_text, + count=detection.count, + ) + if result.ByteSize() > MAX_PROTO_FINDING_BYTES: + raise PrivacyGuardError(ErrorCode.RESULT_LIMIT_EXCEEDED) + return result + + def _limit_deny() -> pb2.HttpRequestResult: return pb2.HttpRequestResult( decision=pb2.DECISION_DENY, reason=LIMIT_REASON, reason_code=LIMIT_REASON_CODE, - has_body=False, ) -def _aggregate_findings(findings: tuple[RequestBodyFinding, ...]) -> list[pb2.Finding]: - groups: OrderedDict[tuple[str, str, str | None, str], int] = OrderedDict() - for request_body_finding in findings: - finding = request_body_finding.finding - pattern_name = ( - None - if finding.metadata is None - else finding.metadata.get(PATTERN_NAME_METADATA_KEY) - ) - key = ( - finding.scanner_name, - finding.entity, - pattern_name, - finding.confidence.value, - ) - groups[key] = groups.get(key, 0) + 1 - if len(groups) > MAX_PROTO_FINDING_GROUPS: - raise PrivacyGuardError(ErrorCode.RESULT_LIMIT_EXCEEDED) - aggregated = [ - pb2.Finding( - type=scanner_name, - label=entity if pattern_name is None else f"{entity}/{pattern_name}", - confidence=confidence, - count=min(count, UINT32_MAX), - ) - for (scanner_name, entity, pattern_name, confidence), count in groups.items() - ] - if any(finding.ByteSize() > MAX_PROTO_FINDING_BYTES for finding in aggregated): - raise PrivacyGuardError(ErrorCode.RESULT_LIMIT_EXCEEDED) - return aggregated +_LOGGER = logging.getLogger(__name__) +_MAX_CACHED_PROCESSORS = 128 -__all__ = ["PrivacyGuardMiddleware", "RequestProcessorLike"] +__all__ = ["PrivacyGuardMiddleware"] diff --git a/projects/privacy-guard/src/privacy_guard/string_validators.py b/projects/privacy-guard/src/privacy_guard/string_validators.py new file mode 100644 index 0000000..9c12506 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/string_validators.py @@ -0,0 +1,44 @@ +"""Reusable string validators and the field types built from them.""" + +from typing import Annotated + +from pydantic import BeforeValidator + +from privacy_guard.constants import MAX_DIAGNOSTIC_TEXT_BYTES + + +def validate_scalar_string(value: object) -> str: + """Validate and return a string containing only Unicode scalar values.""" + if not isinstance(value, str): + raise ValueError("value must be a string") + if any("\ud800" <= character <= "\udfff" for character in value): + raise ValueError("string must contain valid Unicode scalar values") + return value + + +def validate_bounded_metadata_string(value: object) -> str: + """Validate and return a non-empty, size-bounded metadata string.""" + validated = validate_scalar_string(value) + if not validated: + raise ValueError("string must not be empty") + if ( + len(validated) > MAX_DIAGNOSTIC_TEXT_BYTES + or len(validated.encode("utf-8")) > MAX_DIAGNOSTIC_TEXT_BYTES + ): + raise ValueError("metadata exceeds the UTF-8 byte limit") + return validated + + +ScalarString = Annotated[str, BeforeValidator(validate_scalar_string)] +BoundedMetadataString = Annotated[ + str, + BeforeValidator(validate_bounded_metadata_string), +] + + +__all__ = [ + "BoundedMetadataString", + "ScalarString", + "validate_bounded_metadata_string", + "validate_scalar_string", +] diff --git a/projects/privacy-guard/src/privacy_guard/timeout.py b/projects/privacy-guard/src/privacy_guard/timeout.py new file mode 100644 index 0000000..1a30d32 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/timeout.py @@ -0,0 +1,49 @@ +"""A shared monotonic timeout for one entity-processing run.""" + +from __future__ import annotations + +import math +from time import monotonic +from typing import Self + +from pydantic import Field + +from privacy_guard.base import StrictDomainModel +from privacy_guard.constants import MAX_TIMEOUT_SECONDS + + +class TimeoutExpired(Exception): + """Signal that the shared entity-processing timeout has expired.""" + + +class Timeout(StrictDomainModel): + """An immutable monotonic deadline shared across processing stages.""" + + deadline: float = Field(allow_inf_nan=False) + + @classmethod + def from_seconds(cls, seconds: float) -> Self: + """Create a timeout from a finite, positive bounded duration.""" + if ( + isinstance(seconds, bool) + or not isinstance(seconds, int | float) + or not math.isfinite(seconds) + or seconds <= 0 + or seconds > MAX_TIMEOUT_SECONDS + ): + raise ValueError("timeout must be finite, positive, and within the limit") + return cls(deadline=monotonic() + seconds) + + def remaining_seconds(self) -> float: + """Return the positive duration remaining or raise ``TimeoutExpired``.""" + remaining = self.deadline - monotonic() + if remaining <= 0: + raise TimeoutExpired + return remaining + + def raise_if_expired(self) -> None: + """Raise ``TimeoutExpired`` when no time remains.""" + self.remaining_seconds() + + +__all__ = ["Timeout", "TimeoutExpired"] diff --git a/projects/privacy-guard/src/privacy_guard/validation.py b/projects/privacy-guard/src/privacy_guard/validation.py deleted file mode 100644 index 37e0289..0000000 --- a/projects/privacy-guard/src/privacy_guard/validation.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Shared strict, content-safe validation primitives.""" - -from __future__ import annotations - -from typing import Annotated - -from pydantic import BaseModel, BeforeValidator, ConfigDict - -from privacy_guard.constants import MAX_SCANNER_METADATA_BYTES - - -def parse_scalar_string(value: object) -> str: - """Return a string containing only valid Unicode scalar values.""" - if not isinstance(value, str): - raise ValueError("value must be a string") - if any("\ud800" <= character <= "\udfff" for character in value): - raise ValueError("string must contain valid Unicode scalar values") - return value - - -def parse_non_empty_scalar_string(value: object) -> str: - """Return a non-empty Unicode scalar string.""" - parsed = parse_scalar_string(value) - if not parsed: - raise ValueError("string must not be empty") - return parsed - - -def parse_bounded_metadata_string(value: object) -> str: - """Return metadata whose UTF-8 representation is within the package limit.""" - parsed = parse_non_empty_scalar_string(value) - if ( - len(parsed) > MAX_SCANNER_METADATA_BYTES - or len(parsed.encode("utf-8")) > MAX_SCANNER_METADATA_BYTES - ): - raise ValueError("metadata exceeds the UTF-8 byte limit") - return parsed - - -ScalarString = Annotated[str, BeforeValidator(parse_scalar_string)] -NonEmptyScalarString = Annotated[str, BeforeValidator(parse_non_empty_scalar_string)] -BoundedMetadataString = Annotated[str, BeforeValidator(parse_bounded_metadata_string)] - - -class StrictDomainModel(BaseModel): - """Base for immutable domain values parsed without implicit coercion.""" - - model_config = ConfigDict( - strict=True, - frozen=True, - extra="forbid", - hide_input_in_errors=True, - validate_default=True, - ) - - -class StrictSensitiveModel(StrictDomainModel): - """Semantic base for strict models containing repr-hidden sensitive fields.""" diff --git a/projects/privacy-guard/tests/engines/test_base.py b/projects/privacy-guard/tests/engines/test_base.py new file mode 100644 index 0000000..0f6bee9 --- /dev/null +++ b/projects/privacy-guard/tests/engines/test_base.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +from dataclasses import dataclass +from time import monotonic +from typing import Literal + +import pytest +from pydantic import ValidationError + +from privacy_guard.base import StrictDomainModel +from privacy_guard.engines import ( + ConfidenceLevel, + EngineConfig, + EngineContractError, + EntityDetection, + EntityProcessingEngine, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.timeout import Timeout, TimeoutExpired + + +class _Replacement(StrictDomainModel): + strategy: Literal["token"] = "token" + + +class _Config(EngineConfig[_Replacement]): + engine: Literal["test"] = "test" + + +@dataclass(frozen=True) +class _Resources: + prefix: str + + +class _CustomEngine(EntityProcessingEngine[_Config, _Resources]): + supported_strategy = EntityProcessingStrategy.REPLACE + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + detection = EntityDetection( + entity="token", + start=0, + end=len(text), + confidence=0.75, + metadata={"provider": "custom"}, + ) + output = ( + f"{self.resources.prefix}token" + if strategy is EntityProcessingStrategy.REPLACE + else text + ) + return TextProcessingResult(text=output, detections=(detection,)) + + +def test_custom_engine_infers_types_and_needs_no_custom_init() -> None: + config = _Config(replacement=_Replacement()) + resources = _Resources(prefix="[") + + engine = _CustomEngine(config, resources) + + assert _CustomEngine.get_config_type() is _Config + assert _CustomEngine.get_resources_type() is _Resources + assert engine.config is config + assert engine.resources is resources + assert ( + engine.run( + "secret", + strategy=EntityProcessingStrategy.DETECT, + timeout=Timeout.from_seconds(1), + ).text + == "secret" + ) + assert ( + engine.run( + "secret", + strategy=EntityProcessingStrategy.REPLACE, + timeout=Timeout.from_seconds(1), + ).text + == "[token" + ) + + +def test_detection_confidence_and_metadata_are_strict_bounded_values() -> None: + categorical = EntityDetection.model_validate( + { + "entity": "email", + "start": 0, + "end": 1, + "confidence": "high", + "metadata": {"pattern": "email.patterns[0]"}, + } + ) + numeric = EntityDetection( + entity="email", + start=0, + end=1, + confidence=0.25, + ) + + assert categorical.confidence is ConfidenceLevel.HIGH + assert numeric.confidence == 0.25 + assert type(categorical.metadata).__name__ == "mappingproxy" + with pytest.raises(ValidationError): + EntityDetection( + entity="email", + start=0, + end=1, + confidence=1.01, + ) + + +class _DetectOnlyEngine(EntityProcessingEngine[_Config, None]): + supported_strategy = EntityProcessingStrategy.DETECT + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + return TextProcessingResult(text=text, detections=()) + + +def test_detect_only_engine_rejects_replacement_before_running() -> None: + engine = _DetectOnlyEngine(_Config(), None) + + with pytest.raises(EngineContractError): + engine.run( + "text", + strategy=EntityProcessingStrategy.REPLACE, + timeout=Timeout.from_seconds(1), + ) + + +class _MutatingDetectEngine(EntityProcessingEngine[_Config, None]): + supported_strategy = EntityProcessingStrategy.REPLACE + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + return TextProcessingResult( + text="changed", + detections=(EntityDetection(entity="token", start=0, end=len(text)),), + ) + + +def test_detection_strategy_rejects_mutated_text() -> None: + engine = _MutatingDetectEngine(_Config(), None) + + with pytest.raises(EngineContractError): + engine.run( + "text", + strategy=EntityProcessingStrategy.DETECT, + timeout=Timeout.from_seconds(1), + ) + + +class _InvalidSpanEngine(EntityProcessingEngine[_Config, None]): + supported_strategy = EntityProcessingStrategy.DETECT + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + return TextProcessingResult( + text=text, + detections=(EntityDetection(entity="token", start=0, end=len(text) + 1),), + ) + + +def test_engine_boundary_rejects_spans_outside_stage_input() -> None: + engine = _InvalidSpanEngine(_Config(), None) + + with pytest.raises(EngineContractError): + engine.run( + "text", + strategy=EntityProcessingStrategy.DETECT, + timeout=Timeout.from_seconds(1), + ) + + +@pytest.mark.parametrize("seconds", [True, 0, -1, float("inf"), 31]) +def test_timeout_duration_is_strict_positive_and_bounded( + seconds: bool | int | float, +) -> None: + with pytest.raises(ValueError): + Timeout.from_seconds(seconds) + + +def test_expired_timeout_raises_typed_signal() -> None: + timeout = Timeout(deadline=monotonic() - 1) + + with pytest.raises(TimeoutExpired): + timeout.raise_if_expired() diff --git a/projects/privacy-guard/tests/engines/test_regex.py b/projects/privacy-guard/tests/engines/test_regex.py new file mode 100644 index 0000000..d6866bc --- /dev/null +++ b/projects/privacy-guard/tests/engines/test_regex.py @@ -0,0 +1,286 @@ +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor + +import pytest +from pydantic import ValidationError + +import privacy_guard.engines.regex as regex_module +from privacy_guard.engines import ( + EngineConfigurationError, + EngineLimitExceeded, + EntityProcessingStrategy, + RegexEngine, + RegexEngineConfig, +) +from privacy_guard.timeout import Timeout, TimeoutExpired + + +def _config( + patterns: list[dict[str, object]], + *, + replacement: dict[str, object] | None = None, +) -> RegexEngineConfig: + values: dict[str, object] = { + "engine": "regex", + "pattern_catalog": { + "entities": [ + { + "name": "token", + "patterns": patterns, + } + ] + }, + } + if replacement is not None: + values["replacement"] = replacement + return RegexEngineConfig.model_validate(values) + + +def _run( + config: RegexEngineConfig, + text: str, + strategy: EntityProcessingStrategy = EntityProcessingStrategy.DETECT, +) -> tuple[str, list[tuple[str, int, int, str]]]: + result = RegexEngine(config, None).run( + text, + strategy=strategy, + timeout=Timeout.from_seconds(1), + ) + return result.text, [ + ( + detection.entity, + detection.start, + detection.end, + detection.metadata["pattern"], + ) + for detection in result.detections + ] + + +def test_detects_overlaps_and_orders_matches_deterministically() -> None: + config = _config( + [ + {"name": "pair", "pattern": "aa", "confidence": "high"}, + {"name": "suffix", "pattern": "a$", "confidence": "medium"}, + ] + ) + + output, detections = _run(config, "aaa") + + assert output == "aaa" + assert detections == [ + ("token", 0, 2, "pair"), + ("token", 1, 3, "pair"), + ("token", 2, 3, "suffix"), + ] + + +def test_optional_names_derive_identity_without_affecting_internal_marker() -> None: + config = _config( + [ + {"name": "same-name", "pattern": "x", "confidence": "high"}, + {"name": "same_name", "pattern": "y", "confidence": "high"}, + {"pattern": "z", "confidence": "high"}, + ] + ) + + _, detections = _run(config, "xyz") + + assert [item[3] for item in detections] == [ + "same-name", + "same_name", + "token.patterns[2]", + ] + + +def test_numeric_backreferences_keep_original_group_numbers() -> None: + config = _config([{"pattern": r"(a)\1", "confidence": "high"}]) + + _, detections = _run(config, "aa") + + assert [(item[1], item[2]) for item in detections] == [(0, 2)] + + +def test_explicit_flags_are_supported() -> None: + config = _config( + [ + { + "pattern": "^x.$", + "confidence": "high", + "ignore_case": True, + "multiline": True, + "dot_all": True, + "ascii": True, + } + ] + ) + + _, detections = _run(config, "X\n") + + assert [(item[1], item[2]) for item in detections] == [(0, 2)] + + +@pytest.mark.parametrize( + "pattern", + [ + "", + "x*", + "(?Px)", + "(?i:x)", + ], +) +def test_invalid_patterns_are_rejected_content_safely(pattern: str) -> None: + with pytest.raises(ValidationError) as exception_info: + _config([{"pattern": pattern, "confidence": "high"}]) + + if pattern: + assert pattern not in str(exception_info.value) + + +def test_contextual_zero_width_failure_is_atomic_at_runtime() -> None: + config = _config([{"pattern": "(?=a)", "confidence": "high"}]) + engine = RegexEngine(config, None) + + with pytest.raises(EngineConfigurationError): + engine.run( + "a", + strategy=EntityProcessingStrategy.DETECT, + timeout=Timeout.from_seconds(1), + ) + + +def test_duplicate_supplied_names_are_rejected_but_unnamed_patterns_are_not() -> None: + with pytest.raises(ValidationError): + _config( + [ + {"name": "duplicate", "pattern": "x", "confidence": "high"}, + {"name": "duplicate", "pattern": "y", "confidence": "high"}, + ] + ) + + config = _config( + [ + {"pattern": "x", "confidence": "high"}, + {"pattern": "y", "confidence": "high"}, + ] + ) + assert len(config.pattern_catalog.entities[0].patterns) == 2 + + +def test_replacement_selects_ranked_non_overlapping_winners() -> None: + config = _config( + [ + {"name": "long-low", "pattern": "abc", "confidence": "low"}, + {"name": "short-high", "pattern": "bc", "confidence": "high"}, + ], + replacement={"strategy": "template", "template": "<{entity}>"}, + ) + + output, detections = _run( + config, + "abc", + EntityProcessingStrategy.REPLACE, + ) + + assert output == "a" + assert len(detections) == 2 + + +def test_replacement_requires_an_engine_specific_recipe() -> None: + config = _config([{"pattern": "x", "confidence": "high"}]) + + with pytest.raises(EngineConfigurationError): + _run(config, "x", EntityProcessingStrategy.REPLACE) + + +@pytest.mark.parametrize( + "replacement", + [ + {"strategy": "template", "template": "{unknown}"}, + {"strategy": "template", "template": "{entity.attr}"}, + {"strategy": "template", "template": "{entity!r}"}, + {"strategy": "template", "template": "{entity:>10}"}, + {"strategy": "template", "template": "{"}, + ], +) +def test_template_language_allows_only_literal_text_and_entity( + replacement: dict[str, object], +) -> None: + with pytest.raises(ValidationError): + _config( + [{"pattern": "x", "confidence": "high"}], + replacement=replacement, + ) + + +def test_replacement_size_is_projected_before_rendering( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(regex_module, "MAX_BODY_BYTES", 4) + config = _config( + [{"pattern": "x", "confidence": "high"}], + replacement={"strategy": "template", "template": "[{entity}]"}, + ) + + with pytest.raises(EngineLimitExceeded): + _run(config, "x", EntityProcessingStrategy.REPLACE) + + +def test_pattern_search_has_an_enforceable_timeout() -> None: + config = _config([{"pattern": "(a+)+$", "confidence": "high"}]) + engine = RegexEngine(config, None) + + with pytest.raises(TimeoutExpired): + engine.run( + "a" * 100_000 + "!", + strategy=EntityProcessingStrategy.DETECT, + timeout=Timeout.from_seconds(0.001), + ) + + +def test_patterns_compile_during_validation_and_preparation_not_run( + monkeypatch: pytest.MonkeyPatch, +) -> None: + compile_count = 0 + original_compile = regex_module.regex.compile + + def recording_compile(pattern: str, flags: int = 0) -> object: + nonlocal compile_count + compile_count += 1 + return original_compile(pattern, flags) + + monkeypatch.setattr(regex_module.regex, "compile", recording_compile) + config = _config([{"pattern": "x", "confidence": "high"}]) + engine = RegexEngine(config, None) + prepared_count = compile_count + + engine.run( + "x", + strategy=EntityProcessingStrategy.DETECT, + timeout=Timeout.from_seconds(1), + ) + + assert prepared_count > 0 + assert compile_count == prepared_count + + +def test_regex_engine_is_safe_for_concurrent_runs() -> None: + engine = RegexEngine( + _config([{"pattern": "x", "confidence": "high"}]), + None, + ) + + def run(text: str) -> int: + return len( + engine.run( + text, + strategy=EntityProcessingStrategy.DETECT, + timeout=Timeout.from_seconds(1), + ).detections + ) + + with ThreadPoolExecutor(max_workers=4) as executor: + counts = tuple(executor.map(run, ("x",) * 16)) + + assert counts == (1,) * 16 diff --git a/projects/privacy-guard/tests/examples/test_email_scanner.py b/projects/privacy-guard/tests/examples/test_email_scanner.py deleted file mode 100644 index a9aa6a2..0000000 --- a/projects/privacy-guard/tests/examples/test_email_scanner.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import annotations - -import subprocess -import sys -from pathlib import Path - -EXAMPLE_DIRECTORY = Path(__file__).parents[2] / "examples" / "email-scanner" - - -def test_example_scanner_detects_email_and_server_entry_point_imports() -> None: - probe = """ -from middleware_server import EmailScanner -from privacy_guard.scanners import ScannerConfig - -scanner = EmailScanner( - ScannerConfig(name="example_regex", entity_types=frozenset({"email"})) -) -findings = scanner.scan("safe user@example.com text") -assert len(findings) == 1 -assert findings[0].entity == "email" -assert findings[0].start_offset == 5 -assert findings[0].end_offset == 21 -""" - subprocess.run([sys.executable, "-c", probe], cwd=EXAMPLE_DIRECTORY, check=True) - - -def test_example_configuration_targets_its_local_middleware() -> None: - policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() - gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() - gitignore = (EXAMPLE_DIRECTORY / ".gitignore").read_text() - readme = (EXAMPLE_DIRECTORY / "README.md").read_text() - - assert "middleware: privacy-guard-email-scanner" in policy - assert 'name = "privacy-guard-email-scanner"' in gateway - assert 'grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051"' in gateway - assert "gateway.local.toml" in gitignore - assert "action: redact" in policy - assert "entity_types: [email]" in policy - assert "cd projects/privacy-guard/examples/email-scanner" in readme - assert "uv run python middleware_server.py" in readme - assert "YOUR_HOST_IP=" in readme - assert ( - 'sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml' - ) in readme - assert "grep grpc_endpoint gateway.local.toml" in readme - assert "--listen 0.0.0.0:50051" in readme - assert 'openshell-gateway --config "$PWD/gateway.local.toml"' in readme - assert "openshell gateway add" in readme - assert "https://127.0.0.1:17670" in readme - assert "--name openshell" in readme - assert "--name privacy-guard-email" in readme - assert "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude" in readme - assert "host: claude.ai" in policy - assert "Middleware connect failed" in readme - assert '403 "middleware_failed"' in readme - assert "--tail" not in readme - assert "uv run --project" not in readme diff --git a/projects/privacy-guard/tests/examples/test_regex_scanner.py b/projects/privacy-guard/tests/examples/test_regex_scanner.py deleted file mode 100644 index 9fb8dc4..0000000 --- a/projects/privacy-guard/tests/examples/test_regex_scanner.py +++ /dev/null @@ -1,116 +0,0 @@ -import json -import re -from pathlib import Path - -import yaml - -from privacy_guard.constants import PATTERN_NAME_METADATA_KEY -from privacy_guard.scanners import RegexScanner - -EXAMPLE_DIRECTORY = Path(__file__).parents[2] / "examples" / "regex-scanner" - - -def test_example_scanner_detects_email_and_customer_id() -> None: - scanner = RegexScanner.from_yaml(EXAMPLE_DIRECTORY / "regex-scanner.yaml") - - findings = scanner.scan("Email user@example.com, customer CUST-12345678") - - assert [finding.entity for finding in findings] == ["email", "customer-id"] - pattern_names: list[str] = [] - for finding in findings: - assert finding.metadata is not None - pattern_names.append(finding.metadata[PATTERN_NAME_METADATA_KEY]) - assert pattern_names == [ - "common-email", - "prefixed-customer-id", - ] - - -def test_pi_templates_render_for_an_openai_compatible_endpoint() -> None: - model_template = (EXAMPLE_DIRECTORY / "pi-models.template.json").read_text() - policy_template = (EXAMPLE_DIRECTORY / "policy.pi.template.yaml").read_text() - - models = json.loads( - model_template.replace( - "REPLACE_WITH_MODEL_ENDPOINT", "https://api.example.com/v1" - ).replace("REPLACE_WITH_MODEL_ID", "example/model") - ) - policy = yaml.safe_load( - policy_template.replace("REPLACE_WITH_MODEL_HOST", "api.example.com") - ) - - assert models["providers"]["custom"]["baseUrl"] == "https://api.example.com/v1" - assert models["providers"]["custom"]["models"][0]["id"] == "example/model" - assert policy["network_policies"]["pi"]["endpoints"][0]["host"] == "api.example.com" - assert policy["network_middlewares"]["privacy_guard_redaction"]["endpoints"][ - "include" - ] == ["api.example.com"] - - -def test_example_configuration_and_walkthrough_are_aligned() -> None: - policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() - gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() - gitignore = (EXAMPLE_DIRECTORY / ".gitignore").read_text() - readme = (EXAMPLE_DIRECTORY / "README.md").read_text() - pi_policy = (EXAMPLE_DIRECTORY / "policy.pi.template.yaml").read_text() - pi_models = json.loads((EXAMPLE_DIRECTORY / "pi-models.template.json").read_text()) - pi_provider = pi_models["providers"]["custom"] - - assert "middleware: privacy-guard-regex-scanner" in policy - assert 'name = "privacy-guard-regex-scanner"' in gateway - assert 'grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051"' in gateway - assert "gateway.local.toml" in gitignore - assert "action: redact" in policy - assert "entity_types: [email, customer-id]" in policy - assert "cd projects/privacy-guard/examples/regex-scanner" in readme - assert "privacy-guard regex" in readme - assert "--config regex-scanner.yaml" in readme - assert "YOUR_HOST_IP=" in readme - assert ( - 'sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml' - ) in readme - assert "grep grpc_endpoint gateway.local.toml" in readme - assert "--listen 0.0.0.0:50051" in readme - assert 'openshell-gateway --config "$PWD/gateway.local.toml"' in readme - assert "openshell gateway add" in readme - assert "https://127.0.0.1:17670" in readme - assert "--name openshell" in readme - assert "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude" in readme - assert "--from pi" in readme - assert "PI_MODEL_ENDPOINT=\nPI_MODEL_ID=\nexport PI_MODEL_API_KEY=" in readme - assert "PI_MODEL_HOST=${PI_MODEL_ENDPOINT#*://}" in readme - assert "--provider privacy-guard-model" in readme - assert "--credential PI_MODEL_API_KEY" in readme - assert "--no-git-ignore" in readme - assert "--upload pi-models.local.json:/sandbox/.pi/agent/models.json" in readme - assert "--provider custom" in readme - assert '--model "$PI_MODEL_ID"' in readme - assert "interactive TUI" in readme - assert "/usr/bin/pi" in pi_policy - assert "/usr/local/bin/pi" not in pi_policy - assert "host: pi.dev" in pi_policy - assert "host: REPLACE_WITH_MODEL_HOST" in pi_policy - assert "REPLACE_WITH_MODEL_HOST" not in policy - assert pi_provider["baseUrl"] == "REPLACE_WITH_MODEL_ENDPOINT" - assert pi_provider["api"] == "openai-completions" - assert pi_provider["apiKey"] == "$PI_MODEL_API_KEY" - assert pi_provider["models"] == [ - { - "id": "REPLACE_WITH_MODEL_ID", - "name": "REPLACE_WITH_MODEL_ID", - } - ] - assert "pi-models.local.json" in gitignore - assert "policy.local.yaml" in gitignore - assert "auth.json" in gitignore - assert "Middleware connect failed" in readme - assert '403 "middleware_failed"' in readme - assert "--tail" not in readme - assert "privacy-guard-regex" in readme - sandbox_names = re.findall( - r"openshell sandbox create \\\n\s+--name ([a-z0-9-]+)", readme - ) - assert sandbox_names - assert all(name.startswith("privacy-guard-") for name in sandbox_names) - assert all(len(name) <= 19 for name in sandbox_names) - assert "uv run --project" not in readme diff --git a/projects/privacy-guard/tests/request_body/test_base.py b/projects/privacy-guard/tests/request_body/test_base.py deleted file mode 100644 index c1697c5..0000000 --- a/projects/privacy-guard/tests/request_body/test_base.py +++ /dev/null @@ -1,122 +0,0 @@ -from collections.abc import Mapping -from dataclasses import FrozenInstanceError - -import pytest -from pydantic import ValidationError -from typing_extensions import override - -from privacy_guard.config import PolicyConfig -from privacy_guard.request_body import ( - FormatHandler, - FormatHandlerContractError, - RequestBody, - TextBlock, - parse_normalized_body, -) - - -class OpaqueHandler(FormatHandler): - def __init__(self, format_name: str = "opaque") -> None: - super().__init__(format_name=format_name) - - @override - def _normalize(self, raw_body: bytes, policy_config: PolicyConfig) -> RequestBody: - return RequestBody(text_blocks=(), parsed_value=None, original_bytes=raw_body) - - @override - def _reconstruct( - self, - request_body: RequestBody, - replacements_by_path: Mapping[str, str], - ) -> bytes: - return request_body.original_bytes - - -@pytest.mark.parametrize( - "values", - [ - {"path": 3, "text": "text"}, - {"path": "path", "text": 3}, - {"path": "path", "text": "text", "replaceable": 1}, - {"path": "bad\ud800", "text": "text"}, - {"path": "path", "text": "bad\ud800"}, - {"path": "path", "text": "text", "extra": "forbidden"}, - ], -) -def test_text_block_rejects_invalid_fields(values: dict[str, object]) -> None: - with pytest.raises(ValidationError): - TextBlock.model_validate(values) - - -def test_text_block_is_frozen_and_hides_sensitive_fields() -> None: - sentinel = "sensitive-block-8472" - block = TextBlock(path=f"/{sentinel}", text=sentinel) - - with pytest.raises((FrozenInstanceError, ValidationError)): - setattr(block, "text", "changed") - - assert sentinel not in repr(block) - - -@pytest.mark.parametrize( - "values", - [ - {"text_blocks": [], "parsed_value": None, "original_bytes": b"body"}, - {"text_blocks": (object(),), "parsed_value": None, "original_bytes": b"body"}, - {"text_blocks": (), "parsed_value": None, "original_bytes": bytearray(b"body")}, - { - "text_blocks": (), - "parsed_value": None, - "original_bytes": b"body", - "extra": "forbidden", - }, - ], -) -def test_request_body_rejects_invalid_fields(values: dict[str, object]) -> None: - with pytest.raises(ValidationError): - RequestBody.model_validate(values) - - -def test_request_body_is_frozen_and_hides_sensitive_fields() -> None: - sentinel = "sensitive-body-8472" - body = RequestBody( - text_blocks=(TextBlock(path=f"/{sentinel}", text=sentinel),), - parsed_value={"value": sentinel}, - original_bytes=sentinel.encode(), - ) - - with pytest.raises((FrozenInstanceError, ValidationError)): - setattr(body, "original_bytes", b"changed") - - assert sentinel not in repr(body) - - -def test_parse_normalized_body_reuses_exact_instance() -> None: - body = RequestBody(text_blocks=(), parsed_value=None, original_bytes=b"body") - - assert parse_normalized_body(body) is body - - -@pytest.mark.parametrize( - "result", - [ - object(), - {"text_blocks": (), "parsed_value": None, "original_bytes": b"body"}, - ], -) -def test_parse_normalized_body_rejects_non_models(result: object) -> None: - with pytest.raises(FormatHandlerContractError): - parse_normalized_body(result) - - -@pytest.mark.parametrize("format_name", ["", "bad\ud800"]) -def test_handler_rejects_invalid_format_identity(format_name: str) -> None: - with pytest.raises(FormatHandlerContractError): - OpaqueHandler(format_name) - - -def test_handler_identity_is_read_only() -> None: - handler = OpaqueHandler() - - with pytest.raises(AttributeError): - setattr(handler, "format_name", "changed") diff --git a/projects/privacy-guard/tests/request_body/test_json.py b/projects/privacy-guard/tests/request_body/test_json.py deleted file mode 100644 index 493c3d9..0000000 --- a/projects/privacy-guard/tests/request_body/test_json.py +++ /dev/null @@ -1,580 +0,0 @@ -import json -import sys -from copy import deepcopy -from dataclasses import FrozenInstanceError - -import pytest -from pydantic import ValidationError - -import privacy_guard.request_body.json as json_module -from privacy_guard.config import PolicyConfig -from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.request_body import ( - JsonHandler, - RequestBody, - TextBlock, - select_format_handler, -) - - -def _assert_safe_error( - error: PrivacyGuardError, expected_code: ErrorCode, sentinel: str -) -> None: - assert error.code is expected_code - assert error.__cause__ is None - assert sentinel not in str(error) - assert sentinel not in repr(error) - assert all(sentinel not in str(argument) for argument in error.args) - - -def _json_state_value(request_body: RequestBody) -> object: - assert type(request_body.parsed_value) is json_module._JsonBodyState - return request_body.parsed_value.value - - -def test_body_domain_reprs_hide_sensitive_content() -> None: - sensitive_text = "sensitive-value-8472" - text_block = TextBlock(path="/message", text=sensitive_text) - request_body = RequestBody( - text_blocks=(text_block,), - parsed_value={"message": sensitive_text}, - original_bytes=sensitive_text.encode(), - ) - - assert sensitive_text not in repr(text_block) - assert sensitive_text not in repr(request_body) - - -def test_body_domain_reprs_hide_sensitive_json_pointer_tokens() -> None: - sensitive_key = "secret-person@example.com" - text_block = TextBlock(path=f"/{sensitive_key}", text="safe") - request_body = RequestBody( - text_blocks=(text_block,), parsed_value={}, original_bytes=b"{}" - ) - - assert sensitive_key not in repr(text_block) - assert sensitive_key not in repr(request_body) - - -def test_json_body_state_repr_hides_recursive_value() -> None: - sensitive_text = "distinctive-state-value-8472" - request_body = JsonHandler().normalize( - f'{{"message":"{sensitive_text}"}}'.encode(), PolicyConfig() - ) - - assert type(request_body.parsed_value) is json_module._JsonBodyState - assert sensitive_text not in repr(request_body.parsed_value) - - -def test_normalized_json_state_stores_the_validated_json_value_directly() -> None: - request_body = JsonHandler().normalize( - b'{"items":[{"message":"original"}]}', PolicyConfig() - ) - - assert type(request_body.parsed_value) is json_module._JsonBodyState - assert type(request_body.parsed_value.value) is dict - with pytest.raises(FrozenInstanceError): - setattr(request_body.parsed_value, "value", None) - assert type(request_body.parsed_value.value["items"]) is list - - -@pytest.mark.parametrize( - "invalid_value", - [ - b"bytes-are-not-json-strings", - ("tuples", "are", "not", "arrays"), - {1: "object keys must be strings"}, - {"nested": object()}, - float("inf"), - ], -) -def test_json_value_adapter_rejects_non_json_types_strictly( - invalid_value: object, -) -> None: - with pytest.raises(ValidationError): - json_module._JSON_VALUE_ADAPTER.validate_python(invalid_value, strict=True) - - -@pytest.mark.parametrize( - ("raw_body", "expected_value"), - [ - (b"null", None), - (b"true", True), - (b"42", 42), - (b"3.5", 3.5), - (b'"root"', "root"), - (b"[]", []), - (b"{}", {}), - ( - b'{"nested":[null,false,1,2.5,"text"]}', - {"nested": [None, False, 1, 2.5, "text"]}, - ), - ], -) -def test_normalize_stores_strict_recursive_json_roots( - raw_body: bytes, expected_value: object -) -> None: - request_body = JsonHandler().normalize(raw_body, PolicyConfig()) - - assert type(request_body.parsed_value) is json_module._JsonBodyState - assert _json_state_value(request_body) == expected_value - if not isinstance(expected_value, dict | list): - assert type(_json_state_value(request_body)) is type(expected_value) - - -def test_normalize_flat_object_collects_string_leaves() -> None: - raw_body = b'{"message":"hello","model":"model-a","count":2}' - - request_body = JsonHandler().normalize(raw_body, PolicyConfig()) - - assert request_body.original_bytes is raw_body - assert type(request_body.parsed_value) is json_module._JsonBodyState - assert _json_state_value(request_body) == { - "message": "hello", - "model": "model-a", - "count": 2, - } - assert tuple(block for block in request_body.text_blocks if block.replaceable) == ( - TextBlock(path="/message", text="hello"), - TextBlock(path="/model", text="model-a"), - ) - - -@pytest.mark.parametrize( - ("raw_body", "expected_text_blocks"), - [ - ( - b'{"outer":{"inner":"value"}}', - (TextBlock(path="/outer/inner", text="value"),), - ), - ( - b'["first",["nested",3],{"last":"value"}]', - ( - TextBlock(path="/0", text="first"), - TextBlock(path="/1/0", text="nested"), - TextBlock(path="/2/last", text="value"), - ), - ), - ( - b'[null,true,false,3.5,"only-string"]', - (TextBlock(path="/4", text="only-string"),), - ), - (b'"root string"', (TextBlock(path="", text="root string"),)), - (b"{}", ()), - (b"[]", ()), - ( - b'{"z":"first","a":{"y":"second","x":"third"},"m":"fourth"}', - ( - TextBlock(path="/z", text="first"), - TextBlock(path="/a/y", text="second"), - TextBlock(path="/a/x", text="third"), - TextBlock(path="/m", text="fourth"), - ), - ), - ( - b'{"a/b":"slash","til~de":"tilde","both~/":"both"}', - ( - TextBlock(path="/a~1b", text="slash"), - TextBlock(path="/til~0de", text="tilde"), - TextBlock(path="/both~0~1", text="both"), - ), - ), - ( - '{"greeting":"こんにちは","emoji":"🐍"}'.encode(), - ( - TextBlock(path="/greeting", text="こんにちは"), - TextBlock(path="/emoji", text="🐍"), - ), - ), - ( - b'{"escaped_emoji":"\\ud83d\\udc0d"}', - (TextBlock(path="/escaped_emoji", text="🐍"),), - ), - ], -) -def test_normalize_walks_arbitrary_json_in_stable_depth_first_order( - raw_body: bytes, expected_text_blocks: tuple[TextBlock, ...] -) -> None: - request_body = JsonHandler().normalize(raw_body, PolicyConfig()) - - assert ( - tuple(block for block in request_body.text_blocks if block.replaceable) - == expected_text_blocks - ) - - -def test_normalize_exposes_object_keys_as_nonreplaceable_blocks() -> None: - blocks = ( - JsonHandler() - .normalize(b'{"user@example.com":"safe"}', PolicyConfig()) - .text_blocks - ) - - assert blocks == ( - TextBlock( - path="#key:/user@example.com", - text="user@example.com", - replaceable=False, - ), - TextBlock(path="/user@example.com", text="safe"), - ) - - -def test_normalize_rejects_invalid_utf8() -> None: - sensitive_text = "distinctive-sensitive-utf8-8472" - - with pytest.raises(PrivacyGuardError) as exception_info: - JsonHandler().normalize( - f'"{sensitive_text}'.encode() + b'\xff"', PolicyConfig() - ) - - _assert_safe_error( - exception_info.value, ErrorCode.BODY_ENCODING_INVALID, sensitive_text - ) - - -def test_normalize_rejects_invalid_json() -> None: - sensitive_text = "distinctive-sensitive-json-8472" - - with pytest.raises(PrivacyGuardError) as exception_info: - JsonHandler().normalize( - f'{{"message":"{sensitive_text}",}}'.encode(), PolicyConfig() - ) - - _assert_safe_error( - exception_info.value, ErrorCode.BODY_JSON_INVALID, sensitive_text - ) - - -def test_normalize_shape_failure_drops_sensitive_exception_context( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import privacy_guard.request_body.json as json_module - - sensitive_text = "distinctive-sensitive-shape-8472" - monkeypatch.setattr(json_module, "MAX_SCANNED_CHARACTERS", len(sensitive_text) - 1) - - with pytest.raises(PrivacyGuardError) as exception_info: - JsonHandler().normalize(f'"{sensitive_text}"'.encode(), PolicyConfig()) - - _assert_safe_error( - exception_info.value, - ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED, - sensitive_text, - ) - - -@pytest.mark.parametrize("non_finite_value", [b"NaN", b"Infinity", b"-Infinity"]) -def test_normalize_rejects_non_standard_numeric_constants( - non_finite_value: bytes, -) -> None: - with pytest.raises(PrivacyGuardError) as exception_info: - JsonHandler().normalize(b'{"number":' + non_finite_value + b"}", PolicyConfig()) - - assert exception_info.value.code is ErrorCode.BODY_JSON_INVALID - - -def test_normalize_rejects_finite_syntax_that_overflows_to_infinity() -> None: - with pytest.raises(PrivacyGuardError) as exception_info: - JsonHandler().normalize(b'{"number":1e400}', PolicyConfig()) - - assert exception_info.value.code is ErrorCode.BODY_JSON_INVALID - - -def test_normalize_maps_python_huge_integer_limit_to_safe_json_error() -> None: - digit_limit = sys.int_info.default_max_str_digits - raw_body = b'{"number":' + (b"9" * (digit_limit + 1)) + b"}" - - with pytest.raises(PrivacyGuardError) as exception_info: - JsonHandler().normalize(raw_body, PolicyConfig()) - - assert exception_info.value.code is ErrorCode.BODY_JSON_INVALID - assert "999999" not in str(exception_info.value) - - -def test_normalize_rejects_duplicate_object_members_without_leaking_content() -> None: - sensitive_text = "distinctive-duplicate-value-8472" - - with pytest.raises(PrivacyGuardError) as exception_info: - JsonHandler().normalize( - (f'{{"message":"{sensitive_text}","message":"benign"}}').encode(), - PolicyConfig(), - ) - - assert exception_info.value.code is ErrorCode.BODY_JSON_INVALID - assert sensitive_text not in str(exception_info.value) - assert sensitive_text not in repr(exception_info.value) - - -def test_normalize_rejects_unpaired_surrogates_without_leaking_content() -> None: - sensitive_text = "distinctive-surrogate-neighbor-8472" - - with pytest.raises(PrivacyGuardError) as exception_info: - JsonHandler().normalize( - f'{{"message":"{sensitive_text}","invalid":"\\ud800"}}'.encode(), - PolicyConfig(), - ) - - _assert_safe_error( - exception_info.value, - ErrorCode.BODY_JSON_INVALID, - sensitive_text, - ) - - -def test_normalize_rejects_unpaired_surrogate_object_key_without_leaking_content() -> ( - None -): - sensitive_text = "distinctive-surrogate-key-neighbor-8472" - - with pytest.raises(PrivacyGuardError) as exception_info: - JsonHandler().normalize( - f'{{"{sensitive_text}":"safe","\\ud800":"invalid"}}'.encode(), - PolicyConfig(), - ) - - _assert_safe_error( - exception_info.value, - ErrorCode.BODY_JSON_INVALID, - sensitive_text, - ) - - -def test_reconstruct_without_replacements_returns_exact_original_bytes() -> None: - raw_body = b'{\n "message": "hello", "escaped": "\\u263a"\n}\n ' - json_handler = JsonHandler() - request_body = json_handler.normalize(raw_body, PolicyConfig()) - - assert json_handler.reconstruct(request_body, {}) is raw_body - - -def test_reconstruct_replaces_one_string_leaf() -> None: - json_handler = JsonHandler() - request_body = json_handler.normalize( - b'{"message":"hello","count":2}', PolicyConfig() - ) - - reconstructed_body = json_handler.reconstruct(request_body, {"/message": "goodbye"}) - - assert json.loads(reconstructed_body) == {"message": "goodbye", "count": 2} - assert reconstructed_body == b'{"message":"goodbye","count":2}' - - -def test_reconstruct_with_replacements_deepcopies_state_exactly_once( - monkeypatch: pytest.MonkeyPatch, -) -> None: - copy_count = 0 - - def counting_deepcopy(value: object) -> object: - nonlocal copy_count - copy_count += 1 - return deepcopy(value) - - json_handler = JsonHandler() - request_body = json_handler.normalize( - b'{"first":"one","nested":{"second":"two"}}', PolicyConfig() - ) - monkeypatch.setattr(json_module, "deepcopy", counting_deepcopy) - - reconstructed_body = json_handler.reconstruct( - request_body, {"/first": "ONE", "/nested/second": "TWO"} - ) - - assert json.loads(reconstructed_body) == { - "first": "ONE", - "nested": {"second": "TWO"}, - } - assert copy_count == 1 - - -@pytest.mark.parametrize( - ("raw_body", "replacements_by_path", "expected_value"), - [ - ( - b'{"first":"one","nested":{"second":"two"}}', - {"/first": "ONE", "/nested/second": "TWO"}, - {"first": "ONE", "nested": {"second": "TWO"}}, - ), - ( - b'{"message":"hello","other":"keep"}', - {"/message": ""}, - {"message": "", "other": "keep"}, - ), - (b'"root"', {"": "replaced"}, "replaced"), - ( - b'{"a/b":"slash","til~de":"tilde"}', - {"/a~1b": "new slash", "/til~0de": "new tilde"}, - {"a/b": "new slash", "til~de": "new tilde"}, - ), - ( - '{"message":"こんにちは"}'.encode(), - {"/message": "さようなら"}, - {"message": "さようなら"}, - ), - ], -) -def test_reconstruct_applies_explicit_string_replacements( - raw_body: bytes, - replacements_by_path: dict[str, str], - expected_value: object, -) -> None: - json_handler = JsonHandler() - request_body = json_handler.normalize(raw_body, PolicyConfig()) - - reconstructed_body = json_handler.reconstruct(request_body, replacements_by_path) - - assert json.loads(reconstructed_body) == expected_value - assert b"\\u3055" not in reconstructed_body - - -@pytest.mark.parametrize( - ("raw_body", "json_pointer"), - [ - (b'{"message":"secret-8472"}', "/unknown"), - (b'{"message":"secret-8472"}', "message"), - (b'{"~bad":"secret-8472"}', "/~2bad"), - (b'["secret-8472"]', "/-"), - (b'["secret-8472"]', "/01"), - (b'["secret-8472"]', "/1"), - (b'{"count":2}', "/count"), - (b'{"message":"secret-8472"}', ""), - ], -) -def test_reconstruct_rejects_invalid_or_non_string_targets_without_leaking_content( - raw_body: bytes, json_pointer: str -) -> None: - json_handler = JsonHandler() - request_body = json_handler.normalize(raw_body, PolicyConfig()) - - with pytest.raises(PrivacyGuardError) as exception_info: - json_handler.reconstruct(request_body, {json_pointer: "replacement"}) - - assert exception_info.value.code is ErrorCode.BODY_RECONSTRUCTION_INVALID - assert "secret-8472" not in str(exception_info.value) - assert "secret-8472" not in repr(exception_info.value) - assert exception_info.value.__cause__ is None - assert all( - "secret-8472" not in str(argument) for argument in exception_info.value.args - ) - - -@pytest.mark.parametrize("parsed_value", [{"edit": "original"}, object()]) -@pytest.mark.parametrize("replacements_by_path", [{}, {"/edit": "changed"}]) -def test_reconstruct_rejects_foreign_state_without_leaking_content( - parsed_value: object, - replacements_by_path: dict[str, str], -) -> None: - sensitive_text = "distinctive-sensitive-foreign-state-8472" - request_body = RequestBody( - text_blocks=(TextBlock(path="/edit", text="original"),), - parsed_value=parsed_value, - original_bytes=sensitive_text.encode(), - ) - - with pytest.raises(PrivacyGuardError) as exception_info: - JsonHandler().reconstruct(request_body, replacements_by_path) - - _assert_safe_error( - exception_info.value, - ErrorCode.BODY_RECONSTRUCTION_INVALID, - sensitive_text, - ) - - -def test_reconstruct_copy_failure_drops_sensitive_exception_context( - monkeypatch: pytest.MonkeyPatch, -) -> None: - sensitive_text = "distinctive-sensitive-copy-8472" - json_handler = JsonHandler() - request_body = json_handler.normalize(b'{"edit":"original"}', PolicyConfig()) - - def fail_copy(*values: object) -> None: - raise RuntimeError(sensitive_text) - - monkeypatch.setattr(json_module, "deepcopy", fail_copy) - - with pytest.raises(PrivacyGuardError) as exception_info: - json_handler.reconstruct(request_body, {"/edit": "changed"}) - - _assert_safe_error( - exception_info.value, - ErrorCode.BODY_RECONSTRUCTION_INVALID, - sensitive_text, - ) - - -def test_reconstruct_serialization_failure_drops_sensitive_exception_context( - monkeypatch: pytest.MonkeyPatch, -) -> None: - sensitive_text = "distinctive-sensitive-serialization-8472" - json_handler = JsonHandler() - request_body = json_handler.normalize(b'{"edit":"original"}', PolicyConfig()) - - def fail_serialization( - value: object, - *, - ensure_ascii: bool, - separators: tuple[str, str], - allow_nan: bool, - ) -> str: - raise TypeError(sensitive_text) - - monkeypatch.setattr(json_module.json, "dumps", fail_serialization) - - with pytest.raises(PrivacyGuardError) as exception_info: - json_handler.reconstruct(request_body, {"/edit": "changed"}) - - _assert_safe_error( - exception_info.value, - ErrorCode.BODY_RECONSTRUCTION_INVALID, - sensitive_text, - ) - - -def test_reconstruct_rejects_unpaired_surrogate_replacement_without_leaking_body() -> ( - None -): - sensitive_text = "distinctive-reconstruction-neighbor-8472" - json_handler = JsonHandler() - request_body = json_handler.normalize( - f'{{"edit":"original","neighbor":"{sensitive_text}"}}'.encode(), - PolicyConfig(), - ) - - with pytest.raises(PrivacyGuardError) as exception_info: - json_handler.reconstruct(request_body, {"/edit": "\ud800"}) - - assert exception_info.value.code is ErrorCode.BODY_RECONSTRUCTION_INVALID - assert sensitive_text not in str(exception_info.value) - assert sensitive_text not in repr(exception_info.value) - - -def test_reconstruct_does_not_mutate_original_parsed_value() -> None: - json_handler = JsonHandler() - request_body = json_handler.normalize( - b'{"items":[{"message":"original"}],"other":"keep"}', PolicyConfig() - ) - original_value = {"items": [{"message": "original"}], "other": "keep"} - - json_handler.reconstruct(request_body, {"/items/0/message": "changed"}) - - assert type(request_body.parsed_value) is json_module._JsonBodyState - assert _json_state_value(request_body) == original_value - - -def test_select_format_handler_returns_registered_json_singleton() -> None: - json_handler = select_format_handler("json") - - assert isinstance(json_handler, JsonHandler) - assert select_format_handler("json") is json_handler - - -def test_select_format_handler_rejects_unknown_kind_without_fallback() -> None: - sentinel = "unknown-sensitive-format-8472" - - with pytest.raises(PrivacyGuardError) as exception_info: - select_format_handler(sentinel) - - assert exception_info.value.code is ErrorCode.BODY_FORMAT_UNSUPPORTED - assert sentinel not in str(exception_info.value) - assert sentinel not in repr(exception_info.value) diff --git a/projects/privacy-guard/tests/scanner_helpers.py b/projects/privacy-guard/tests/scanner_helpers.py deleted file mode 100644 index 81188ea..0000000 --- a/projects/privacy-guard/tests/scanner_helpers.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Deterministic scanners used only by Privacy Guard tests.""" - -from __future__ import annotations - -import re - -from typing_extensions import override - -from privacy_guard.scanners import ( - Confidence, - Finding, - ScanBudget, - Scanner, - ScannerConfig, -) - - -class DeterministicEmailScanner(Scanner[ScannerConfig]): - _EMAIL = re.compile( - r"(? tuple[Finding, ...]: - return tuple( - Finding( - entity="email", - scanner_name=self.scanner_name, - start_offset=match.start(), - end_offset=match.end(), - confidence=Confidence.HIGH, - ) - for match in self._EMAIL.finditer(text_block) - ) diff --git a/projects/privacy-guard/tests/scanners/test_regex.py b/projects/privacy-guard/tests/scanners/test_regex.py deleted file mode 100644 index 419b4f0..0000000 --- a/projects/privacy-guard/tests/scanners/test_regex.py +++ /dev/null @@ -1,277 +0,0 @@ -from __future__ import annotations - -from concurrent.futures import ThreadPoolExecutor -from io import BytesIO -from pathlib import Path -from time import monotonic - -import pytest -from typing_extensions import override - -import privacy_guard.scanners.regex as regex_module -from privacy_guard.errors import ErrorCode, ErrorKind, PrivacyGuardError -from privacy_guard.scanners import RegexScanner, ScanBudget, ScanBudgetExceeded - - -def _write(path: Path, text: str) -> Path: - path.write_text(text, encoding="utf-8") - return path - - -def _single(patterns: str) -> str: - return f""" -- name: token - patterns: -{patterns} -""" - - -def test_single_profile_reports_overlaps_names_confidence_and_unicode( - tmp_path: Path, -) -> None: - path = _write( - tmp_path / "entities.yaml", - _single( - """ - name: whole-token - regex: 'aba' - confidence: high - - name: suffix-token - regex: 'ba' - confidence: medium -""" - ), - ) - - findings = RegexScanner.from_yaml(path).scan("🐍aba") - - assert [ - ( - item.metadata["pattern_name"] if item.metadata is not None else None, - item.start_offset, - item.end_offset, - item.confidence.value, - ) - for item in findings - ] == [("whole-token", 1, 4, "high"), ("suffix-token", 2, 4, "medium")] - - -def test_one_pattern_uses_overlapping_iteration(tmp_path: Path) -> None: - path = _write( - tmp_path / "overlap.yaml", - _single(" - name: pair\n regex: 'aa'\n confidence: high\n"), - ) - - findings = RegexScanner.from_yaml(path).scan("aaa") - - assert [(item.start_offset, item.end_offset) for item in findings] == [ - (0, 2), - (1, 3), - ] - - -def test_hyphen_normalization_coexists_with_underscore_name(tmp_path: Path) -> None: - path = _write( - tmp_path / "names.yaml", - _single( - """ - name: same-name - regex: 'x' - confidence: high - - name: same_name - regex: 'y' - confidence: high -""" - ), - ) - - findings = RegexScanner.from_yaml(path).scan("xy") - - assert [ - item.metadata["pattern_name"] if item.metadata is not None else None - for item in findings - ] == ["same-name", "same_name"] - - -def test_numeric_backreferences_keep_their_group_numbers(tmp_path: Path) -> None: - path = _write( - tmp_path / "backref.yaml", - _single( - " - name: repeated\n regex: '(a)\\1'\n confidence: high\n" - ), - ) - - finding = RegexScanner.from_yaml(path).scan("aa")[0] - assert finding.metadata is not None - assert finding.metadata["pattern_name"] == "repeated" - - -def test_profiles_require_selection_and_do_not_merge(tmp_path: Path) -> None: - path = _write( - tmp_path / "profiles.yaml", - """ -profiles: - first: - - name: alpha - patterns: - - name: alpha - regex: 'a' - confidence: high - second: - - name: beta - patterns: - - name: beta - regex: 'b' - confidence: high -""", - ) - - with pytest.raises(PrivacyGuardError) as missing: - RegexScanner.from_yaml(path) - assert missing.value.code is ErrorCode.SCANNER_CONFIG_INVALID - - scanner = RegexScanner.from_yaml(path, "second") - assert scanner.supported_entity_types == frozenset({"beta"}) - assert [item.entity for item in scanner.scan("ab")] == ["beta"] - - -@pytest.mark.parametrize( - "document", - [ - "- name: x\n name: y\n patterns: []\n", - "- &entity\n name: x\n patterns: []\n", - "- !unsafe {name: x, patterns: []}\n", - "- name: x\n patterns:\n" - " [{name: p, regex: 'x', confidence: high, extra: true}]\n", - "- name: x\n patterns: [{name: p, regex: '(?Px)', confidence: high}]\n", - "- name: x\n patterns: [{name: p, regex: '(?i:x)', confidence: high}]\n", - "- name: x\n patterns: [{name: p, regex: 'x*', confidence: high}]\n", - ], -) -def test_invalid_yaml_and_patterns_use_one_content_safe_error( - tmp_path: Path, document: str -) -> None: - path = _write(tmp_path / "sensitive-name-8472.yaml", document) - - with pytest.raises(PrivacyGuardError) as exception_info: - RegexScanner.from_yaml(path) - - assert exception_info.value.code is ErrorCode.SCANNER_CONFIG_INVALID - assert exception_info.value.kind is ErrorKind.INVALID_INPUT - assert "8472" not in str(exception_info.value) - - -def test_configuration_read_is_bounded_before_parsing( - monkeypatch: pytest.MonkeyPatch, -) -> None: - class RecordingBytesIO(BytesIO): - def __init__(self) -> None: - super().__init__(b"x" * 100) - self.read_sizes: list[int | None] = [] - - @override - def read(self, size: int | None = -1, /) -> bytes: - self.read_sizes.append(size) - return super().read(size) - - stream = RecordingBytesIO() - monkeypatch.setattr(regex_module, "MAX_SCANNER_CONFIG_BYTES", 64) - monkeypatch.setattr(Path, "open", lambda path, mode: stream) - - with pytest.raises(PrivacyGuardError) as exception_info: - RegexScanner.from_yaml("ignored.yaml") - - assert exception_info.value.code is ErrorCode.SCANNER_CONFIG_INVALID - assert stream.read_sizes == [65] - - -def test_inactive_profiles_are_fully_validated(tmp_path: Path) -> None: - path = _write( - tmp_path / "profiles.yaml", - """ -profiles: - selected: - - name: alpha - patterns: [{name: alpha, regex: 'a', confidence: high}] - broken: - - name: beta - patterns: [{name: beta, regex: '(?Pb)', confidence: high}] -""", - ) - - with pytest.raises(PrivacyGuardError) as exception_info: - RegexScanner.from_yaml(path, "selected") - - assert exception_info.value.code is ErrorCode.SCANNER_CONFIG_INVALID - assert exception_info.value.kind is ErrorKind.INVALID_INPUT - - -def test_excessive_yaml_nesting_is_rejected_safely(tmp_path: Path) -> None: - document = "value" - for _ in range(20): - document = f"[{document}]" - path = _write(tmp_path / "nested.yaml", document) - - with pytest.raises(PrivacyGuardError) as exception_info: - RegexScanner.from_yaml(path) - - assert exception_info.value.code is ErrorCode.SCANNER_CONFIG_INVALID - assert exception_info.value.kind is ErrorKind.INVALID_INPUT - - -def test_deeply_nested_regex_is_a_content_safe_configuration_error( - tmp_path: Path, -) -> None: - nested_expression = "(" * 1_000 + "x" + ")" * 1_000 - path = _write( - tmp_path / "deep-regex.yaml", - _single( - " - name: nested\n" - f" regex: '{nested_expression}'\n" - " confidence: high\n" - ), - ) - - with pytest.raises(PrivacyGuardError) as exception_info: - RegexScanner.from_yaml(path) - - assert exception_info.value.code is ErrorCode.SCANNER_CONFIG_INVALID - assert exception_info.value.kind is ErrorKind.INVALID_INPUT - - -def test_contextual_zero_width_is_a_runtime_configuration_failure( - tmp_path: Path, -) -> None: - path = _write( - tmp_path / "contextual.yaml", - _single(" - name: zero\n regex: '(?=a)'\n confidence: high\n"), - ) - scanner = RegexScanner.from_yaml(path) - - with pytest.raises(PrivacyGuardError) as exception_info: - scanner.scan("a") - - assert exception_info.value.code is ErrorCode.SCANNER_CONFIG_INVALID - assert exception_info.value.kind is ErrorKind.INTERNAL - - -def test_standalone_scan_exposes_typed_budget_exhaustion(tmp_path: Path) -> None: - path = _write( - tmp_path / "budget.yaml", - _single(" - name: literal\n regex: 'x'\n confidence: high\n"), - ) - scanner = RegexScanner.from_yaml(path) - - with pytest.raises(ScanBudgetExceeded): - scanner.scan("x", budget=ScanBudget(deadline=monotonic() - 1)) - - -def test_scanner_instance_supports_concurrent_calls(tmp_path: Path) -> None: - path = _write( - tmp_path / "concurrent.yaml", - _single(" - name: literal\n regex: 'x'\n confidence: high\n"), - ) - scanner = RegexScanner.from_yaml(path) - - with ThreadPoolExecutor(max_workers=4) as executor: - results = tuple(executor.map(scanner.scan, ("x",) * 16)) - - assert all(len(findings) == 1 for findings in results) diff --git a/projects/privacy-guard/tests/scanners/test_scanner_models.py b/projects/privacy-guard/tests/scanners/test_scanner_models.py deleted file mode 100644 index 21deb94..0000000 --- a/projects/privacy-guard/tests/scanners/test_scanner_models.py +++ /dev/null @@ -1,175 +0,0 @@ -import math -from types import MappingProxyType - -import pytest -from pydantic import ValidationError -from typing_extensions import override - -from privacy_guard.scanners import ( - Finding, - RequestBodyFinding, - ScanBudget, - Scanner, - ScannerConfig, -) - - -def test_finding_names_the_scanner_that_produced_it() -> None: - finding = Finding( - entity="email", - scanner_name="example-scanner", - start_offset=4, - end_offset=18, - ) - - assert finding.scanner_name == "example-scanner" - - -def test_finding_accepts_general_immutable_string_metadata() -> None: - finding = Finding( - entity="email", - scanner_name="scanner", - start_offset=0, - end_offset=1, - metadata={"rule": "common-email", "source": "customer-catalog"}, - ) - - assert finding.metadata == { - "rule": "common-email", - "source": "customer-catalog", - } - assert isinstance(finding.metadata, MappingProxyType) - assert "customer-catalog" not in repr(finding) - - -def test_scanner_config_owns_identity_and_entity_types() -> None: - config = ScannerConfig(name="s" * 2048, entity_types=frozenset({"email", "phone"})) - - assert config.name == "s" * 2048 - assert config.entity_types == frozenset({"email", "phone"}) - with pytest.raises(ValidationError): - setattr(config, "name", "changed") - - -@pytest.mark.parametrize( - "values", - [ - {"name": "", "entity_types": frozenset()}, - {"name": 3, "entity_types": frozenset()}, - {"name": "scanner"}, - {"name": "scanner", "entity_types": "email"}, - {"name": "scanner", "entity_types": frozenset({3})}, - ], -) -def test_scanner_config_rejects_invalid_metadata(values: dict[str, object]) -> None: - with pytest.raises(ValidationError): - ScannerConfig.model_validate(values) - - -@pytest.mark.parametrize( - "values", - [ - {"entity": "", "scanner_name": "scanner", "start_offset": 0, "end_offset": 1}, - {"entity": "email", "scanner_name": "", "start_offset": 0, "end_offset": 1}, - { - "entity": "\U0001f4a3" * 257, - "scanner_name": "scanner", - "start_offset": 0, - "end_offset": 1, - }, - { - "entity": "email", - "scanner_name": "\U0001f4a3" * 257, - "start_offset": 0, - "end_offset": 1, - }, - { - "entity": "email", - "scanner_name": "scanner", - "start_offset": True, - "end_offset": 1, - }, - { - "entity": "email", - "scanner_name": "scanner", - "start_offset": -1, - "end_offset": 1, - }, - { - "entity": "email", - "scanner_name": "scanner", - "start_offset": 1, - "end_offset": 1, - }, - { - "entity": "email", - "scanner_name": "scanner", - "start_offset": 0, - "end_offset": 1, - "confidence": "high", - }, - { - "entity": "email", - "scanner_name": "scanner", - "start_offset": 0, - "end_offset": 1, - "metadata": {"key": 3}, - }, - { - "entity": "email", - "scanner_name": "scanner", - "start_offset": 0, - "end_offset": 1, - "text_block_path": "path", - }, - ], -) -def test_finding_rejects_invalid_or_extra_fields(values: dict[str, object]) -> None: - with pytest.raises(ValidationError): - Finding.model_validate(values) - - -def test_request_body_finding_requires_hidden_path_and_models_are_frozen() -> None: - finding = Finding( - entity="email", scanner_name="scanner", start_offset=0, end_offset=1 - ) - request_body_finding = RequestBodyFinding( - finding=finding, text_block_path="sensitive/path" - ) - - assert request_body_finding.text_block_path == "sensitive/path" - assert "sensitive/path" not in repr(request_body_finding) - assert not hasattr(finding, "text_block_path") - with pytest.raises(ValidationError): - setattr(finding, "entity", "token") - with pytest.raises(ValidationError): - RequestBodyFinding.model_validate( - { - "finding": finding, - } - ) - - -def test_scanner_initialize_runs_after_validated_config_is_available() -> None: - class InitializingScanner(Scanner[ScannerConfig]): - initialized_name: str | None = None - - @override - def _initialize(self) -> None: - self.initialized_name = self.scanner_name - - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - return () - - scanner = InitializingScanner( - ScannerConfig(name="initialized", entity_types=frozenset()) - ) - - assert scanner.initialized_name == "initialized" - - -@pytest.mark.parametrize("deadline", [True, math.inf, math.nan, "soon"]) -def test_scan_budget_uses_strict_pydantic_validation(deadline: object) -> None: - with pytest.raises(ValidationError): - ScanBudget.model_validate({"deadline": deadline}) diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 5d82909..af2a9d5 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -1,576 +1,286 @@ +"""Registry-based server lifecycle and discovery CLI tests.""" + +from __future__ import annotations + +import asyncio +import json import logging import re -from collections.abc import Sequence -from pathlib import Path import grpc import pytest -from google.protobuf import empty_pb2, message_factory -from google.protobuf.message import Message from typer.testing import CliRunner, Result -from typing_extensions import override -from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 -from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc -from privacy_guard.constants import MAX_BODY_BYTES +from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES +from privacy_guard.engines import EntityProcessingStrategy from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.processor import RequestProcessor -from privacy_guard.scanners import ScannerConfig from privacy_guard.service import server as server_module -from privacy_guard.service.server import MiddlewareServer, app, create_server, serve +from privacy_guard.service.server import ( + MiddlewareServer, + app, + create_default_registry, + create_server, + serve, +) from privacy_guard.service.servicer import PrivacyGuardMiddleware -from ..scanner_helpers import DeterministicEmailScanner - _ANSI_STYLE_PATTERN = re.compile(r"\x1b\[[0-9;]*m") -class LifecycleServerFake(grpc.aio.Server): - """Nominal gRPC server fake for lifecycle-only server tests.""" - - def __init__(self) -> None: - self.stopped = False +class LifecycleServerFake: + """Minimal async-server fake for lifecycle-only tests.""" - @override - def add_generic_rpc_handlers( - self, generic_rpc_handlers: Sequence[grpc.GenericRpcHandler] + def __init__( + self, + *, + bound_port: int = 50051, + bind_error: RuntimeError | None = None, + start_error: RuntimeError | None = None, ) -> None: - raise AssertionError("generic handler registration is not under test") + self.bound_port = bound_port + self.bind_error = bind_error + self.start_error = start_error + self.addresses: list[str] = [] + self.started = False + self.waited = False + self.stop_graces: list[float | None] = [] - @override def add_insecure_port(self, address: str) -> int: - raise NotImplementedError - - @override - def add_secure_port( - self, address: str, server_credentials: grpc.ServerCredentials - ) -> int: - raise AssertionError("secure binding is not under test") + self.addresses.append(address) + if self.bind_error is not None: + raise self.bind_error + return self.bound_port - @override async def start(self) -> None: - raise NotImplementedError - - @override - async def stop(self, grace: float | None) -> None: - self.stopped = True + if self.start_error is not None: + raise self.start_error + self.started = True - @override async def wait_for_termination(self, timeout: float | None = None) -> bool: - raise NotImplementedError - + del timeout + self.waited = True + return True -def _middleware() -> PrivacyGuardMiddleware: - scanner = DeterministicEmailScanner( - ScannerConfig(name="test_email", entity_types=frozenset({"email"})) - ) - return PrivacyGuardMiddleware(RequestProcessor([scanner])) + async def stop(self, grace: float | None) -> None: + self.stop_graces.append(grace) def _plain_output(result: Result) -> str: return _ANSI_STYLE_PATTERN.sub("", result.output) -def test_middleware_server_wires_scanner_and_has_a_default_listen_address( - monkeypatch: pytest.MonkeyPatch, -) -> None: - served: list[tuple[str, PrivacyGuardMiddleware]] = [] - - async def record_serve(servicer: PrivacyGuardMiddleware, listen: str) -> None: - served.append((listen, servicer)) - - monkeypatch.setattr(server_module, "serve", record_serve) - scanner = DeterministicEmailScanner( - ScannerConfig(name="test_email", entity_types=frozenset({"email"})) - ) - - MiddlewareServer(scanner=scanner).serve() - - assert len(served) == 1 - assert served[0][0] == "127.0.0.1:50051" - assert isinstance(served[0][1], PrivacyGuardMiddleware) - - -def test_cli_reports_expected_configuration_failure_without_traceback( - monkeypatch: pytest.MonkeyPatch, -) -> None: - expected = PrivacyGuardError(ErrorCode.SCANNER_CONFIG_INVALID) - - def reject_configuration(*args: object, **kwargs: object) -> None: - raise expected - - monkeypatch.setattr(server_module.RegexScanner, "from_yaml", reject_configuration) - - result = CliRunner().invoke( - app, - ["regex", "--config", "sensitive-path-8472.yaml"], - ) - - assert result.exit_code == 1 - assert result.output == f"{expected}\n" - assert "Traceback" not in result.output - assert "8472" not in result.output - - -def test_cli_reports_bind_failure_without_traceback( - monkeypatch: pytest.MonkeyPatch, -) -> None: - expected = PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) - scanner = DeterministicEmailScanner( - ScannerConfig(name="regex", entity_types=frozenset({"email"})) - ) - - monkeypatch.setattr( - server_module.RegexScanner, - "from_yaml", - lambda *args, **kwargs: scanner, - ) +def _middleware() -> PrivacyGuardMiddleware: + return PrivacyGuardMiddleware(create_default_registry()) - def reject_listen_address(self: MiddlewareServer, listen: str) -> None: - raise expected - monkeypatch.setattr(MiddlewareServer, "serve", reject_listen_address) +def test_default_registry_contains_the_builtin_regex_engine() -> None: + registry = create_default_registry() - result = CliRunner().invoke( - app, - ["regex", "--config", "scanner-config.yaml", "--listen", "bad:8472"], - ) + assert registry.is_finalized is True + assert registry.engine_names == ("regex",) + description = registry.describe_engines()[0] + assert description.engine == "regex" + assert description.supported_strategy is EntityProcessingStrategy.REPLACE - assert result.exit_code == 1 - assert result.output == f"{expected}\n" - assert "Traceback" not in result.output - assert "8472" not in result.output - -@pytest.mark.parametrize("profile", [None, "customer-support"]) -def test_cli_profile_is_optional_and_forwarded_only_when_supplied( +def test_middleware_server_uses_an_injected_registry_and_default_address( monkeypatch: pytest.MonkeyPatch, - profile: str | None, ) -> None: - calls: list[tuple[str, str | None, str]] = [] - scanner = DeterministicEmailScanner( - ScannerConfig(name="regex", entity_types=frozenset({"email"})) - ) + registry = create_default_registry() + served: list[tuple[PrivacyGuardMiddleware, str]] = [] - def load_scanner( - path: str | Path, - selected_profile: str | None, - *, - scanner_name: str, - ) -> DeterministicEmailScanner: - calls.append((str(path), selected_profile, scanner_name)) - return scanner + async def record_serve(servicer: PrivacyGuardMiddleware, listen: str) -> None: + served.append((servicer, listen)) + await servicer.close() - monkeypatch.setattr(server_module.RegexScanner, "from_yaml", load_scanner) - monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) - arguments = ["regex", "--config", "scanner-config.yaml"] - if profile is not None: - arguments.extend(("--profile", profile)) + monkeypatch.setattr(server_module, "serve", record_serve) - result = CliRunner().invoke(app, arguments) + MiddlewareServer(registry=registry, log_request_content=True).serve() - assert result.exit_code == 0 - assert calls == [("scanner-config.yaml", profile, "regex")] + assert len(served) == 1 + assert served[0][1] == "127.0.0.1:50051" + assert served[0][0]._registry is registry + assert served[0][0]._processors._log_request_content is True -def test_cli_forwards_custom_scanner_name_to_builtin( +def test_create_server_sets_transport_limits_and_registers_servicer( monkeypatch: pytest.MonkeyPatch, ) -> None: - calls: list[tuple[str, str | None, str]] = [] - scanner = DeterministicEmailScanner( - ScannerConfig(name="custom-regex", entity_types=frozenset({"email"})) - ) + fake_server = object() + server_options: list[tuple[int, tuple[tuple[str, int], ...]]] = [] + registrations: list[tuple[PrivacyGuardMiddleware, object]] = [] - def load_scanner( - path: str | Path, - selected_profile: str | None, + def fake_server_factory( *, - scanner_name: str, - ) -> DeterministicEmailScanner: - calls.append((str(path), selected_profile, scanner_name)) - return scanner - - monkeypatch.setattr(server_module.RegexScanner, "from_yaml", load_scanner) - monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) - - result = CliRunner().invoke( - app, - [ - "regex", - "--config", - "scanner-config.yaml", - "--scanner-name", - "custom-regex", - ], - ) - - assert result.exit_code == 0 - assert calls == [("scanner-config.yaml", None, "custom-regex")] - + maximum_concurrent_rpcs: int, + options: tuple[tuple[str, int], ...], + ) -> object: + server_options.append((maximum_concurrent_rpcs, options)) + return fake_server + + def record_registration( + servicer: PrivacyGuardMiddleware, + server: object, + ) -> None: + registrations.append((servicer, server)) -def test_cli_logs_server_start_without_configuration_details( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - scanner = DeterministicEmailScanner( - ScannerConfig(name="custom-regex", entity_types=frozenset({"email"})) - ) + middleware = _middleware() + monkeypatch.setattr(grpc.aio, "server", fake_server_factory) monkeypatch.setattr( - server_module.RegexScanner, - "from_yaml", - lambda *args, **kwargs: scanner, + server_module.pb2_grpc, + "add_SupervisorMiddlewareServicer_to_server", + record_registration, ) - monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) + try: + result = create_server(middleware) + finally: + asyncio.run(middleware.close()) - with caplog.at_level(logging.INFO, logger="privacy_guard.service.server"): - result = CliRunner().invoke( - app, - [ - "regex", - "--config", - "sensitive-config-8472.yaml", - "--listen", - "127.0.0.1:50051", - "--scanner-name", - "custom-regex", - ], + assert result is fake_server + assert server_options == [ + ( + MAX_CONCURRENT_RPCS, + (("grpc.max_receive_message_length", MAX_RECEIVE_MESSAGE_BYTES),), ) + ] + assert registrations == [(middleware, fake_server)] - assert result.exit_code == 0 - assert "privacy_guard_server_starting" in caplog.text - assert "scanner=custom-regex" in caplog.text - assert "listen=127.0.0.1:50051" in caplog.text - assert "8472" not in caplog.text +def test_cli_help_exposes_only_pipeline_server_and_discovery_commands() -> None: + result = CliRunner().invoke(app, ["--help"]) -def test_cli_always_requires_scanner_configuration() -> None: - result = CliRunner().invoke(app, ["regex"]) + assert result.exit_code == 0 + output = _plain_output(result) + assert "serve" in output + assert "schema" in output + assert "engines" in output + assert "--debug" in output + assert "--debug-log-content" in output + assert "--config" not in output + assert "--profile" not in output + assert "--scanner-name" not in output - assert result.exit_code == 2 - assert "--config" in _plain_output(result) +def test_cli_engines_describes_the_installed_engine() -> None: + result = CliRunner().invoke(app, ["engines"]) -def test_cli_exposes_root_and_builtin_scanner_help() -> None: - root_help = CliRunner().invoke(app, ["--help"]) + assert result.exit_code == 0 + assert result.output.startswith("regex\treplace\t") + assert "Detect overlapping regex matches" in result.output - assert root_help.exit_code == 0 - plain_root_help = _plain_output(root_help) - assert "privacy-guard" in plain_root_help - assert "regex" in plain_root_help - assert "--debug" in plain_root_help - assert "--debug-log-content" in plain_root_help - scanner_help = CliRunner().invoke(app, ["regex", "--help"]) +def test_cli_schema_prints_the_finalized_discriminated_policy_schema() -> None: + result = CliRunner().invoke(app, ["schema"]) - assert scanner_help.exit_code == 0 - plain_scanner_help = _plain_output(scanner_help) - assert "built-in RegexScanner" in plain_scanner_help - assert "--config" in plain_scanner_help - assert "--listen" in plain_scanner_help - assert "--profile" in plain_scanner_help - assert "--scanner-name" in plain_scanner_help + assert result.exit_code == 0 + schema = json.loads(result.output) + serialized = json.dumps(schema, sort_keys=True) + assert '"propertyName": "engine"' in serialized + assert '"regex"' in serialized + assert '"on_detection"' in serialized -def test_cli_enables_explicit_request_content_logging( +def test_cli_serve_forwards_operational_options_only( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: - processor_options: list[bool] = [] - scanner = DeterministicEmailScanner( - ScannerConfig(name="regex", entity_types=frozenset({"email"})) - ) - real_request_processor = server_module.RequestProcessor + calls: list[tuple[str, bool]] = [] - def create_processor( - scanners: Sequence[DeterministicEmailScanner], - *, - log_request_content: bool = False, - ) -> RequestProcessor: - processor_options.append(log_request_content) - return real_request_processor( - scanners, - log_request_content=log_request_content, - ) + def record_serve(self: MiddlewareServer, listen: str) -> None: + calls.append((listen, self._servicer._processors._log_request_content)) - monkeypatch.setattr( - server_module.RegexScanner, - "from_yaml", - lambda *args, **kwargs: scanner, - ) - monkeypatch.setattr(server_module, "RequestProcessor", create_processor) - monkeypatch.setattr(MiddlewareServer, "serve", lambda self, listen: None) + monkeypatch.setattr(MiddlewareServer, "serve", record_serve) with caplog.at_level(logging.WARNING, logger="privacy_guard.service.server"): result = CliRunner().invoke( app, - [ - "--debug-log-content", - "regex", - "--config", - "scanner-config.yaml", - ], + ["--debug-log-content", "serve", "--listen", "127.0.0.1:50052"], ) assert result.exit_code == 0 - assert processor_options == [True] + assert calls == [("127.0.0.1:50052", True)] assert "privacy_guard_request_content_logging_enabled" in caplog.text - assert "complete_request_bodies_may_contain_secrets" in caplog.text - - -def test_cli_rejects_builtin_options_before_subcommand() -> None: - result = CliRunner().invoke( - app, - [ - "--profile", - "customer-support", - "regex", - ], - ) - - assert result.exit_code == 2 - assert "--profile" in _plain_output(result) - - -@pytest.mark.asyncio -async def test_create_server_accepts_injected_servicer_and_serves_loopback_rpcs() -> ( - None -): - class FalseyMiddleware(PrivacyGuardMiddleware): - def __bool__(self) -> bool: - return False - - @override - async def Describe( - self, - request: object, - context: grpc.aio.ServicerContext[object, pb2.MiddlewareManifest], - ) -> pb2.MiddlewareManifest: - manifest = await super().Describe(request, context) - manifest.name = "injected-falsey-middleware" - return manifest - - scanner = DeterministicEmailScanner( - ScannerConfig(name="test_email", entity_types=frozenset({"email"})) - ) - middleware = FalseyMiddleware(RequestProcessor([scanner])) - server = create_server(middleware) - port = server.add_insecure_port("127.0.0.1:0") - assert port != 0 - await server.start() - - try: - async with grpc.aio.insecure_channel(f"127.0.0.1:{port}") as channel: - stub = pb2_grpc.SupervisorMiddlewareStub(channel) - empty_message_type: object = message_factory.GetMessageClass( - empty_pb2.DESCRIPTOR.message_types_by_name["Empty"] - ) - if not isinstance(empty_message_type, type): - raise AssertionError("Empty factory returned a non-type") - empty_message: object = empty_message_type() - if not isinstance(empty_message, Message): - raise AssertionError("Empty factory returned a non-message") - if ( - empty_message.DESCRIPTOR - is not empty_pb2.DESCRIPTOR.message_types_by_name["Empty"] - ): - raise AssertionError("Empty factory returned the wrong message type") - manifest = await stub.Describe(empty_message) - result = await stub.EvaluateHttpRequest( - pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - body=b'{"message":"user@example.com"}', - ) - ) - - assert manifest.name == "injected-falsey-middleware" - assert result.decision == pb2.DECISION_ALLOW - assert result.has_body is True - assert result.body == b'{"message":"[email]"}' - finally: - await server.stop(grace=0) - await middleware.close() - - -@pytest.mark.asyncio -async def test_loopback_accepts_body_at_advertised_limit_and_rejects_larger_body() -> ( - None -): - middleware = _middleware() - server = create_server(middleware) - port = server.add_insecure_port("127.0.0.1:0") - assert port != 0 - await server.start() - - try: - async with grpc.aio.insecure_channel(f"127.0.0.1:{port}") as channel: - stub = pb2_grpc.SupervisorMiddlewareStub(channel) - at_limit = b'"' + (b"x" * (MAX_BODY_BYTES - 2)) + b'"' - allowed = await stub.EvaluateHttpRequest( - pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - body=at_limit, - ) - ) - - assert allowed.decision == pb2.DECISION_ALLOW - - with pytest.raises(grpc.aio.AioRpcError) as exception_info: - await stub.EvaluateHttpRequest( - pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - body=at_limit + b" ", - ) - ) - - assert exception_info.value.code() is grpc.StatusCode.INVALID_ARGUMENT - details = exception_info.value.details() - assert details is not None - assert ErrorCode.REQUEST_BODY_TOO_LARGE.value in details - finally: - await server.stop(grace=0) - await middleware.close() @pytest.mark.asyncio -async def test_loopback_real_findings_cover_observe_redact_and_block() -> None: - scanner = DeterministicEmailScanner( - ScannerConfig(name="test_email", entity_types=frozenset({"email"})) - ) - middleware = PrivacyGuardMiddleware(RequestProcessor([scanner])) - server = create_server(middleware) - port = server.add_insecure_port("127.0.0.1:0") - await server.start() - body = b'{"first":"a@example.com","second":"b@example.com"}' - try: - async with grpc.aio.insecure_channel(f"127.0.0.1:{port}") as channel: - stub = pb2_grpc.SupervisorMiddlewareStub(channel) - observe = await stub.EvaluateHttpRequest( - pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - config={"on_finding": {"action": "observe"}}, - body=body, - ) - ) - redact = await stub.EvaluateHttpRequest( - pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - config={"on_finding": {"action": "redact"}}, - body=body, - ) - ) - block = await stub.EvaluateHttpRequest( - pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - config={"on_finding": {"action": "block"}}, - body=body, - ) - ) - - assert observe.decision == pb2.DECISION_ALLOW - assert not observe.has_body - assert [(item.type, item.label, item.count) for item in observe.findings] == [ - ("test_email", "email", 2) - ] - assert redact.decision == pb2.DECISION_ALLOW - assert redact.has_body - assert redact.body == b'{"first":"[email]","second":"[email]"}' - assert redact.findings[0].count == 2 - assert block.decision == pb2.DECISION_DENY - assert block.reason_code == "privacy_guard_blocked" - assert not block.has_body and block.body == b"" - assert block.findings[0].count == 2 - finally: - await server.stop(grace=0) - await middleware.close() - - -@pytest.mark.asyncio -async def test_serve_rejects_bind_failure_and_stops_server( +@pytest.mark.parametrize( + ("fake_server", "sensitive_address"), + [ + (LifecycleServerFake(bound_port=0), "invalid-sensitive-listen-8472"), + ( + LifecycleServerFake( + bind_error=RuntimeError("invalid-sensitive-listen-9472") + ), + "invalid-sensitive-listen-9472", + ), + ], +) +async def test_serve_sanitizes_bind_failures_and_closes_resources( monkeypatch: pytest.MonkeyPatch, + fake_server: LifecycleServerFake, + sensitive_address: str, ) -> None: - class BindFailureServer(LifecycleServerFake): - @override - def add_insecure_port(self, address: str) -> int: - return 0 - - @override - async def start(self) -> None: - raise AssertionError("start must not run after bind failure") + closed: list[PrivacyGuardMiddleware] = [] - @override - async def wait_for_termination(self, timeout: float | None = None) -> bool: - raise AssertionError("wait must not run after bind failure") + async def record_close(servicer: PrivacyGuardMiddleware) -> None: + closed.append(servicer) - fake_server = BindFailureServer() - monkeypatch.setattr( - server_module, - "create_server", - lambda _: fake_server, - ) + middleware = _middleware() + monkeypatch.setattr(server_module, "create_server", lambda _: fake_server) + monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - with pytest.raises(PrivacyGuardError) as exception_info: - await serve(_middleware(), "invalid-sensitive-listen-8472") + with pytest.raises(PrivacyGuardError) as captured: + await serve(middleware, sensitive_address) - assert exception_info.value.code is ErrorCode.SERVER_BIND_FAILED - assert "Hint:" in str(exception_info.value) - assert "8472" not in str(exception_info.value) - assert fake_server.stopped is True + assert captured.value.code is ErrorCode.SERVER_BIND_FAILED + assert captured.value.__cause__ is None + assert sensitive_address not in str(captured.value) + assert fake_server.started is False + assert fake_server.waited is False + assert fake_server.stop_graces == [0] + assert closed == [middleware] @pytest.mark.asyncio -async def test_serve_translates_runtime_bind_failure_and_stops_server( +async def test_serve_starts_waits_and_closes_on_normal_termination( monkeypatch: pytest.MonkeyPatch, ) -> None: - class BindFailureServer(LifecycleServerFake): - @override - def add_insecure_port(self, address: str) -> int: - raise RuntimeError("invalid-sensitive-listen-8472") - - @override - async def start(self) -> None: - raise AssertionError("start must not run after bind failure") + fake_server = LifecycleServerFake() + closed: list[PrivacyGuardMiddleware] = [] - @override - async def wait_for_termination(self, timeout: float | None = None) -> bool: - raise AssertionError("wait must not run after bind failure") + async def record_close(servicer: PrivacyGuardMiddleware) -> None: + closed.append(servicer) - fake_server = BindFailureServer() + middleware = _middleware() monkeypatch.setattr(server_module, "create_server", lambda _: fake_server) + monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - with pytest.raises(PrivacyGuardError) as exception_info: - await serve(_middleware(), "invalid-sensitive-listen-8472") + await serve(middleware, "127.0.0.1:50053") - assert exception_info.value.code is ErrorCode.SERVER_BIND_FAILED - assert exception_info.value.__cause__ is None - assert "8472" not in str(exception_info.value) - assert fake_server.stopped is True + assert fake_server.addresses == ["127.0.0.1:50053"] + assert fake_server.started is True + assert fake_server.waited is True + assert fake_server.stop_graces == [0] + assert closed == [middleware] @pytest.mark.asyncio -async def test_startup_failure_stops_server_and_propagates( +async def test_serve_closes_resources_when_startup_fails( monkeypatch: pytest.MonkeyPatch, ) -> None: - class StartFailureServer(LifecycleServerFake): - @override - def add_insecure_port(self, address: str) -> int: - return 12345 - - @override - async def start(self) -> None: - raise RuntimeError("startup failed") + fake_server = LifecycleServerFake(start_error=RuntimeError("startup failed")) + closed: list[PrivacyGuardMiddleware] = [] - @override - async def wait_for_termination(self, timeout: float | None = None) -> bool: - raise AssertionError("wait must not run after startup failure") + async def record_close(servicer: PrivacyGuardMiddleware) -> None: + closed.append(servicer) - fake_server = StartFailureServer() - monkeypatch.setattr( - server_module, - "create_server", - lambda _: fake_server, - ) + middleware = _middleware() + monkeypatch.setattr(server_module, "create_server", lambda _: fake_server) + monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) with pytest.raises(RuntimeError, match="startup failed"): - await serve(_middleware(), "127.0.0.1:12345") + await serve(middleware, "127.0.0.1:50054") - assert fake_server.stopped is True + assert fake_server.waited is False + assert fake_server.stop_graces == [0] + assert closed == [middleware] diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index ca3d58c..3af4052 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -1,508 +1,132 @@ -import json -import traceback -from collections.abc import Mapping -from typing import Never +"""Service boundary tests over the canonical OpenShell-owned protobuf.""" + +from __future__ import annotations + +import asyncio -import grpc import pytest -from pydantic import ValidationError -from typing_extensions import override +from google.protobuf import json_format +from google.protobuf.message import Message -import privacy_guard.service.servicer as servicer_module from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 -from privacy_guard.config import PolicyAction, PolicyConfig -from privacy_guard.constants import ( - BLOCK_REASON, - BLOCK_REASON_CODE, - MAX_BODY_BYTES, - SERVICE_NAME, - SERVICE_VERSION, -) from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.payloads import ( - InterceptedRequest, - ProcessingDecision, - ProcessingResult, -) -from privacy_guard.processor import RequestProcessor -from privacy_guard.scanners import ( - Finding, - RequestBodyFinding, - ScanBudget, - Scanner, - ScannerConfig, -) +from privacy_guard.service.server import create_default_registry from privacy_guard.service.servicer import PrivacyGuardMiddleware -from ..scanner_helpers import DeterministicEmailScanner - - -class AbortedRpc(Exception): - pass - - -class RecordingContext: - def __init__(self) -> None: - self.code: grpc.StatusCode | None = None - self.details: str | None = None - async def abort(self, code: grpc.StatusCode, details: str) -> Never: - self.code = code - self.details = details - raise AbortedRpc - - -class FakeProcessor: - def __init__(self, result: ProcessingResult | None = None) -> None: - self.result = result or ProcessingResult(decision=ProcessingDecision.ALLOW) - self.validated_configs: list[PolicyConfig] = [] - self.requests: list[InterceptedRequest] = [] - - def __bool__(self) -> bool: - return False - - def validate_policy_config(self, policy_config: PolicyConfig) -> None: - self.validated_configs.append(policy_config) - - def process(self, request: InterceptedRequest) -> ProcessingResult: - self.requests.append(request) - return self.result +def _values(action: str = "replace") -> dict[str, object]: + return { + "entity_processing": { + "stages": [ + { + "config": { + "engine": "regex", + "pattern_catalog": { + "entities": [ + { + "name": "email", + "patterns": [ + { + "pattern": r"[a-z]+@[a-z]+\.[a-z]+", + "confidence": "high", + } + ], + } + ] + }, + "replacement": { + "strategy": "template", + "template": "[{entity}]", + }, + } + } + ] + }, + "on_detection": {"action": action}, + } -def _assert_safe_translation(error: PrivacyGuardError, sentinel: str) -> None: - assert error.__cause__ is None - assert sentinel not in str(error) - assert sentinel not in repr(error) - assert sentinel not in repr(error.args) - assert sentinel not in "".join(traceback.format_exception(error)) +def _proto_config(values: dict[str, object]) -> Message: + result = pb2.ValidateConfigRequest().config + json_format.ParseDict(values, result) + return result -def _evaluation( - *, - body: bytes = b'{"message":"hello"}', - config: Mapping[str, object] | None = None, - phase: pb2.SupervisorMiddlewarePhase = ( - pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS - ), - headers: list[pb2.HttpHeader] | None = None, -) -> pb2.HttpRequestEvaluation: +def _request(body: bytes, *, action: str = "replace") -> pb2.HttpRequestEvaluation: return pb2.HttpRequestEvaluation( - phase=phase, - config=config or {}, - headers=headers or [], + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + config=_proto_config(_values(action)), body=body, ) -def _middleware() -> PrivacyGuardMiddleware: - scanner = DeterministicEmailScanner( - ScannerConfig(name="test_email", entity_types=frozenset({"email"})) - ) - return PrivacyGuardMiddleware(RequestProcessor([scanner])) - - -def test_describe_advertises_one_pre_credentials_http_binding() -> None: - manifest = _middleware()._describe() - - assert manifest.name == SERVICE_NAME == "privacy-guard" - assert manifest.service_version == SERVICE_VERSION == "0.1.0" - assert len(manifest.bindings) == 1 - binding = manifest.bindings[0] - assert binding.operation == pb2.SUPERVISOR_MIDDLEWARE_OPERATION_HTTP_REQUEST - assert binding.phase == pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS - assert binding.max_body_bytes == MAX_BODY_BYTES == 4 * 1024 * 1024 - assert binding.timeout == "" - - -@pytest.mark.parametrize("action", [None, *PolicyAction]) -def test_validate_config_accepts_defaults_and_actions( - action: PolicyAction | None, -) -> None: - values: dict[str, object] = {} - if action is not None: - values["on_finding"] = {"action": action.value} - - response = _middleware()._validate_config(pb2.ValidateConfigRequest(config=values)) - - assert response.valid is True - assert response.reason == "" - - -@pytest.mark.parametrize( - "config", - [ - {"unknown": "sensitive-config-value-8472"}, - {"on_finding": {"action": "invalid-sensitive-action-8472"}}, - {"debug_inject_text": "sensitive-injection-value-8472"}, - {"body_format": "unknown-sensitive-format-8472"}, - { - "on_finding": { - "action": "redact", - "entity_types": ["sensitive-entity-typo-8472"], - } - }, - ], -) -def test_validate_config_rejects_malformed_values_without_echoing_them( - config: Mapping[str, object], -) -> None: - response = _middleware()._validate_config(pb2.ValidateConfigRequest(config=config)) - - assert response.valid is False - assert "Hint:" in response.reason - assert "8472" not in response.reason - - -def test_proto_policy_translation_discards_sensitive_exception_chain( - monkeypatch: pytest.MonkeyPatch, -) -> None: - sentinel = "sensitive-proto-config-8472" - - def reject_proto(config: object) -> dict[str, object]: - raise ValueError(sentinel) - - monkeypatch.setattr(servicer_module.json_format, "MessageToDict", reject_proto) - - with pytest.raises(PrivacyGuardError) as exception_info: - servicer_module._policy_from_proto(pb2.ValidateConfigRequest().config) - - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - _assert_safe_translation(exception_info.value, sentinel) - - -def test_intercepted_request_translation_discards_sensitive_validation_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - sentinel = "sensitive-intercepted-request-8472" - validation_error = ValidationError.from_exception_data( - "InterceptedRequest", - [ - { - "type": "value_error", - "loc": ("raw_body",), - "input": sentinel, - "ctx": {"error": ValueError(sentinel)}, - } - ], - ) - - def reject_request(**values: object) -> Never: - raise validation_error - - monkeypatch.setattr(servicer_module, "InterceptedRequest", reject_request) - - with pytest.raises(PrivacyGuardError) as exception_info: - servicer_module._request_from_proto(_evaluation()) - - assert exception_info.value.code is ErrorCode.UNEXPECTED_SERVICE_FAILURE - _assert_safe_translation(exception_info.value, sentinel) - - -def test_validate_config_uses_domain_parser_and_processor_validation() -> None: - processor = FakeProcessor() - servicer = PrivacyGuardMiddleware(processor) - - response = servicer._validate_config( - pb2.ValidateConfigRequest(config={"on_finding": {"action": "observe"}}) - ) - - assert response.valid is True - assert processor.validated_configs[0].on_finding.action is PolicyAction.OBSERVE - - -@pytest.mark.asyncio -async def test_requests_without_findings_and_bodyless_requests_are_unchanged() -> None: - servicer = _middleware() - - ordinary = await servicer._evaluate_http_request(_evaluation()) - bodyless = await servicer._evaluate_http_request(_evaluation(body=b"")) - - for response in (ordinary, bodyless): - assert response.decision == pb2.DECISION_ALLOW - assert response.has_body is False - assert response.body == b"" - assert response.reason == "" - assert response.reason_code == "" - - -@pytest.mark.asyncio -async def test_body_larger_than_advertised_limit_is_rejected_before_processing() -> ( - None -): - processor = FakeProcessor() - context = RecordingContext() - servicer = PrivacyGuardMiddleware(processor) - - with pytest.raises(AbortedRpc): - await servicer._evaluate_rpc( - _evaluation(body=b"x" * (MAX_BODY_BYTES + 1)), - context, - ) - - assert context.code is grpc.StatusCode.INVALID_ARGUMENT - assert context.details is not None - assert ErrorCode.REQUEST_BODY_TOO_LARGE.value in context.details - assert processor.requests == [] - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("body", "expected"), - [ - ( - b'{"messages":[{"content":"user@example.com"}]}', - {"messages": [{"content": "[email]"}]}, - ), - ( - b'{"contents":[{"parts":[{"text":"user@example.com"}]}]}', - {"contents": [{"parts": [{"text": "[email]"}]}]}, - ), - ], -) -async def test_deterministic_finding_crosses_full_proto_domain_boundary( - body: bytes, expected: object -) -> None: - response = await PrivacyGuardMiddleware( - RequestProcessor( - [ - DeterministicEmailScanner( - ScannerConfig(name="test_email", entity_types=frozenset({"email"})) - ) - ] - ) - )._evaluate_http_request(_evaluation(body=body)) - - assert response.decision == pb2.DECISION_ALLOW - assert response.has_body is True - assert json.loads(response.body) == expected - - -@pytest.mark.asyncio -async def test_first_content_type_header_is_passed_to_processor() -> None: - processor = FakeProcessor() - servicer = PrivacyGuardMiddleware(processor) - - await servicer._evaluate_http_request( - _evaluation( - headers=[ - pb2.HttpHeader(name="content-type", value="first/type"), - pb2.HttpHeader(name="content-type", value="second/type"), - ] - ) - ) - - assert processor.requests[0].content_type == "first/type" - - -@pytest.mark.asyncio -async def test_explicit_empty_replacement_sets_has_body() -> None: - processor = FakeProcessor( - ProcessingResult(decision=ProcessingDecision.ALLOW, replacement_body=b"") - ) - - response = await PrivacyGuardMiddleware(processor)._evaluate_http_request( - _evaluation() - ) - - assert response.decision == pb2.DECISION_ALLOW - assert response.has_body is True - assert response.body == b"" - - -@pytest.mark.asyncio -async def test_deny_has_stable_reason_code_and_never_returns_a_body() -> None: - processor = FakeProcessor( - ProcessingResult( - decision=ProcessingDecision.DENY, - replacement_body=None, - reason_code=BLOCK_REASON_CODE, - ) - ) - - response = await PrivacyGuardMiddleware(processor)._evaluate_http_request( - _evaluation() - ) - - assert response.decision == pb2.DECISION_DENY - assert response.reason == BLOCK_REASON - assert response.reason_code == BLOCK_REASON_CODE - assert response.has_body is False - assert response.body == b"" - - -@pytest.mark.asyncio -async def test_findings_aggregate_in_first_observed_group_order() -> None: - findings = tuple( - RequestBodyFinding(finding=finding, text_block_path=path) - for finding, path in ( - ( - Finding( - entity="email", - scanner_name="scanner-a", - start_offset=0, - end_offset=1, - ), - "a", - ), - ( - Finding( - entity="token", - scanner_name="scanner-b", - start_offset=1, - end_offset=2, - ), - "a", - ), - ( - Finding( - entity="email", - scanner_name="scanner-a", - start_offset=2, - end_offset=3, - ), - "b", - ), - ) - ) - processor = FakeProcessor( - ProcessingResult(decision=ProcessingDecision.ALLOW, findings=findings) - ) +def test_copied_proto_remains_the_current_openshell_contract() -> None: + evaluation = pb2.HttpRequestEvaluation() + finding = pb2.Finding() - response = await PrivacyGuardMiddleware(processor)._evaluate_http_request( - _evaluation() - ) + assert isinstance(evaluation.config, Message) + assert not hasattr(evaluation, "config_fingerprint") + assert not hasattr(finding, "source") - assert [ - (finding.type, finding.label, finding.count) for finding in response.findings - ] == [ - ("scanner-a", "email", 2), - ("scanner-b", "token", 1), - ] - assert all(finding.confidence == "high" for finding in response.findings) - assert all(not finding.severity for finding in response.findings) +def test_validate_config_is_pure_and_reports_invalid_config() -> None: + middleware = PrivacyGuardMiddleware(create_default_registry()) -@pytest.mark.asyncio -async def test_pattern_metadata_is_included_in_aggregate_label() -> None: - finding = RequestBodyFinding( - finding=Finding( - entity="email", - metadata={"pattern_name": "common-email"}, - scanner_name="regex", - start_offset=0, - end_offset=1, - ), - text_block_path="path", + valid = middleware._validate_config( + pb2.ValidateConfigRequest(config=_proto_config(_values())) ) - processor = FakeProcessor( - ProcessingResult(decision=ProcessingDecision.ALLOW, findings=(finding,)) + invalid = middleware._validate_config( + pb2.ValidateConfigRequest(config=_proto_config({"on_detection": {}})) ) - response = await PrivacyGuardMiddleware(processor)._evaluate_http_request( - _evaluation() - ) - - assert [(item.type, item.label) for item in response.findings] == [ - ("regex", "email/common-email") - ] - - -@pytest.mark.asyncio -async def test_more_than_32_finding_groups_stably_denies() -> None: - findings = tuple( - RequestBodyFinding( - finding=Finding( - entity=f"entity-{index}", - scanner_name="scanner", - start_offset=0, - end_offset=1, - ), - text_block_path="path", - ) - for index in range(33) - ) - processor = FakeProcessor( - ProcessingResult(decision=ProcessingDecision.ALLOW, findings=findings) - ) - response = await PrivacyGuardMiddleware(processor)._evaluate_http_request( - _evaluation() - ) - - assert response.decision == pb2.DECISION_DENY - assert response.reason_code == "privacy_guard_limit_exceeded" - assert not response.findings - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("evaluation", "expected_code"), - [ - ( - _evaluation(phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_UNSPECIFIED), - ErrorCode.REQUEST_PHASE_INVALID, - ), - (_evaluation(body=b'"bad\xff"'), ErrorCode.BODY_ENCODING_INVALID), - (_evaluation(body=b'{"bad":}'), ErrorCode.BODY_JSON_INVALID), - (_evaluation(config={"body_format": "xml"}), ErrorCode.BODY_FORMAT_UNSUPPORTED), - ], -) -async def test_invalid_input_maps_to_invalid_argument_with_safe_catalog_details( - evaluation: pb2.HttpRequestEvaluation, expected_code: ErrorCode -) -> None: - context = RecordingContext() - - with pytest.raises(AbortedRpc): - await _middleware()._evaluate_rpc(evaluation, context) - - assert context.code is grpc.StatusCode.INVALID_ARGUMENT - assert context.details is not None - assert expected_code.value in context.details - assert "Hint:" in context.details - assert "8472" not in context.details - - -@pytest.mark.asyncio -async def test_processor_runtime_failure_maps_to_internal_catalog_error() -> None: - class RaisingProcessor(FakeProcessor): - @override - def process(self, request: InterceptedRequest) -> ProcessingResult: - raise RuntimeError("sensitive-runtime-failure-8472") + assert valid.valid is True + assert invalid.valid is False + assert "config_invalid" in invalid.reason - context = RecordingContext() - with pytest.raises(AbortedRpc): - await PrivacyGuardMiddleware(RaisingProcessor())._evaluate_rpc( - _evaluation(), context - ) +def test_evaluation_decodes_one_utf8_text_and_encodes_replacement() -> None: + async def evaluate() -> pb2.HttpRequestResult: + middleware = PrivacyGuardMiddleware(create_default_registry()) + try: + return await middleware._evaluate_http_request(_request(b"email a@b.com")) + finally: + await middleware.close() - assert context.code is grpc.StatusCode.INTERNAL - assert context.details is not None - assert ErrorCode.UNEXPECTED_SERVICE_FAILURE.value in context.details - assert "8472" not in context.details + result = asyncio.run(evaluate()) + assert result.decision == pb2.DECISION_ALLOW + assert result.has_body is True + assert result.body == b"email [email]" + assert len(result.findings) == 1 + assert result.findings[0].type == "detected_entity" + assert result.findings[0].label == "email (regex[1])" -@pytest.mark.asyncio -async def test_cataloged_scanner_failure_maps_to_internal_status() -> None: - class RaisingScanner(Scanner[ScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - raise RuntimeError("sensitive-scanner-failure-8472") - context = RecordingContext() - scanner = RaisingScanner(ScannerConfig(name="raising", entity_types=frozenset())) - servicer = PrivacyGuardMiddleware(RequestProcessor([scanner])) +def test_invalid_utf8_fails_before_invoking_an_engine() -> None: + async def evaluate() -> None: + middleware = PrivacyGuardMiddleware(create_default_registry()) + try: + with pytest.raises(PrivacyGuardError) as captured: + await middleware._evaluate_http_request(_request(b"\xff")) + assert captured.value.code is ErrorCode.BODY_ENCODING_INVALID + finally: + await middleware.close() - with pytest.raises(AbortedRpc): - await servicer._evaluate_rpc(_evaluation(), context) + asyncio.run(evaluate()) - assert context.code is grpc.StatusCode.INTERNAL - assert context.details is not None - assert ErrorCode.SCANNER_EXECUTION_FAILED.value in context.details - assert "8472" not in context.details +def test_detect_returns_no_body_mutation() -> None: + async def evaluate() -> pb2.HttpRequestResult: + middleware = PrivacyGuardMiddleware(create_default_registry()) + try: + return await middleware._evaluate_http_request( + _request(b"a@b.com", action="detect") + ) + finally: + await middleware.close() -def test_servicer_repr_does_not_retain_request_or_config_content() -> None: - servicer = _middleware() + result = asyncio.run(evaluate()) - assert "sensitive-body-8472" not in repr(servicer) - assert "sensitive-injection-8472" not in repr(servicer) + assert result.decision == pb2.DECISION_ALLOW + assert result.has_body is False + assert result.body == b"" diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py index ffd21df..3cfd74b 100644 --- a/projects/privacy-guard/tests/test_config.py +++ b/projects/privacy-guard/tests/test_config.py @@ -1,369 +1,264 @@ +from __future__ import annotations + +from collections.abc import Callable +from copy import deepcopy + import pytest from pydantic import ValidationError from privacy_guard.config import ( - ActionConfig, - BlockActionConfig, - ObserveActionConfig, PolicyAction, - PolicyActionConfig, - PolicyConfig, - RedactActionConfig, + configuration_fingerprint, ) -from privacy_guard.constants import MAX_SCANNER_METADATA_BYTES -from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.request_body import select_format_handler -from privacy_guard.scanners import Confidence -from privacy_guard.validation import ( - BoundedMetadataString, - NonEmptyScalarString, - ScalarString, - StrictDomainModel, +from privacy_guard.engine_registry import EngineRegistry +from privacy_guard.engines import ( + RegexEngine, + RegexEngineConfig, + RegexPatternCatalog, ) +from privacy_guard.errors import ErrorCode, PrivacyGuardError -class _ValidationFixture(StrictDomainModel): - scalar: ScalarString - non_empty: NonEmptyScalarString - metadata: BoundedMetadataString - - -class _InvalidDefaultFixture(StrictDomainModel): - value: NonEmptyScalarString = "" - - -def _wire_on_finding(action: str, **values: object) -> dict[str, object]: - return {"on_finding": {"action": action, **values}} - - -def test_defaults_use_a_complete_redact_action() -> None: - config = PolicyConfig() +def _registry() -> EngineRegistry: + registry = EngineRegistry() + registry.register(RegexEngine) + registry.finalize() + return registry + + +def _config( + *, + action: str = "detect", + replacement: dict[str, object] | None = None, + stage_name: str | None = None, +): + engine_config = { + "engine": "regex", + "pattern_catalog": { + "entities": [ + { + "name": "email", + "patterns": [ + { + "pattern": r"\buser@example\.com\b", + "confidence": "high", + } + ], + } + ] + }, + } + if replacement is not None: + engine_config["replacement"] = replacement + stage = {"config": engine_config} + if stage_name is not None: + stage["name"] = stage_name + return { + "entity_processing": {"stages": [stage]}, + "on_detection": {"action": action}, + } - assert config.body_format == "json" - assert type(config.on_finding) is RedactActionConfig - assert config.on_finding.action is PolicyAction.REDACT - assert config.on_finding.entity_types is None - assert config.on_finding.minimum_confidence is None - assert config.on_finding.template == "[{entity}]" - assert [action.value for action in PolicyAction] == ["observe", "redact", "block"] +@pytest.mark.parametrize("action", list(PolicyAction)) +def test_policy_action_uses_detect_block_replace(action: PolicyAction) -> None: + replacement: dict[str, object] | None = ( + {"strategy": "template", "template": "[{entity}]"} + if action is PolicyAction.REPLACE + else None + ) + config = _registry().validate_config( + _config(action=action.value, replacement=replacement) + ) -@pytest.mark.parametrize( - ("action", "expected_type"), - [ - (PolicyAction.OBSERVE, ObserveActionConfig), - (PolicyAction.REDACT, RedactActionConfig), - (PolicyAction.BLOCK, BlockActionConfig), - ], -) -def test_wire_parser_selects_discriminated_action_model( - action: PolicyAction, expected_type: type[ActionConfig] -) -> None: - config = PolicyConfig.from_mapping(_wire_on_finding(action.value)) + assert config.on_detection.action is action + assert [item.value for item in PolicyAction] == ["detect", "block", "replace"] - assert type(config.on_finding) is expected_type - assert config.on_finding.action is action +def test_known_discriminator_constructs_the_exact_engine_config() -> None: + config = _registry().validate_config(_config()) + stage = config.entity_processing.stages[0] -@pytest.mark.parametrize( - "action", - [ - ObserveActionConfig(), - BlockActionConfig( - entity_types=frozenset({"email"}), - minimum_confidence=Confidence.HIGH, - ), - RedactActionConfig(template="[{entity}]"), - ], -) -def test_discriminated_action_serialization_round_trips( - action: PolicyActionConfig, -) -> None: - serialized = PolicyConfig(on_finding=action).model_dump(mode="json") - reparsed = PolicyConfig.from_mapping(serialized) + assert type(stage.config) is RegexEngineConfig + assert type(stage.config.pattern_catalog) is RegexPatternCatalog + assert stage.config.pattern_catalog.entities[0].patterns[0].name is None + assert stage.diagnostic_name(1) == "regex[1]" - assert serialized["on_finding"]["action"] == action.action.value - assert type(reparsed.on_finding) is type(action) - assert reparsed == PolicyConfig(on_finding=action) +def test_explicit_stage_name_is_the_diagnostic_source() -> None: + config = _registry().validate_config(_config(stage_name="credentials")) -def test_on_finding_schema_declares_action_discriminator() -> None: - action_schema = PolicyConfig.model_json_schema()["properties"]["on_finding"] + assert config.entity_processing.stages[0].diagnostic_name(1) == "credentials" - assert action_schema["discriminator"] == { - "propertyName": "action", - "mapping": { - "observe": "#/$defs/ObserveActionConfig", - "block": "#/$defs/BlockActionConfig", - "redact": "#/$defs/RedactActionConfig", - }, - } - -@pytest.mark.parametrize("confidence", list(Confidence)) -def test_wire_parser_accepts_confidence_strings(confidence: Confidence) -> None: - config = PolicyConfig.from_mapping( - _wire_on_finding("observe", minimum_confidence=confidence.value) +def test_discriminated_union_round_trip_preserves_concrete_fields() -> None: + registry = _registry() + parsed = registry.validate_config( + _config( + action="replace", + replacement={"strategy": "template", "template": "[{entity}]"}, + ) + ) + serialized = parsed.model_dump(mode="json") + reparsed = registry.validate_config(serialized) + + assert type(reparsed.entity_processing.stages[0].config) is RegexEngineConfig + assert reparsed == parsed + assert serialized["entity_processing"]["stages"][0]["config"]["engine"] == "regex" + assert ( + serialized["entity_processing"]["stages"][0]["config"]["replacement"][ + "strategy" + ] + == "template" ) - - assert config.on_finding.minimum_confidence is confidence -def test_null_minimum_confidence_selects_every_confidence() -> None: - config = PolicyConfig.from_mapping( - _wire_on_finding("observe", minimum_confidence=None) +def test_generated_schema_declares_the_engine_discriminator() -> None: + schema = _registry().configuration_json_schema() + definitions = _required_dict(schema, "$defs") + stage_definition = next( + definition + for name, definition in definitions.items() + if isinstance(name, str) and name.startswith("EntityProcessingStage") ) + properties = _required_dict(stage_definition, "properties") + config_schema = _required_dict(properties, "config") - assert config.on_finding.minimum_confidence is None + assert config_schema["discriminator"] == { + "mapping": {"regex": "#/$defs/RegexEngineConfig"}, + "propertyName": "engine", + } -@pytest.mark.parametrize( - "values", - [ - {"unknown": "value"}, - {"action": {"kind": "observe"}}, - {"on_finding": "observe"}, - {"on_finding": {"kind": "observe"}}, - {"on_finding": {"action": "audit"}}, - {"on_finding": {}}, - {"body_format": ""}, - {"debug_inject_path": "/message"}, - {"debug_inject_text": " suffix"}, - {"redaction_template": "[redacted]"}, - {"entity_types": ["email"]}, - {"minimum_confidence": "high"}, - ], -) -def test_rejects_invalid_or_misplaced_fields(values: dict[str, object]) -> None: - with pytest.raises(PrivacyGuardError) as exception_info: - PolicyConfig.from_mapping(values) +def _required_dict(mapping: object, key: str): + assert isinstance(mapping, dict) + value = mapping.get(key) + assert isinstance(value, dict) + return value - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - assert exception_info.value.__cause__ is None +def test_runtime_config_rejects_a_catalog_file_path() -> None: + values = _config() + values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( + "./patterns.yaml" + ) -@pytest.mark.parametrize( - "values", - [ - {"body_format": True}, - _wire_on_finding("redact", template=True), - _wire_on_finding("observe", minimum_confidence=True), - {"on_finding": {"action": True}}, - ], -) -def test_does_not_coerce_wrong_scalar_types(values: dict[str, object]) -> None: with pytest.raises(PrivacyGuardError) as exception_info: - PolicyConfig.from_mapping(values) + _registry().validate_config(values) assert exception_info.value.code is ErrorCode.CONFIG_INVALID -@pytest.mark.parametrize( - "values", - [ - {"body_format": "safe\ud800sentinel"}, - {"on_finding": {"action": "safe\ud800sentinel"}}, - _wire_on_finding("redact", template="safe\ud800sentinel"), - _wire_on_finding("observe", minimum_confidence="safe\ud800sentinel"), - ], -) -def test_rejects_unpaired_unicode_surrogates(values: dict[str, object]) -> None: +def test_replace_requires_a_replacement_recipe_on_every_stage() -> None: with pytest.raises(PrivacyGuardError) as exception_info: - PolicyConfig.from_mapping(values) + _registry().validate_config(_config(action="replace")) assert exception_info.value.code is ErrorCode.CONFIG_INVALID -def test_accepts_action_finding_criteria() -> None: - config = PolicyConfig.from_mapping( - _wire_on_finding( - "block", - entity_types=["email", "api_key"], - minimum_confidence="high", +@pytest.mark.parametrize("action", ["detect", "block"]) +def test_dormant_replacement_recipe_is_valid_for_detection_only_actions( + action: str, +) -> None: + config = _registry().validate_config( + _config( + action=action, + replacement={"strategy": "template", "template": "[redacted]"}, ) ) - assert config.on_finding.entity_types == frozenset({"email", "api_key"}) - assert config.on_finding.minimum_confidence is Confidence.HIGH - - -@pytest.mark.parametrize( - "values", - [ - {"on_finding": PolicyAction.BLOCK}, - _wire_on_finding("block", entity_types=("email",)), - _wire_on_finding("block", entity_types={"email"}), - _wire_on_finding("block", entity_types="email"), - _wire_on_finding("block", entity_types=["email", 1]), - ], -) -def test_wire_parser_rejects_non_wire_forms(values: dict[str, object]) -> None: - with pytest.raises(PrivacyGuardError) as exception_info: - PolicyConfig.from_mapping(values) - - assert exception_info.value.code is ErrorCode.CONFIG_INVALID + assert config.entity_processing.stages[0].config.replacement is not None @pytest.mark.parametrize( - "values", + "mutation", [ - [("on_finding", {"action": "block"})], - (("on_finding", {"action": "block"}),), - iter([("on_finding", {"action": "block"})]), + lambda values: values.update({"body_format": "json"}), + lambda values: values.update({"on_finding": {"action": "observe"}}), + lambda values: values["on_detection"].update({"action": "observe"}), + lambda values: values["on_detection"].update({"action": "redact"}), + lambda values: values["entity_processing"]["stages"][0]["config"].update( + {"kind": "regex"} + ), + lambda values: values["entity_processing"]["stages"][0]["config"].update( + {"preset": "pii"} + ), ], ) -def test_wire_parser_rejects_non_mapping_pair_iterables(values: object) -> None: - with pytest.raises(PrivacyGuardError) as exception_info: - PolicyConfig.from_mapping(values) - - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - assert exception_info.value.__cause__ is None - - -def test_direct_construction_is_strict_model_to_model_use() -> None: - action = BlockActionConfig( - entity_types=frozenset({"email"}), - minimum_confidence=Confidence.HIGH, - ) - config = PolicyConfig(on_finding=action) - - assert config.on_finding is action - - discriminated = PolicyConfig.model_validate({"on_finding": {"action": "block"}}) - assert type(discriminated.on_finding) is BlockActionConfig - with pytest.raises(ValidationError): - BlockActionConfig.model_validate({"entity_types": ["email"]}) - with pytest.raises(ValidationError): - BlockActionConfig.model_validate({"minimum_confidence": "high"}) - - -def test_template_is_valid_only_for_redact_action() -> None: - redact = PolicyConfig.from_mapping( - _wire_on_finding("redact", template="[redacted]") - ) - - assert type(redact.on_finding) is RedactActionConfig - assert redact.on_finding.template == "[redacted]" - for action in ("observe", "block"): - with pytest.raises(PrivacyGuardError): - PolicyConfig.from_mapping(_wire_on_finding(action, template="[redacted]")) - +def test_removed_or_unknown_policy_fields_are_rejected( + mutation: Callable[[dict[str, object]], None], +) -> None: + values = _config() + mutation(values) -@pytest.mark.parametrize( - "entities", - [ - [""], - ["email", "email"], - ["safe\ud800sentinel"], - ["x" * (MAX_SCANNER_METADATA_BYTES + 1)], - ["😀" * (MAX_SCANNER_METADATA_BYTES // 4 + 1)], - ], -) -def test_rejects_invalid_entity_names(entities: list[str]) -> None: with pytest.raises(PrivacyGuardError): - PolicyConfig.from_mapping(_wire_on_finding("observe", entity_types=entities)) + _registry().validate_config(values) -def test_none_selects_all_entities_and_empty_set_selects_none() -> None: - all_entities = PolicyConfig.from_mapping( - _wire_on_finding("observe", entity_types=None) +def test_stage_list_is_non_empty_and_explicit_names_are_unique() -> None: + empty = _config() + empty["entity_processing"]["stages"] = [] + duplicate = _config(stage_name="same") + duplicate["entity_processing"]["stages"].append( + deepcopy(duplicate["entity_processing"]["stages"][0]) ) - no_entities = PolicyConfig.from_mapping( - _wire_on_finding("observe", entity_types=[]) - ) - - assert all_entities.on_finding.entity_types is None - assert no_entities.on_finding.entity_types == frozenset() + with pytest.raises(PrivacyGuardError): + _registry().validate_config(empty) + with pytest.raises(PrivacyGuardError): + _registry().validate_config(duplicate) -def test_accepts_entity_name_at_exact_utf8_metadata_limit() -> None: - entity = "😀" * (MAX_SCANNER_METADATA_BYTES // 4) - config = PolicyConfig.from_mapping( - _wire_on_finding("observe", entity_types=[entity]) +def test_explicit_stage_name_cannot_collide_with_a_derived_name() -> None: + values = _config(stage_name="regex[2]") + values["entity_processing"]["stages"].append( + deepcopy(_config()["entity_processing"]["stages"][0]) ) - assert config.on_finding.entity_types == frozenset({entity}) - - -def test_shared_validation_types_are_strict_scalar_safe_and_validate_defaults() -> None: - fixture = _ValidationFixture( - scalar="", - non_empty="value", - metadata="metadata", + with pytest.raises(PrivacyGuardError): + _registry().validate_config(values) + + +def test_regex_pattern_names_are_optional_but_supplied_names_are_unique() -> None: + values = _config() + patterns = values["entity_processing"]["stages"][0]["config"]["pattern_catalog"][ + "entities" + ][0]["patterns"] + patterns.extend( + [ + {"pattern": "second", "confidence": "low"}, + {"name": "named", "pattern": "third", "confidence": "medium"}, + ] ) + config = _registry().validate_config(values) - assert fixture.scalar == "" - with pytest.raises(ValidationError): - _ValidationFixture.model_validate( - {"scalar": 1, "non_empty": "value", "metadata": "metadata"} - ) - with pytest.raises(ValidationError): - _ValidationFixture(scalar="\ud800", non_empty="value", metadata="metadata") - with pytest.raises(ValidationError): - _InvalidDefaultFixture() - + regex_config = config.entity_processing.stages[0].config + assert isinstance(regex_config, RegexEngineConfig) + parsed_patterns = regex_config.pattern_catalog.entities[0].patterns + assert [pattern.name for pattern in parsed_patterns] == [None, None, "named"] -@pytest.mark.parametrize("template", ["[redacted]", "[{entity}]", "{{{entity}}}"]) -def test_accepts_static_and_label_only_templates(template: str) -> None: - config = PolicyConfig.from_mapping(_wire_on_finding("redact", template=template)) + patterns.append({"name": "named", "pattern": "duplicate", "confidence": "high"}) + with pytest.raises(PrivacyGuardError): + _registry().validate_config(values) - assert type(config.on_finding) is RedactActionConfig - assert config.on_finding.template == template +def test_canonical_fingerprint_covers_concrete_expanded_config() -> None: + registry = _registry() + first = registry.validate_config(_config()) + equivalent = registry.validate_config(deepcopy(_config())) + changed_values = _config() + changed_values["entity_processing"]["stages"][0]["config"]["pattern_catalog"][ + "entities" + ][0]["patterns"][0]["confidence"] = "low" + changed = registry.validate_config(changed_values) -@pytest.mark.parametrize( - "template", - ["{unknown}", "{entity!r}", "{entity:>10}", "{", "}", "{}", "{0}"], -) -def test_rejects_unsafe_or_malformed_templates(template: str) -> None: - with pytest.raises(PrivacyGuardError) as exception_info: - PolicyConfig.from_mapping(_wire_on_finding("redact", template=template)) + assert configuration_fingerprint(first) == configuration_fingerprint(equivalent) + assert configuration_fingerprint(first) != configuration_fingerprint(changed) - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - -def test_models_are_frozen_and_sensitive_fields_are_hidden_from_repr() -> None: - sentinel = "sensitive-config-value-8472" - config = PolicyConfig.from_mapping( - { - "body_format": sentinel, - **_wire_on_finding( - "redact", - template=sentinel, - entity_types=[sentinel], - ), - } - ) +def test_models_are_frozen_and_hide_engine_configuration_from_repr() -> None: + config = _registry().validate_config(_config()) + pattern = "sensitive-pattern-value" with pytest.raises(ValidationError): - setattr(config, "body_format", "json") - with pytest.raises(ValidationError): - setattr(config.on_finding, "minimum_confidence", Confidence.HIGH) - assert sentinel not in repr(config) - assert sentinel not in repr(config.on_finding) - - -def test_validation_failure_does_not_leak_input_or_pydantic_error() -> None: - sentinel = "sensitive-invalid-config-value-8472" - - with pytest.raises(PrivacyGuardError) as exception_info: - PolicyConfig.from_mapping({"body_format": sentinel + "\ud800"}) - - error = exception_info.value - assert error.code is ErrorCode.CONFIG_INVALID - assert sentinel not in str(error) - assert sentinel not in repr(error) - assert sentinel not in repr(error.args) - assert error.__cause__ is None - - -def test_select_format_handler_error_does_not_leak_unknown_format() -> None: - sentinel = "sensitive-unknown-format-8472" - - with pytest.raises(PrivacyGuardError) as exception_info: - select_format_handler(sentinel) - - assert exception_info.value.code is ErrorCode.BODY_FORMAT_UNSUPPORTED - assert sentinel not in str(exception_info.value) - assert sentinel not in repr(exception_info.value) + setattr(config.on_detection, "action", PolicyAction.BLOCK) + assert pattern not in repr(config) diff --git a/projects/privacy-guard/tests/test_engine_registry.py b/projects/privacy-guard/tests/test_engine_registry.py new file mode 100644 index 0000000..2d3cdb6 --- /dev/null +++ b/projects/privacy-guard/tests/test_engine_registry.py @@ -0,0 +1,186 @@ +"""Tests for entity-processing engine registration and schema finalization.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Literal + +import pytest +from pydantic import field_validator + +from privacy_guard.base import StrictDomainModel +from privacy_guard.engine_registry import EngineRegistry, EngineRegistryError +from privacy_guard.engines import ( + EngineConfig, + EntityProcessingEngine, + EntityProcessingStrategy, + RegexEngine, + TextProcessingResult, +) +from privacy_guard.timeout import Timeout + + +class AcmeReplacement(StrictDomainModel): + strategy: Literal["token"] = "token" + + +class AcmeConfig(EngineConfig[AcmeReplacement]): + engine: Literal["acme-pii"] = "acme-pii" + entities: tuple[str, ...] + + @field_validator("entities", mode="before") + @classmethod + def _entities_are_a_tuple(cls, value: object) -> object: + if not isinstance(value, list | tuple): + raise ValueError("entities must be a list") + return tuple(value) + + +@dataclass(frozen=True) +class AcmeResources: + prefix: str + + +class AcmeEngine(EntityProcessingEngine[AcmeConfig, AcmeResources]): + supported_strategy = EntityProcessingStrategy.REPLACE + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + del strategy, timeout + return TextProcessingResult(text=text, detections=()) + + +class DetectConfig(EngineConfig[AcmeReplacement]): + engine: Literal["detect-only"] = "detect-only" + + +class DetectEngine(EntityProcessingEngine[DetectConfig, None]): + supported_strategy = EntityProcessingStrategy.DETECT + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + del strategy, timeout + return TextProcessingResult(text=text, detections=()) + + +def _acme_values(*, action: str = "detect") -> dict[str, object]: + return { + "entity_processing": { + "stages": [ + { + "config": { + "engine": "acme-pii", + "entities": ["account"], + "replacement": {"strategy": "token"}, + } + } + ] + }, + "on_detection": {"action": action}, + } + + +def test_custom_engine_config_joins_the_exact_discriminated_union() -> None: + resources = AcmeResources(prefix="token") + registry = EngineRegistry() + registry.register(RegexEngine) + registry.register(AcmeEngine, resources=resources) + registry.finalize() + + config = registry.validate_config(_acme_values(action="replace")) + engine = registry.create_engine(config.entity_processing.stages[0].config) + + assert type(config.entity_processing.stages[0].config) is AcmeConfig + assert type(engine) is AcmeEngine + assert engine.config is config.entity_processing.stages[0].config + assert engine.resources is resources + assert registry.engine_names == ("regex", "acme-pii") + + +def test_detection_only_engine_is_rejected_for_replace_action() -> None: + registry = EngineRegistry() + registry.register(DetectEngine) + registry.finalize() + values = { + "entity_processing": {"stages": [{"config": {"engine": "detect-only"}}]}, + "on_detection": {"action": "replace"}, + } + + with pytest.raises(Exception): + registry.validate_config(values) + + +def test_registry_is_frozen_after_finalize_and_finalize_is_idempotent() -> None: + registry = EngineRegistry() + registry.register(RegexEngine) + first_type = registry.finalize() + + assert registry.finalize() is first_type + with pytest.raises(EngineRegistryError): + registry.register(DetectEngine) + + +def test_registry_rejects_duplicate_discriminators_and_resource_mismatch() -> None: + registry = EngineRegistry() + registry.register(AcmeEngine, resources=AcmeResources(prefix="token")) + + with pytest.raises(EngineRegistryError): + registry.register(AcmeEngine, resources=AcmeResources(prefix="other")) + with pytest.raises(EngineRegistryError): + EngineRegistry().register(AcmeEngine) + with pytest.raises(EngineRegistryError): + EngineRegistry().register(DetectEngine, resources=object()) + + +def test_describe_does_not_construct_an_engine() -> None: + class CountingEngine(EntityProcessingEngine[DetectConfig, None]): + supported_strategy = EntityProcessingStrategy.DETECT + initialized = 0 + + def _initialize(self) -> None: + type(self).initialized += 1 + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + del strategy, timeout + return TextProcessingResult(text=text, detections=()) + + registry = EngineRegistry() + registry.register(CountingEngine) + registry.finalize() + + descriptions = registry.describe_engines() + + assert CountingEngine.initialized == 0 + assert descriptions[0].engine == "detect-only" + assert descriptions[0].supported_strategy is EntityProcessingStrategy.DETECT + properties_value = descriptions[0].configuration_schema["properties"] + assert isinstance(properties_value, Mapping) + properties = { + key: value for key, value in properties_value.items() if isinstance(key, str) + } + engine_value = properties["engine"] + assert isinstance(engine_value, Mapping) + engine = {key: value for key, value in engine_value.items() if isinstance(key, str)} + assert engine["const"] == "detect-only" + + +def test_registry_requires_at_least_one_engine() -> None: + with pytest.raises(EngineRegistryError): + EngineRegistry().finalize() diff --git a/projects/privacy-guard/tests/test_errors.py b/projects/privacy-guard/tests/test_errors.py index 8aaf539..eee5509 100644 --- a/projects/privacy-guard/tests/test_errors.py +++ b/projects/privacy-guard/tests/test_errors.py @@ -28,7 +28,7 @@ def test_every_error_code_has_one_safe_complete_specification() -> None: def test_error_kinds_distinguish_invalid_input_from_internal_failures() -> None: assert PrivacyGuardError(ErrorCode.CONFIG_INVALID).kind is ErrorKind.INVALID_INPUT assert ( - PrivacyGuardError(ErrorCode.SCANNER_EXECUTION_FAILED).kind is ErrorKind.INTERNAL + PrivacyGuardError(ErrorCode.ENGINE_EXECUTION_FAILED).kind is ErrorKind.INTERNAL ) assert ( PrivacyGuardError(ErrorCode.CONFIG_INVALID).component is ErrorComponent.CONFIG diff --git a/projects/privacy-guard/tests/test_hardening.py b/projects/privacy-guard/tests/test_hardening.py deleted file mode 100644 index 79634a2..0000000 --- a/projects/privacy-guard/tests/test_hardening.py +++ /dev/null @@ -1,600 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging -import threading -import tracemalloc -from typing import Never - -import grpc -import pytest -from pydantic import ValidationError -from typing_extensions import override - -import privacy_guard.service.servicer as servicer_module -from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 -from privacy_guard.config import PolicyConfig -from privacy_guard.constants import MAX_BODY_BYTES -from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.payloads import ( - InterceptedRequest, - ProcessingDecision, - ProcessingResult, -) -from privacy_guard.processor import RequestProcessor -from privacy_guard.request_body import JsonHandler -from privacy_guard.scanners import ( - Confidence, - Finding, - RequestBodyFinding, - ScanBudget, - Scanner, - ScannerConfig, -) -from privacy_guard.service.servicer import PrivacyGuardMiddleware - -from .scanner_helpers import DeterministicEmailScanner - - -def _request( - body: bytes, - *, - action_kind: str = "redact", - entity_types: list[str] | None = None, - minimum_confidence: str | None = None, -) -> InterceptedRequest: - return InterceptedRequest( - raw_body=body, - policy_config=PolicyConfig.from_mapping( - { - "on_finding": { - "action": action_kind, - "entity_types": entity_types, - "minimum_confidence": minimum_confidence, - } - } - ), - ) - - -class UnexpectedAbortContext: - async def abort(self, code: grpc.StatusCode, details: str) -> Never: - raise AssertionError("successful evaluation must not abort") - - -@pytest.mark.parametrize( - ("depth", "allowed"), - [(64, True), (65, False)], -) -def test_json_nesting_limit_exact_boundary(depth: int, allowed: bool) -> None: - body = (b"[" * depth) + b'"safe"' + (b"]" * depth) - if allowed: - assert ( - JsonHandler().normalize(body, PolicyConfig()).text_blocks[-1].text == "safe" - ) - else: - with pytest.raises(PrivacyGuardError) as exception_info: - JsonHandler().normalize(body, PolicyConfig()) - assert exception_info.value.code is ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED - - -@pytest.mark.parametrize("depth", [255, 300, 500, 900]) -def test_json_nesting_beyond_adapter_recursion_limit_is_shape_error(depth: int) -> None: - body = (b"[" * depth) + b'"safe"' + (b"]" * depth) - - with pytest.raises(PrivacyGuardError) as exception_info: - JsonHandler().normalize(body, PolicyConfig()) - - assert exception_info.value.code is ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED - assert exception_info.value.__cause__ is None - - -def test_json_walker_bounds_allocation_at_exact_aggregate_boundaries( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import privacy_guard.request_body.json as json_module - - monkeypatch.setattr(json_module, "MAX_TEXT_BLOCKS", 2) - monkeypatch.setattr(json_module, "MAX_SCANNED_CHARACTERS", 2) - assert len(JsonHandler().normalize(b'{"a":"b"}', PolicyConfig()).text_blocks) == 2 - - monkeypatch.setattr(json_module, "MAX_TEXT_BLOCKS", 1) - with pytest.raises(PrivacyGuardError) as block_error: - JsonHandler().normalize(b'{"a":"b"}', PolicyConfig()) - assert block_error.value.code is ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED - - monkeypatch.setattr(json_module, "MAX_TEXT_BLOCKS", 2) - monkeypatch.setattr(json_module, "MAX_SCANNED_CHARACTERS", 1) - with pytest.raises(PrivacyGuardError) as text_error: - JsonHandler().normalize(b'{"a":"b"}', PolicyConfig()) - assert text_error.value.code is ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED - - -def test_json_walker_keeps_wide_container_traversal_allocation_bounded() -> None: - import privacy_guard.request_body.json as json_module - - raw_body = b"[" + (b"0," * 99_999) + b"0]" - handler = JsonHandler() - policy = PolicyConfig() - - tracemalloc.start() - try: - request_body = handler.normalize(raw_body, policy) - _, peak_bytes = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - - assert request_body.text_blocks == () - # Parsing and the one strict typed boundary peaked near 1.8 MiB locally. - assert peak_bytes < 8 * 1024 * 1024 - - assert type(request_body.parsed_value) is json_module._JsonBodyState - tracemalloc.start() - try: - assert tuple(handler._iter_text_blocks(request_body.parsed_value.value)) == () - _, walker_peak_bytes = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - - # The incremental walker retains iterator state only for nesting depth. - assert walker_peak_bytes < 64 * 1024 - - -def test_processor_checks_handler_aggregate_boundaries_contextually( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import privacy_guard.processor as processor_module - - processor = RequestProcessor( - [ - DeterministicEmailScanner( - ScannerConfig(name="test_email", entity_types=frozenset({"email"})) - ) - ] - ) - monkeypatch.setattr(processor_module, "MAX_TEXT_BLOCKS", 2) - monkeypatch.setattr(processor_module, "MAX_SCANNED_CHARACTERS", 2) - assert ( - processor.process(_request(b'{"a":"b"}')).decision is ProcessingDecision.ALLOW - ) - - monkeypatch.setattr(processor_module, "MAX_TEXT_BLOCKS", 1) - with pytest.raises(PrivacyGuardError) as block_error: - processor.process(_request(b'{"a":"b"}')) - assert block_error.value.code is ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED - - monkeypatch.setattr(processor_module, "MAX_TEXT_BLOCKS", 2) - monkeypatch.setattr(processor_module, "MAX_SCANNED_CHARACTERS", 1) - with pytest.raises(PrivacyGuardError) as text_error: - processor.process(_request(b'{"a":"b"}')) - assert text_error.value.code is ErrorCode.REQUEST_SHAPE_LIMIT_EXCEEDED - - -@pytest.mark.parametrize( - ("action", "expected"), - [ - ("observe", ProcessingDecision.ALLOW), - ("block", ProcessingDecision.DENY), - ("redact", ProcessingDecision.DENY), - ], -) -def test_json_key_findings_are_observed_blocked_and_never_rewritten( - action: str, expected: ProcessingDecision -) -> None: - scanner = DeterministicEmailScanner( - ScannerConfig(name="test_email", entity_types=frozenset({"email"})) - ) - result = RequestProcessor([scanner]).process( - _request(b'{"user@example.com":"safe","model":"safe"}', action_kind=action) - ) - - assert result.decision is expected - assert len(result.findings) == 1 - assert result.findings[0].text_block_path == "#key:/user@example.com" - assert result.replacement_body is None - - -@pytest.mark.parametrize("action_kind", ["observe", "block", "redact"]) -def test_finding_criteria_are_owned_by_every_action(action_kind: str) -> None: - class MixedScanner(Scanner[ScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - return ( - Finding( - entity="email", - scanner_name=self.scanner_name, - start_offset=0, - end_offset=1, - confidence=Confidence.LOW, - ), - Finding( - entity="token", - scanner_name=self.scanner_name, - start_offset=1, - end_offset=2, - confidence=Confidence.HIGH, - ), - ) - - result = RequestProcessor( - [ - MixedScanner( - ScannerConfig(name="mixed", entity_types=frozenset({"email", "token"})) - ) - ] - ).process( - _request( - b'"ab"', - action_kind=action_kind, - entity_types=["token"], - minimum_confidence="medium", - ) - ) - - assert [ - request_body_finding.finding.entity for request_body_finding in result.findings - ] == ["token"] - - -def test_unknown_entity_filter_is_content_safe_and_multi_scanner_union_is_valid() -> ( - None -): - class EmailScanner(Scanner[ScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - return () - - class TokenScanner(Scanner[ScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - return () - - processor = RequestProcessor( - ( - EmailScanner( - ScannerConfig(name="email", entity_types=frozenset({"email"})) - ), - TokenScanner( - ScannerConfig(name="token", entity_types=frozenset({"token"})) - ), - ) - ) - processor.validate_policy_config( - PolicyConfig.from_mapping( - { - "on_finding": { - "action": "redact", - "entity_types": ["email", "token"], - } - } - ) - ) - typo = "sensitive-email-typo-8472" - with pytest.raises(PrivacyGuardError) as exception_info: - processor.validate_policy_config( - PolicyConfig.from_mapping( - { - "on_finding": { - "action": "redact", - "entity_types": [typo], - } - } - ) - ) - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - assert typo not in str(exception_info.value) - with pytest.raises(PrivacyGuardError) as process_error: - processor.process(_request(b'"safe"', entity_types=[typo])) - assert process_error.value.code is ErrorCode.CONFIG_INVALID - assert typo not in str(process_error.value) - - -def test_multi_scanner_overlap_is_retained_for_observe_and_resolved_for_redact() -> ( - None -): - class FirstScanner(Scanner[ScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - return ( - Finding( - entity="short", - scanner_name=self.scanner_name, - start_offset=1, - end_offset=3, - ), - ) - - class SecondScanner(Scanner[ScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - return ( - Finding( - entity="long", - scanner_name=self.scanner_name, - start_offset=0, - end_offset=4, - ), - ) - - processor = RequestProcessor( - ( - FirstScanner( - ScannerConfig(name="first", entity_types=frozenset({"short"})) - ), - SecondScanner( - ScannerConfig(name="second", entity_types=frozenset({"long"})) - ), - ) - ) - observed = processor.process(_request(b'"abcd"', action_kind="observe")) - redacted = processor.process(_request(b'"abcd"', action_kind="redact")) - blocked = processor.process(_request(b'"abcd"', action_kind="block")) - - assert len(observed.findings) == 2 - assert redacted.replacement_body == b'"[long]"' - assert { - request_body_finding.finding.scanner_name - for request_body_finding in redacted.findings - } == { - "first", - "second", - } - assert blocked.decision is ProcessingDecision.DENY - assert len(blocked.findings) == 2 - - -def test_finding_excess_stably_denies_without_returning_partial_findings( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import privacy_guard.scanners.base as scanner_module - - class NoisyScanner(Scanner[ScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - return ( - Finding( - entity="a", - scanner_name=self.scanner_name, - start_offset=0, - end_offset=1, - ), - Finding( - entity="b", - scanner_name=self.scanner_name, - start_offset=1, - end_offset=2, - ), - ) - - monkeypatch.setattr(scanner_module, "MAX_FINDINGS_PER_BLOCK", 1) - scanner = NoisyScanner( - ScannerConfig(name="noisy", entity_types=frozenset({"a", "b"})) - ) - result = RequestProcessor([scanner]).process(_request(b'"ab"')) - - assert result.decision is ProcessingDecision.DENY - assert result.reason_code == "privacy_guard_limit_exceeded" - assert result.findings == () - - -def test_scanner_identity_is_non_empty_and_finding_entities_are_bounded() -> None: - with pytest.raises(ValidationError): - ScannerConfig(name="", entity_types=frozenset()) - - with pytest.raises(ValidationError): - Finding( - entity="x" * 1025, - scanner_name="bounded", - start_offset=0, - end_offset=1, - ) - - -@pytest.mark.parametrize(("block_count", "allowed"), [(16, True), (17, False)]) -def test_request_finding_limit_exact_boundary(block_count: int, allowed: bool) -> None: - class DenseScanner(Scanner[ScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - return tuple( - Finding( - entity="unit", - scanner_name=self.scanner_name, - start_offset=index, - end_offset=index + 1, - ) - for index in range(256) - ) - - body = ( - "[" + ",".join('"' + ("x" * 256) + '"' for _ in range(block_count)) + "]" - ).encode() - result = RequestProcessor( - [DenseScanner(ScannerConfig(name="dense", entity_types=frozenset({"unit"})))] - ).process(_request(body, action_kind="observe")) - - assert result.decision is ( - ProcessingDecision.ALLOW if allowed else ProcessingDecision.DENY - ) - assert len(result.findings) == (4096 if allowed else 0) - - -@pytest.mark.asyncio -async def test_slow_scan_does_not_stall_unrelated_rpc() -> None: - started = threading.Event() - release = threading.Event() - - class SlowScanner(Scanner[ScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - started.set() - release.wait(timeout=2) - return () - - servicer = PrivacyGuardMiddleware( - RequestProcessor( - [SlowScanner(ScannerConfig(name="slow", entity_types=frozenset()))] - ) - ) - evaluation = pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - body=b'"safe"', - ) - scan_task = asyncio.create_task(servicer._evaluate_http_request(evaluation)) - assert await asyncio.to_thread(started.wait, 1) - try: - manifest = await asyncio.wait_for( - asyncio.to_thread(servicer._describe), timeout=0.1 - ) - assert manifest.name == "privacy-guard" - finally: - release.set() - await scan_task - await servicer.close() - - -@pytest.mark.asyncio -async def test_cancelled_rpc_holds_scan_slot_until_worker_really_finishes() -> None: - entered = 0 - first_started = threading.Event() - release = threading.Event() - - class BlockingScanner(Scanner[ScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - nonlocal entered - entered += 1 - first_started.set() - release.wait(timeout=2) - return () - - servicer = PrivacyGuardMiddleware( - RequestProcessor( - [BlockingScanner(ScannerConfig(name="blocking", entity_types=frozenset()))] - ) - ) - servicer._scan_slots = asyncio.Semaphore(1) - evaluation = pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, body=b'"safe"' - ) - first = asyncio.create_task(servicer._evaluate_http_request(evaluation)) - assert await asyncio.to_thread(first_started.wait, 1) - first.cancel() - with pytest.raises(asyncio.CancelledError): - await first - - second = asyncio.create_task(servicer._evaluate_http_request(evaluation)) - await asyncio.sleep(0.05) - assert entered == 1 - release.set() - await asyncio.wait_for(second, 1) - assert entered == 2 - await servicer.close() - - -@pytest.mark.asyncio -async def test_operational_log_never_contains_body_or_match( - caplog: pytest.LogCaptureFixture, -) -> None: - sentinel = "sentinel@example.com" - evaluation = pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - context=pb2.RequestContext(request_id="request-42"), - config={"on_finding": {"action": "observe"}}, - body=(f'{{"message":"{sentinel}"}}').encode(), - ) - scanner = DeterministicEmailScanner( - ScannerConfig(name="test_email", entity_types=frozenset({"email"})) - ) - servicer = PrivacyGuardMiddleware(RequestProcessor([scanner])) - with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): - await servicer._evaluate_rpc(evaluation, UnexpectedAbortContext()) - await servicer.close() - - assert "privacy_guard_evaluation" in caplog.text - assert sentinel not in caplog.text - - -def test_operational_log_fields_use_typed_construction() -> None: - extra = servicer_module._evaluation_log_extra( - request_id="request-42", - started=0.0, - action="allow", - finding_count=1, - failure=None, - ) - - request_id: str = extra["request_id"] - duration_ms: float = extra["duration_ms"] - action: str = extra["action"] - finding_count: int = extra["finding_count"] - error_code: str | None = extra["error_code"] - assert request_id == "request-42" - assert duration_ms >= 0 - assert action == "allow" - assert finding_count == 1 - assert error_code is None - - -@pytest.mark.asyncio -@pytest.mark.parametrize("size", [MAX_BODY_BYTES, MAX_BODY_BYTES + 1]) -async def test_replacement_body_limit_exact_boundary(size: int) -> None: - class ReplacementProcessor: - def validate_policy_config(self, policy_config: PolicyConfig) -> None: - pass - - def process(self, request: InterceptedRequest) -> ProcessingResult: - return ProcessingResult( - decision=ProcessingDecision.ALLOW, replacement_body=b"x" * size - ) - - response = await PrivacyGuardMiddleware( - ReplacementProcessor() - )._evaluate_http_request( - pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, body=b"{}" - ) - ) - assert response.decision == ( - pb2.DECISION_ALLOW if size == MAX_BODY_BYTES else pb2.DECISION_DENY - ) - assert response.has_body is (size == MAX_BODY_BYTES) - - -@pytest.mark.asyncio -@pytest.mark.parametrize(("extra_byte", "allowed"), [(False, True), (True, False)]) -async def test_aggregate_finding_encoded_limit_exact_boundary( - monkeypatch: pytest.MonkeyPatch, extra_byte: bool, allowed: bool -) -> None: - entity = "x" * (900 + int(extra_byte)) - exact_size = pb2.Finding( - type="scanner", label="x" * 900, confidence="high", count=1 - ).ByteSize() - monkeypatch.setattr(servicer_module, "MAX_PROTO_FINDING_BYTES", exact_size) - - class FindingProcessor: - def validate_policy_config(self, policy_config: PolicyConfig) -> None: - pass - - def process(self, request: InterceptedRequest) -> ProcessingResult: - return ProcessingResult( - decision=ProcessingDecision.ALLOW, - findings=( - RequestBodyFinding( - finding=Finding( - entity=entity, - scanner_name="scanner", - start_offset=0, - end_offset=1, - ), - text_block_path="path", - ), - ), - ) - - response = await PrivacyGuardMiddleware(FindingProcessor())._evaluate_http_request( - pb2.HttpRequestEvaluation( - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, body=b"{}" - ) - ) - assert response.decision == (pb2.DECISION_ALLOW if allowed else pb2.DECISION_DENY) - assert len(response.findings) == int(allowed) diff --git a/projects/privacy-guard/tests/test_payloads.py b/projects/privacy-guard/tests/test_payloads.py deleted file mode 100644 index cb03734..0000000 --- a/projects/privacy-guard/tests/test_payloads.py +++ /dev/null @@ -1,118 +0,0 @@ -import pytest -from pydantic import ValidationError - -from privacy_guard.config import PolicyConfig -from privacy_guard.payloads import ( - InterceptedRequest, - ProcessingDecision, - ProcessingResult, -) -from privacy_guard.scanners import Finding, RequestBodyFinding - - -def test_processing_decision_values() -> None: - assert ProcessingDecision.ALLOW.value == "allow" - assert ProcessingDecision.DENY.value == "deny" - - -def test_payloads_are_frozen_and_hide_body_content_from_repr() -> None: - sensitive_body = b"sensitive-body-8472" - request = InterceptedRequest( - raw_body=sensitive_body, - content_type="application/json", - policy_config=PolicyConfig(), - ) - result = ProcessingResult( - decision=ProcessingDecision.ALLOW, - replacement_body=sensitive_body, - ) - - with pytest.raises(ValidationError): - setattr(request, "raw_body", b"changed") - with pytest.raises(ValidationError): - setattr(result, "replacement_body", b"changed") - assert repr(sensitive_body) not in repr(request) - assert repr(sensitive_body) not in repr(result) - - -def test_processing_result_repr_hides_sensitive_structural_path() -> None: - sensitive_key = "secret-person@example.com" - finding = RequestBodyFinding( - finding=Finding( - entity="email", - scanner_name="scanner", - start_offset=0, - end_offset=1, - ), - text_block_path=f"#key:/{sensitive_key}", - ) - result = ProcessingResult(decision=ProcessingDecision.ALLOW, findings=(finding,)) - - assert sensitive_key not in repr(finding) - assert sensitive_key not in repr(result) - - -def test_empty_replacement_is_distinct_from_no_replacement() -> None: - no_replacement = ProcessingResult(decision=ProcessingDecision.ALLOW) - empty_replacement = ProcessingResult( - decision=ProcessingDecision.ALLOW, replacement_body=b"" - ) - - assert no_replacement.replacement_body is None - assert empty_replacement.replacement_body == b"" - assert no_replacement != empty_replacement - - -def test_processing_result_defaults_to_an_empty_findings_tuple() -> None: - result = ProcessingResult(decision=ProcessingDecision.ALLOW) - - assert result.findings == () - assert type(result.findings) is tuple - assert result.reason_code is None - - -def test_intercepted_request_is_a_passive_record_without_body_method() -> None: - assert not hasattr(InterceptedRequest, "body") - - -@pytest.mark.parametrize( - ("model", "values"), - [ - ( - InterceptedRequest, - { - "raw_body": bytearray(b"body"), - "policy_config": PolicyConfig(), - }, - ), - ( - InterceptedRequest, - { - "raw_body": b"body", - "policy_config": {"on_finding": {"action": "redact"}}, - }, - ), - (ProcessingResult, {"decision": "allow"}), - ( - ProcessingResult, - {"decision": ProcessingDecision.ALLOW, "findings": []}, - ), - ], -) -def test_payloads_reject_dataclass_era_coercions( - model: type[InterceptedRequest] | type[ProcessingResult], - values: dict[str, object], -) -> None: - with pytest.raises(ValidationError): - model.model_validate(values) - - -def test_payloads_forbid_extra_fields() -> None: - with pytest.raises(ValidationError): - InterceptedRequest.model_validate( - { - "raw_body": b"body", - "policy_config": PolicyConfig(), - "legacy_body": b"body", - } - ) diff --git a/projects/privacy-guard/tests/test_processor.py b/projects/privacy-guard/tests/test_processor.py deleted file mode 100644 index 8645fd1..0000000 --- a/projects/privacy-guard/tests/test_processor.py +++ /dev/null @@ -1,786 +0,0 @@ -import logging -import math -from collections.abc import Mapping - -import pytest -from pydantic import ValidationError -from typing_extensions import override - -from privacy_guard.config import PolicyAction, PolicyConfig -from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.payloads import ( - InterceptedRequest, - ProcessingDecision, - ProcessingResult, -) -from privacy_guard.processor import RequestProcessor -from privacy_guard.request_body import FormatHandler, RequestBody, TextBlock -from privacy_guard.scanners import ( - Finding, - RequestBodyFinding, - ScanBudget, - ScanBudgetExceeded, - Scanner, - ScannerConfig, - ScannerContractError, - ScannerFindingLimitExceeded, - parse_scanner_output, -) - - -class RecordingScanner(Scanner[ScannerConfig]): - def __init__( - self, findings_by_text: Mapping[str, tuple[Finding, ...]] | None = None - ) -> None: - self.findings_by_text = findings_by_text or {} - entity_types = frozenset( - finding.entity - for findings in self.findings_by_text.values() - for finding in findings - ) - super().__init__(ScannerConfig(name="recording", entity_types=entity_types)) - self.calls: list[str] = [] - - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - self.calls.append(text_block) - return self.findings_by_text.get(text_block, ()) - - -class RecordingHandler(FormatHandler): - def __init__( - self, - text_blocks: tuple[TextBlock, ...] = (), - reconstructed: bytes | None = None, - ) -> None: - super().__init__(format_name="opaque") - self.text_blocks = text_blocks - self.reconstructed = reconstructed - self.normalize_calls = 0 - self.reconstruct_calls = 0 - self.replacements: dict[str, str] | None = None - - @override - def _normalize(self, raw_body: bytes, policy_config: PolicyConfig) -> RequestBody: - self.normalize_calls += 1 - return RequestBody( - text_blocks=self.text_blocks, - parsed_value=None, - original_bytes=raw_body, - ) - - @override - def _reconstruct( - self, - request_body: RequestBody, - replacements_by_path: Mapping[str, str], - ) -> bytes: - self.reconstruct_calls += 1 - self.replacements = dict(replacements_by_path) - return ( - request_body.original_bytes - if self.reconstructed is None - else self.reconstructed - ) - - -def _config( - *, - body_format: str = "opaque", - action_kind: str = "redact", - entity_types: list[str] | None = None, - minimum_confidence: str | None = None, - template: str = "[{entity}]", -) -> PolicyConfig: - on_finding: dict[str, object] = { - "action": action_kind, - "entity_types": entity_types, - "minimum_confidence": minimum_confidence, - } - if action_kind == PolicyAction.REDACT.value: - on_finding["template"] = template - return PolicyConfig.from_mapping( - {"body_format": body_format, "on_finding": on_finding} - ) - - -def _request( - raw_body: bytes = b"body", policy_config: PolicyConfig | None = None -) -> InterceptedRequest: - return InterceptedRequest( - raw_body=raw_body, - content_type="application/test", - policy_config=policy_config or _config(), - ) - - -def _processor( - scanner: Scanner[ScannerConfig], handler: FormatHandler -) -> RequestProcessor: - return RequestProcessor([scanner], {handler.format_name: handler}) - - -def _finding( - start: int, - end: int, - *, - entity: str = "secret", - scanner_name: str = "recording", -) -> Finding: - return Finding( - entity=entity, - scanner_name=scanner_name, - start_offset=start, - end_offset=end, - ) - - -def _request_body_finding( - start: int, - end: int, - *, - entity: str = "secret", - scanner_name: str = "recording", - text_block_path: str, -) -> RequestBodyFinding: - return RequestBodyFinding( - finding=_finding(start, end, entity=entity, scanner_name=scanner_name), - text_block_path=text_block_path, - ) - - -def test_scanner_visits_each_text_block_and_reconstructs_once() -> None: - scanner = RecordingScanner() - handler = RecordingHandler( - ( - TextBlock(path="text_block::alpha", text="first"), - TextBlock(path="text_block::beta", text="second"), - ) - ) - - result = _processor(scanner, handler).process(_request()) - - assert result == ProcessingResult(decision=ProcessingDecision.ALLOW) - assert scanner.calls == ["first", "second"] - assert handler.normalize_calls == 1 - assert handler.reconstruct_calls == 1 - assert handler.replacements == {} - - -def test_debug_log_reports_safe_processing_phase_metrics( - caplog: pytest.LogCaptureFixture, -) -> None: - sentinel = "sensitive@example.com" - scanner = RecordingScanner( - {sentinel: (_finding(0, len(sentinel), entity="email"),)} - ) - handler = RecordingHandler( - (TextBlock(path="sensitive-path-8472", text=sentinel),), - reconstructed=b"[email]", - ) - - with caplog.at_level(logging.DEBUG, logger="privacy_guard.processor"): - _processor(scanner, handler).process(_request(raw_body=sentinel.encode())) - - assert "phase=normalize" in caplog.text - assert "phase=scan" in caplog.text - assert "phase=reconstruct" in caplog.text - assert f"body_bytes={len(sentinel.encode())}" in caplog.text - assert "text_block_count=1" in caplog.text - assert f"scanned_characters={len(sentinel)}" in caplog.text - assert "finding_count=1" in caplog.text - assert sentinel not in caplog.text - assert "8472" not in caplog.text - - -def test_request_content_logging_reports_received_and_forwarded_bodies( - caplog: pytest.LogCaptureFixture, -) -> None: - sentinel = "sensitive@example.com" - scanner = RecordingScanner( - {sentinel: (_finding(0, len(sentinel), entity="email"),)} - ) - handler = RecordingHandler( - (TextBlock(path="email", text=sentinel),), - reconstructed=b"[email]", - ) - processor = RequestProcessor( - [scanner], - {handler.format_name: handler}, - log_request_content=True, - ) - - with caplog.at_level(logging.DEBUG, logger="privacy_guard.processor"): - processor.process(_request(raw_body=sentinel.encode())) - - assert f"stage=received body={sentinel.encode()!r}" in caplog.text - assert "stage=forwarded body=b'[email]'" in caplog.text - - -def test_empty_scanner_sequence_is_rejected_at_construction() -> None: - with pytest.raises(PrivacyGuardError) as exception_info: - RequestProcessor([]) - - assert exception_info.value.code is ErrorCode.SCANNER_OUTPUT_INVALID - - -@pytest.mark.parametrize( - "timeout", - [True, False, 0, -1, math.inf, math.nan, 31], -) -def test_processor_rejects_invalid_scan_timeout(timeout: float) -> None: - with pytest.raises(ValueError): - RequestProcessor([RecordingScanner()], scan_timeout_seconds=timeout) - - -def test_one_scan_budget_is_shared_across_blocks_and_exhaustion_discards_findings() -> ( - None -): - class BudgetScanner(Scanner[ScannerConfig]): - def __init__(self) -> None: - super().__init__( - ScannerConfig(name="budget", entity_types=frozenset({"secret"})) - ) - self.budgets: list[ScanBudget] = [] - - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - self.budgets.append(budget) - if text_block == "second": - raise ScanBudgetExceeded - return ( - Finding( - entity="secret", - scanner_name=self.scanner_name, - start_offset=0, - end_offset=1, - ), - ) - - scanner = BudgetScanner() - handler = RecordingHandler( - ( - TextBlock(path="first", text="first"), - TextBlock(path="second", text="second"), - ) - ) - - result = _processor(scanner, handler).process( - _request(policy_config=_config(action_kind=PolicyAction.OBSERVE.value)) - ) - - assert result.decision is ProcessingDecision.DENY - assert result.reason_code == "privacy_guard_limit_exceeded" - assert result.findings == () - assert len(scanner.budgets) == 2 - assert scanner.budgets[0] is scanner.budgets[1] - - -def test_duplicate_scanner_names_are_rejected_at_construction() -> None: - with pytest.raises(PrivacyGuardError) as exception_info: - RequestProcessor([RecordingScanner(), RecordingScanner()]) - - assert exception_info.value.code is ErrorCode.SCANNER_OUTPUT_INVALID - - -def test_scanner_infers_validates_and_uses_concrete_config_type() -> None: - class PrefixScannerConfig(ScannerConfig): - finding_entity: str - - class PrefixScanner(Scanner[PrefixScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - return ( - Finding( - entity=self.config.finding_entity, - scanner_name=self.scanner_name, - start_offset=0, - end_offset=1, - ), - ) - - config = PrefixScannerConfig( - name="prefix", - entity_types=frozenset({"custom"}), - finding_entity="custom", - ) - scanner = PrefixScanner(config) - RequestProcessor([scanner]) - - assert PrefixScanner.get_config_type() is PrefixScannerConfig - assert scanner.config is config - assert scanner.config.finding_entity == "custom" - assert scanner.supported_entity_types is scanner.config.entity_types - assert scanner.scan("x")[0].entity == "custom" - pytest.raises( - ScannerContractError, - PrefixScanner, - ScannerConfig(name="wrong-config-type", entity_types=frozenset()), - ) - - -def test_empty_supported_entity_catalog_rejects_enabled_filter() -> None: - processor = RequestProcessor([RecordingScanner()]) - - with pytest.raises(PrivacyGuardError) as exception_info: - processor.validate_policy_config( - PolicyConfig.from_mapping( - {"on_finding": {"action": "observe", "entity_types": ["email"]}} - ) - ) - - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - - -def test_handler_registry_key_must_match_immutable_identity() -> None: - with pytest.raises(PrivacyGuardError) as exception_info: - RequestProcessor([RecordingScanner()], {"wrong": RecordingHandler()}) - - assert exception_info.value.code is ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID - - -def test_malformed_normalize_output_maps_to_handler_contract_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - handler = RecordingHandler() - monkeypatch.setattr(handler, "_normalize", lambda raw_body, policy: object()) - - with pytest.raises(PrivacyGuardError) as exception_info: - _processor(RecordingScanner(), handler).process(_request()) - - assert exception_info.value.code is ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID - - -def test_non_bytes_reconstruct_output_maps_to_handler_contract_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - handler = RecordingHandler() - monkeypatch.setattr( - handler, - "_reconstruct", - lambda request_body, replacements: bytearray(b"body"), - ) - - with pytest.raises(PrivacyGuardError) as exception_info: - _processor(RecordingScanner(), handler).process(_request()) - - assert exception_info.value.code is ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID - - -def test_validate_policy_config_selects_and_validates_without_processing() -> None: - handler = RecordingHandler() - processor = _processor(RecordingScanner(), handler) - - processor.validate_policy_config(_config()) - - assert handler.normalize_calls == 0 - assert handler.reconstruct_calls == 0 - - -def test_bodyless_request_skips_normalization_scanning_and_reconstruction() -> None: - scanner = RecordingScanner() - handler = RecordingHandler((TextBlock(path="text_block::alpha", text="not used"),)) - - result = _processor(scanner, handler).process(_request(b"")) - - assert result == ProcessingResult(decision=ProcessingDecision.ALLOW) - assert handler.normalize_calls == 0 - assert scanner.calls == [] - assert handler.reconstruct_calls == 0 - - -def test_empty_json_object_scans_nothing_and_reconstructs_once() -> None: - scanner = RecordingScanner() - processor = RequestProcessor([scanner]) - request = InterceptedRequest( - raw_body=b"{}", - content_type="application/json", - policy_config=PolicyConfig(), - ) - - result = processor.process(request) - - assert result == ProcessingResult(decision=ProcessingDecision.ALLOW) - assert scanner.calls == [] - - -def test_observe_attaches_paths_without_mutating_body() -> None: - original = _finding(1, 3) - scanner = RecordingScanner({"abcd": (original,)}) - handler = RecordingHandler((TextBlock(path="text_block::alpha", text="abcd"),)) - - result = _processor(scanner, handler).process( - _request(policy_config=_config(action_kind=PolicyAction.OBSERVE.value)) - ) - - assert result.decision is ProcessingDecision.ALLOW - assert result.replacement_body is None - assert result.findings == ( - _request_body_finding(1, 3, text_block_path="text_block::alpha"), - ) - assert not hasattr(original, "text_block_path") - assert handler.replacements == {} - - -@pytest.mark.parametrize( - ("text", "findings", "template", "expected"), - [ - ("abcde", (_finding(1, 3),), "X", "aXde"), - ( - "abcdef", - (_finding(3, 5, entity="b"), _finding(0, 2, entity="a")), - "[{entity}]", - "[a]c[b]f", - ), - ("a🐍éz", (_finding(1, 3),), "X", "aXz"), - ], -) -def test_redact_replaces_sorted_non_overlapping_character_spans( - text: str, findings: tuple[Finding, ...], template: str, expected: str -) -> None: - scanner = RecordingScanner({text: findings}) - handler = RecordingHandler( - (TextBlock(path="text_block::alpha", text=text),), b"changed" - ) - - result = _processor(scanner, handler).process( - _request(policy_config=_config(template=template)) - ) - - assert result.replacement_body == b"changed" - assert handler.replacements == {"text_block::alpha": expected} - - -def test_static_redaction_template_is_supported() -> None: - scanner = RecordingScanner({"secret": (_finding(0, 6),)}) - handler = RecordingHandler( - (TextBlock(path="text_block::alpha", text="secret"),), b"x" - ) - - _processor(scanner, handler).process( - _request(policy_config=_config(template="[redacted]")) - ) - - assert handler.replacements == {"text_block::alpha": "[redacted]"} - - -def test_redaction_expansion_is_denied_before_render_or_reconstruction( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import privacy_guard.processor as processor_module - - scanner = RecordingScanner({"ab": (_finding(0, 1), _finding(1, 2))}) - handler = RecordingHandler((TextBlock(path="text_block::alpha", text="ab"),)) - monkeypatch.setattr(processor_module, "MAX_BODY_BYTES", 8) - - def unexpected_render(*args: object) -> str: - raise AssertionError("redaction must not be rendered after preflight denial") - - monkeypatch.setattr(processor_module, "_redact_text", unexpected_render) - result = _processor(scanner, handler).process( - _request(policy_config=_config(template="x" * 64)) - ) - - assert result.decision is ProcessingDecision.DENY - assert result.reason_code == "privacy_guard_limit_exceeded" - assert handler.reconstruct_calls == 0 - - -def test_serialized_redaction_expansion_is_denied_after_reconstruction( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import privacy_guard.processor as processor_module - - scanner = RecordingScanner({"x": (_finding(0, 1),)}) - monkeypatch.setattr(processor_module, "MAX_BODY_BYTES", 8) - - result = RequestProcessor([scanner]).process( - InterceptedRequest( - raw_body=b'"x"', - policy_config=_config(body_format="json", template='"' * 6), - ) - ) - - assert result.decision is ProcessingDecision.DENY - assert result.reason_code == "privacy_guard_limit_exceeded" - assert result.replacement_body is None - - -def test_block_denies_with_findings_and_suppresses_reconstruction() -> None: - scanner = RecordingScanner({"secret": (_finding(0, 6),)}) - handler = RecordingHandler((TextBlock(path="text_block::alpha", text="secret"),)) - config = _config(action_kind=PolicyAction.BLOCK.value) - - result = _processor(scanner, handler).process(_request(policy_config=config)) - - assert result.decision is ProcessingDecision.DENY - assert result.replacement_body is None - assert result.reason_code == "privacy_guard_blocked" - assert len(result.findings) == 1 - assert handler.reconstruct_calls == 0 - - -def test_block_allows_and_reconstructs_when_no_findings_exist() -> None: - handler = RecordingHandler((TextBlock(path="text_block::alpha", text="safe"),)) - - result = _processor(RecordingScanner(), handler).process( - _request(policy_config=_config(action_kind=PolicyAction.BLOCK.value)) - ) - - assert result == ProcessingResult(decision=ProcessingDecision.ALLOW) - assert handler.reconstruct_calls == 1 - - -def test_findings_keep_scanner_identity_and_text_block_order_then_span_order() -> None: - scanner = RecordingScanner( - { - "first": ( - _finding(3, 5, entity="late"), - _finding(0, 2, entity="early"), - ), - "second": (_finding(1, 3, entity="second"),), - } - ) - handler = RecordingHandler( - ( - TextBlock(path="text_block::one", text="first"), - TextBlock(path="text_block::two", text="second"), - ) - ) - - result = _processor(scanner, handler).process( - _request(policy_config=_config(action_kind=PolicyAction.OBSERVE.value)) - ) - - assert [ - ( - request_body_finding.finding.entity, - request_body_finding.finding.scanner_name, - request_body_finding.text_block_path, - ) - for request_body_finding in result.findings - ] == [ - ("early", "recording", "text_block::one"), - ("late", "recording", "text_block::one"), - ("second", "recording", "text_block::two"), - ] - - -@pytest.mark.parametrize( - "normalized", - [ - RequestBody( - text_blocks=( - TextBlock(path="same", text="a"), - TextBlock(path="same", text="b"), - ), - parsed_value=None, - original_bytes=b"body", - ), - RequestBody(text_blocks=(), parsed_value=None, original_bytes=b"different"), - ], -) -def test_contextually_invalid_normalized_body_is_rejected_before_scanning( - normalized: RequestBody, -) -> None: - scanner = RecordingScanner() - - class ReturningHandler(RecordingHandler): - @override - def _normalize( - self, raw_body: bytes, policy_config: PolicyConfig - ) -> RequestBody: - self.normalize_calls += 1 - return normalized - - with pytest.raises(PrivacyGuardError) as exception_info: - _processor(scanner, ReturningHandler()).process(_request()) - - assert exception_info.value.code is ErrorCode.FORMAT_HANDLER_OUTPUT_INVALID - assert scanner.calls == [] - - -@pytest.mark.parametrize("scanner_result", [[], (object(),)]) -def test_invalid_scanner_output_shape_is_rejected(scanner_result: object) -> None: - with pytest.raises(ScannerContractError): - parse_scanner_output(scanner_result) - - -def test_request_body_finding_is_not_valid_scanner_output() -> None: - finding = _finding(0, 1) - request_body_finding = RequestBodyFinding(finding=finding, text_block_path="path") - - with pytest.raises(ScannerContractError): - parse_scanner_output((request_body_finding,)) - - -def test_exact_finding_instance_is_reused_by_output_validation() -> None: - finding = _finding(0, 1) - - parsed = parse_scanner_output((finding,)) - - assert parsed[0] is finding - - -def test_scanner_output_limit_is_checked_before_element_validation( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import privacy_guard.scanners.base as scanner_module - - monkeypatch.setattr(scanner_module, "MAX_FINDINGS_PER_BLOCK", 1) - - with pytest.raises(ScannerFindingLimitExceeded): - parse_scanner_output((object(), object())) - - -@pytest.mark.parametrize( - "overrides", - [ - {"entity": ""}, - {"scanner_name": ""}, - {"entity": "bad\ud800"}, - {"scanner_name": "recording\ud800"}, - {"start_offset": True}, - {"start_offset": -1}, - {"start_offset": 1, "end_offset": 1}, - {"confidence": "high"}, - ], -) -def test_invalid_finding_fields_fail_at_model_construction( - overrides: dict[str, object], -) -> None: - values: dict[str, object] = { - "entity": "secret", - "scanner_name": "recording", - "start_offset": 0, - "end_offset": 1, - **overrides, - } - with pytest.raises(ValidationError): - Finding.model_validate(values) - - -@pytest.mark.parametrize( - "finding", - [ - _finding(0, 1, scanner_name="other"), - _finding(1, 5), - ], -) -def test_contextually_invalid_finding_is_rejected(finding: Finding) -> None: - scanner = RecordingScanner({"abcd": (finding,)}) - handler = RecordingHandler((TextBlock(path="text_block::alpha", text="abcd"),)) - - with pytest.raises(PrivacyGuardError) as exception_info: - _processor(scanner, handler).process(_request()) - - assert exception_info.value.code is ErrorCode.SCANNER_OUTPUT_INVALID - assert handler.reconstruct_calls == 0 - - -def test_finding_entity_must_be_declared_by_scanner_config() -> None: - class UndeclaredEntityScanner(Scanner[ScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - return ( - _finding(0, 1, entity="undeclared", scanner_name=self.scanner_name), - ) - - scanner = UndeclaredEntityScanner( - ScannerConfig(name="declared", entity_types=frozenset({"declared"})) - ) - handler = RecordingHandler((TextBlock(path="path", text="x"),)) - - with pytest.raises(PrivacyGuardError) as exception_info: - _processor(scanner, handler).process(_request()) - - assert exception_info.value.code is ErrorCode.SCANNER_OUTPUT_INVALID - - -def test_adjacent_scanner_findings_are_accepted() -> None: - scanner = RecordingScanner({"abcd": (_finding(0, 2), _finding(2, 4))}) - handler = RecordingHandler( - (TextBlock(path="text_block::alpha", text="abcd"),), b"x" - ) - - result = _processor(scanner, handler).process(_request()) - - assert result.decision is ProcessingDecision.ALLOW - assert len(result.findings) == 2 - - -@pytest.mark.parametrize( - ("collaborator", "expected_code"), - [ - ("scanner", ErrorCode.SCANNER_EXECUTION_FAILED), - ("handler", ErrorCode.FORMAT_HANDLER_EXECUTION_FAILED), - ], -) -def test_collaborator_exceptions_are_replaced_without_partial_result_or_content( - collaborator: str, expected_code: ErrorCode -) -> None: - sentinel = "sensitive-collaborator-exception-8472" - - class RaisingScanner(Scanner[ScannerConfig]): - @override - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - raise RuntimeError(sentinel) - - class RaisingHandler(RecordingHandler): - @override - def _normalize( - self, raw_body: bytes, policy_config: PolicyConfig - ) -> RequestBody: - raise RuntimeError(sentinel) - - scanner: Scanner[ScannerConfig] = ( - RaisingScanner(ScannerConfig(name="raising", entity_types=frozenset())) - if collaborator == "scanner" - else RecordingScanner() - ) - handler: FormatHandler = ( - RaisingHandler() - if collaborator == "handler" - else RecordingHandler((TextBlock(path="text_block::alpha", text="text"),)) - ) - - with pytest.raises(PrivacyGuardError) as exception_info: - _processor(scanner, handler).process(_request()) - - assert exception_info.value.code is expected_code - assert exception_info.value.__cause__ is None - assert sentinel not in str(exception_info.value) - assert sentinel not in repr(exception_info.value) - - -def test_cataloged_handler_error_propagates_unchanged() -> None: - expected = PrivacyGuardError(ErrorCode.BODY_JSON_INVALID) - - class CatalogRaisingHandler(RecordingHandler): - @override - def _normalize( - self, raw_body: bytes, policy_config: PolicyConfig - ) -> RequestBody: - raise expected - - with pytest.raises(PrivacyGuardError) as exception_info: - _processor(RecordingScanner(), CatalogRaisingHandler()).process(_request()) - - assert exception_info.value is expected - - -def test_reconstruction_equal_to_original_returns_no_replacement() -> None: - handler = RecordingHandler( - (TextBlock(path="text_block::alpha", text="text"),), reconstructed=b"body" - ) - - result = _processor(RecordingScanner(), handler).process(_request(b"body")) - - assert result.replacement_body is None - - -def test_processing_result_is_immutable() -> None: - result = ProcessingResult(decision=ProcessingDecision.ALLOW) - - with pytest.raises(ValidationError): - setattr(result, "decision", ProcessingDecision.DENY) diff --git a/projects/privacy-guard/tests/test_request_processor.py b/projects/privacy-guard/tests/test_request_processor.py new file mode 100644 index 0000000..7fe65ef --- /dev/null +++ b/projects/privacy-guard/tests/test_request_processor.py @@ -0,0 +1,109 @@ +"""RequestProcessor tests for the one-text, ordered-stage contract.""" + +from __future__ import annotations + +from privacy_guard.config import PolicyAction +from privacy_guard.engine_registry import EngineRegistry +from privacy_guard.engines import RegexEngine +from privacy_guard.request_processor import RequestDecision, RequestProcessor + + +def _values(action: PolicyAction) -> dict[str, object]: + return { + "entity_processing": { + "stages": [ + { + "name": "people", + "config": { + "engine": "regex", + "pattern_catalog": { + "entities": [ + { + "name": "person", + "patterns": [ + { + "pattern": "Alice", + "confidence": "high", + } + ], + } + ] + }, + "replacement": { + "strategy": "template", + "template": "[{entity}]", + }, + }, + }, + { + "config": { + "engine": "regex", + "pattern_catalog": { + "entities": [ + { + "name": "marker", + "patterns": [ + { + "pattern": "person", + "confidence": "medium", + } + ], + } + ] + }, + "replacement": { + "strategy": "template", + "template": "<{entity}>", + }, + }, + }, + ] + }, + "on_detection": {"action": action.value}, + } + + +def _processor(action: PolicyAction) -> RequestProcessor: + registry = EngineRegistry() + registry.register(RegexEngine) + registry.finalize() + config = registry.validate_config(_values(action)) + stages = tuple( + ( + stage.diagnostic_name(index), + registry.create_engine(stage.config), + ) + for index, stage in enumerate(config.entity_processing.stages, start=1) + ) + return RequestProcessor(config, stages) + + +def test_replace_runs_stages_sequentially_over_the_current_text() -> None: + result = _processor(PolicyAction.REPLACE).process("Hello Alice") + + assert result.decision is RequestDecision.ALLOW + assert result.replacement_text == "Hello []" + assert tuple( + (item.entity, item.source_stage, item.count) + for item in result.detection_summaries + ) == ( + ("person", "people", 1), + ("marker", "regex[2]", 1), + ) + + +def test_detect_reports_without_returning_replacement_text() -> None: + result = _processor(PolicyAction.DETECT).process("Hello Alice") + + assert result.decision is RequestDecision.ALLOW + assert result.replacement_text is None + assert tuple(item.entity for item in result.detection_summaries) == ("person",) + + +def test_block_is_a_processor_disposition_not_an_engine_strategy() -> None: + result = _processor(PolicyAction.BLOCK).process("Hello Alice") + + assert result.decision is RequestDecision.DENY + assert result.replacement_text is None + assert result.reason_code == "privacy_guard_blocked" + assert tuple(item.entity for item in result.detection_summaries) == ("person",) diff --git a/projects/privacy-guard/tests/test_single_block_composition.py b/projects/privacy-guard/tests/test_single_block_composition.py deleted file mode 100644 index 3f75d59..0000000 --- a/projects/privacy-guard/tests/test_single_block_composition.py +++ /dev/null @@ -1,47 +0,0 @@ -import json - -import pytest - -from privacy_guard.config import PolicyConfig -from privacy_guard.request_body import JsonHandler - - -@pytest.mark.parametrize( - ("raw_body", "selected_text_block_path", "expected_value"), - [ - ( - b'{"messages":[{"role":"user","content":"hello"}],"model":"model-a"}', - "/messages/0/content", - { - "messages": [{"role": "user", "content": "hello [test suffix]"}], - "model": "model-a", - }, - ), - ( - b'{"contents":[{"parts":[{"text":"hello"}]}],"model":"model-b"}', - "/contents/0/parts/0/text", - { - "contents": [{"parts": [{"text": "hello [test suffix]"}]}], - "model": "model-b", - }, - ), - ], -) -def test_one_selected_text_block_can_be_explicitly_replaced( - raw_body: bytes, selected_text_block_path: str, expected_value: object -) -> None: - json_handler = JsonHandler() - request_body = json_handler.normalize(raw_body, PolicyConfig()) - selected_text_block = next( - text_block - for text_block in request_body.text_blocks - if text_block.path == selected_text_block_path - ) - - test_suffix = " [test suffix]" - replacement_text = selected_text_block.text + test_suffix - reconstructed_body = json_handler.reconstruct( - request_body, {selected_text_block.path: replacement_text} - ) - - assert json.loads(reconstructed_body) == expected_value diff --git a/projects/privacy-guard/uv.lock b/projects/privacy-guard/uv.lock index b7f208b..fe2a9ae 100644 --- a/projects/privacy-guard/uv.lock +++ b/projects/privacy-guard/uv.lock @@ -136,7 +136,7 @@ dependencies = [ { name = "grpcio" }, { name = "protobuf" }, { name = "pydantic" }, - { name = "pyyaml" }, + { name = "regex" }, { name = "typer" }, { name = "typing-extensions" }, ] @@ -154,7 +154,7 @@ requires-dist = [ { name = "grpcio", specifier = ">=1.81.1,<2" }, { name = "protobuf", specifier = ">=6.33.5,<7" }, { name = "pydantic", specifier = ">=2.11,<3" }, - { name = "pyyaml", specifier = ">=6.0.2,<7" }, + { name = "regex", specifier = ">=2026.7.19,<2027" }, { name = "typer", specifier = ">=0.16,<1" }, { name = "typing-extensions", specifier = ">=4.12,<5" }, ] @@ -338,58 +338,107 @@ wheels = [ ] [[package]] -name = "pyyaml" -version = "6.0.3" +name = "regex" +version = "2026.7.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, - { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, - { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, - { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, - { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, - { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, - { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, - { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, - { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, - { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, - { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, - { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, - { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, - { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, - { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, - { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, - { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, - { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, - { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, - { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, - { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, - { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, - { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, - { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, - { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, - { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, - { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, - { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, - { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, - { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, - { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, - { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, - { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, - { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, - { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, - { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, - { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, - { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, - { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, - { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, - { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, + { url = "https://files.pythonhosted.org/packages/05/e5/cef4de2bac939280b68d32adc659478845238a8274f2f79c465063f590ad/regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c", size = 494012, upload-time = "2026-07-19T00:16:39.927Z" }, + { url = "https://files.pythonhosted.org/packages/ff/87/e86f51eb117457bb7803132ffe5cb6e2841e2b5bea4cc85d397f3c6e257d/regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae", size = 295281, upload-time = "2026-07-19T00:16:41.433Z" }, + { url = "https://files.pythonhosted.org/packages/41/2e/2360c41d8080a3d9ec7e5c90fad6eab3b50192869d10e9a5609e48c8177b/regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5", size = 290615, upload-time = "2026-07-19T00:16:43.058Z" }, + { url = "https://files.pythonhosted.org/packages/cf/69/b65ba4344efbc771b28fe5dde84cbbb6c8f9551165952fe78def5b9dde6a/regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20", size = 791804, upload-time = "2026-07-19T00:16:44.662Z" }, + { url = "https://files.pythonhosted.org/packages/81/b6/a40dfa0dc6224b36f620c00296eacc830489cbf8c2837b6750dfe6170375/regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a", size = 861723, upload-time = "2026-07-19T00:16:46.412Z" }, + { url = "https://files.pythonhosted.org/packages/e3/02/735991dee71abd83196a7962f7ed8bf5aa05720ff06e2d3ff896a85e2bbb/regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7", size = 905932, upload-time = "2026-07-19T00:16:47.956Z" }, + { url = "https://files.pythonhosted.org/packages/45/6c/e7098d8b846ccdbf431d8c081b61e496526a27a28094ed09e0dce21b3f54/regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc", size = 801407, upload-time = "2026-07-19T00:16:49.43Z" }, + { url = "https://files.pythonhosted.org/packages/8a/18/34b69274e2649bcc7d9b089c2b2983fb2632d8ecf667e359593be9072e79/regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864", size = 774448, upload-time = "2026-07-19T00:16:51.352Z" }, + { url = "https://files.pythonhosted.org/packages/bb/e6/0a72247d025585fd3800b98e040b84d562a88af6303347100484849f4f01/regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af", size = 783297, upload-time = "2026-07-19T00:16:53.071Z" }, + { url = "https://files.pythonhosted.org/packages/b1/aa/c4f65ae7dd02a36b323a70c4cff326e1f3442361aaebc9311100a130d54f/regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5", size = 854736, upload-time = "2026-07-19T00:16:54.607Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/668082bcc817b9e694189b84997aeba7385b7779faa6711788679c482e35/regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82", size = 763298, upload-time = "2026-07-19T00:16:56.289Z" }, + { url = "https://files.pythonhosted.org/packages/4b/fb/2d07ad555e7af88aa5f867fdafa47a8d945ee237c20af3ebceb46a820835/regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78", size = 844430, upload-time = "2026-07-19T00:16:57.933Z" }, + { url = "https://files.pythonhosted.org/packages/51/15/c82a471fe3dce56f03745635b43aa456c40dc0db089e07ef148b331507d1/regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327", size = 789683, upload-time = "2026-07-19T00:16:59.583Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/7532a2c59d56f5398902c20de60f0c9a5d1cd364e42a051b48e1b210be7b/regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d", size = 266778, upload-time = "2026-07-19T00:17:01.032Z" }, + { url = "https://files.pythonhosted.org/packages/83/2b/cf1bc631db154eb95520d9d5dbc2371ff77a0f014bbf7d748fed8496aa63/regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965", size = 277983, upload-time = "2026-07-19T00:17:02.571Z" }, + { url = "https://files.pythonhosted.org/packages/8d/bd/56ceaf170e875d5a6761bf2bfd0d040f1cacc896850d5e40cb29b11bbd06/regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e", size = 276961, upload-time = "2026-07-19T00:17:04.135Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b9/d11d7e501ac8fd7d617684423ebb9561e0b998481c1e4cbc0cb212c5d74a/regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d", size = 496778, upload-time = "2026-07-19T00:17:05.677Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a9/a5ab6f312f24318019170dc485d5421fe4f89e43a98640da50d95a8a7041/regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd", size = 297122, upload-time = "2026-07-19T00:17:07.59Z" }, + { url = "https://files.pythonhosted.org/packages/b3/63/4cab4d7f2d384a144d420b763d97674cb70619c878ea6fcd7640d0e62143/regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6", size = 292009, upload-time = "2026-07-19T00:17:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/22/85/102a81b218298957d4ea7d2f084fae537a71add9d6ff93c8e67284c5f45e/regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797", size = 796708, upload-time = "2026-07-19T00:17:11.542Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/dc136af5629938a037cd2b304c12240e132ec92f38be8ff9cc89af2a1f2d/regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18", size = 865651, upload-time = "2026-07-19T00:17:13.312Z" }, + { url = "https://files.pythonhosted.org/packages/e0/75/67402ae3cd9c8c988a4c805d15ee3eef015e7ca4cb112cf3e640fc1f4153/regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511", size = 911756, upload-time = "2026-07-19T00:17:15.015Z" }, + { url = "https://files.pythonhosted.org/packages/2a/8e/096d00c7c480ef2ff4265349b14e2261d4ab787ba1f74e2e80d1c58079c3/regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68", size = 801798, upload-time = "2026-07-19T00:17:17.208Z" }, + { url = "https://files.pythonhosted.org/packages/f0/41/e7ecac6edb5722417f85cc67eaf386322fbe8acf6918ec2fdc37c20dd9d0/regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11", size = 776933, upload-time = "2026-07-19T00:17:19.347Z" }, + { url = "https://files.pythonhosted.org/packages/6f/69/03c9b3f058d66403e0ca2c938696e81d51cd4c6d47ec5265f02f96948d9a/regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986", size = 784338, upload-time = "2026-07-19T00:17:21.057Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f7/b38ab3d43f284afbb618fcd15d0e77eb786ae461ce1f6bc7494619ddc0f2/regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b", size = 860452, upload-time = "2026-07-19T00:17:23.119Z" }, + { url = "https://files.pythonhosted.org/packages/15/5c/ff60ef0571121714f3cf9920bc183071e384a10b556d042e0fdb06cc07a5/regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb", size = 765958, upload-time = "2026-07-19T00:17:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0f/bd34021162c0ab47f9a315bd56cd5642e920c8e5668a75ef6c6a6fca590d/regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035", size = 851765, upload-time = "2026-07-19T00:17:26.993Z" }, + { url = "https://files.pythonhosted.org/packages/2a/20/a2ca43edade0595cccfdc98636739f536d9e26898e7dbddc2b9e98898953/regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a", size = 789714, upload-time = "2026-07-19T00:17:28.699Z" }, + { url = "https://files.pythonhosted.org/packages/5d/47/e02db4015d424fc83c00ea0ac8c5e5ec14397943de9abf909d5ce3a25931/regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5", size = 267157, upload-time = "2026-07-19T00:17:31.051Z" }, + { url = "https://files.pythonhosted.org/packages/08/8e/c780c131f79b42ed22d1bd7da4096c2c35f813e835acd02ef0f018bd892c/regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312", size = 277777, upload-time = "2026-07-19T00:17:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4c/e4d7e086449bdf379d89774bf1f89dc4a41943f3c5a6125a03905b34b5fb/regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d", size = 277136, upload-time = "2026-07-19T00:17:34.803Z" }, + { url = "https://files.pythonhosted.org/packages/5d/3d/84165e4299ff76f3a40fe1f2abf939e976f693383a08d2beea6af62bd2c1/regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40", size = 496552, upload-time = "2026-07-19T00:17:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/a65293e6e4cf28eb7ee1be5335a5386c40d6742e9f47fafc8fec785e16c7/regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38", size = 296983, upload-time = "2026-07-19T00:17:38.816Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/2d0564e93d87bc48618360ddca232a2ca612bbdf53ce8465d45ca5ce14ee/regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11", size = 291832, upload-time = "2026-07-19T00:17:40.726Z" }, + { url = "https://files.pythonhosted.org/packages/07/cd/42dfbabff3dfc9603c501c0e2e2c5adbb09d127b267bf5348de0af338c15/regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13", size = 796775, upload-time = "2026-07-19T00:17:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/f6a4839f2b934e3eed5973fd07f5929ee97d4c98939fb275ea23c274ee16/regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae", size = 865687, upload-time = "2026-07-19T00:17:44.185Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/b47d6c36049bc59806a50bd4c86ced70bbe058d787f80281b1d7a9b0e024/regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da", size = 911962, upload-time = "2026-07-19T00:17:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/2a/be/ff61f28f9273658cfe23acbbac5217221f6519960ed401e61dfdab12bc35/regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15", size = 801817, upload-time = "2026-07-19T00:17:48.25Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bb/8b4f7f26b333f9f79e1b453613c39bb4776f51d38ae66dd0ba31d6b354ca/regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f", size = 776908, upload-time = "2026-07-19T00:17:50.183Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/610110fc5921d380516d03c26b652555f08aa0d23ea78a771231873c3638/regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939", size = 784426, upload-time = "2026-07-19T00:17:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/1ef9e2a83a5947c57ebff0b377cb5727c3d5ec1992317a320d035cd0dbb6/regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96", size = 860600, upload-time = "2026-07-19T00:17:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/a0/02/073af33a3ec149241d11c80acea91e722aa0adbf05addd50f251c4fe89c3/regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220", size = 765950, upload-time = "2026-07-19T00:17:56.041Z" }, + { url = "https://files.pythonhosted.org/packages/81/a9/d1e9f819dc394a568ef370cd56cf25394e957a2235f8370f23b576e5a475/regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc", size = 851794, upload-time = "2026-07-19T00:17:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/03/3a/8ae83eda7579feacdf984e71fb9e70635fb6f832eeddca58427ec4fca926/regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2", size = 789845, upload-time = "2026-07-19T00:17:59.97Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/c195cbfe5a75fdec64d8f6554fd15237b837919d2c61bdc141d7c807b08b/regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404", size = 267135, upload-time = "2026-07-19T00:18:01.958Z" }, + { url = "https://files.pythonhosted.org/packages/b2/80/a11de8404b7272b70acb45c1c05987cce60b45d5693da2e176f0e390d564/regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e", size = 277747, upload-time = "2026-07-19T00:18:04.121Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/0f5c8eff1b4f1f3d83276d365fccecf666afcc7d947420943bf394d07adb/regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8", size = 277129, upload-time = "2026-07-19T00:18:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4c/44b74742052cedda40f9ae469532a037112f7311a36669a891fba8984bb0/regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2", size = 501134, upload-time = "2026-07-19T00:18:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/f0/45/bbd038b5e39ee5613a5a689290145b40058cc152c41de9cc23639d2b9734/regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda", size = 299418, upload-time = "2026-07-19T00:18:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/c5bde94b4cedfd5850d64c3f08222d8e1600e84f6ee71d9b44b4b8163f74/regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff", size = 294486, upload-time = "2026-07-19T00:18:11.188Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/2f5e107cb26c960b781967178899daf2787a7ab151844ed3c01d6fc95474/regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1", size = 811643, upload-time = "2026-07-19T00:18:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/37/d4/a2f963406d7d73a62eed84ba05a258afb6cad1b21aa4517443ce40506b78/regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf", size = 871081, upload-time = "2026-07-19T00:18:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/45/a3/44be546340bedb15f13063f5e7fe16793ea4d9ea2e805d09bd174ac27724/regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732", size = 917372, upload-time = "2026-07-19T00:18:16.724Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e0870b0fd2a40dba0074e4b76e514b21313d37946c9248453e34ec43923e/regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a", size = 816089, upload-time = "2026-07-19T00:18:18.617Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/957e8e22690ad6634572b39b71f130a6105f4d0718bb16849eac00fff147/regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba", size = 785206, upload-time = "2026-07-19T00:18:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/186e410941e731037c01166069ab86da9f65e8f8110c18009ccf4bd623ee/regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc", size = 800431, upload-time = "2026-07-19T00:18:22.716Z" }, + { url = "https://files.pythonhosted.org/packages/73/9f/e4e10e023d291d64a33e246610b724493bf1ce98e0e59c9b7c837e5acfb7/regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62", size = 864906, upload-time = "2026-07-19T00:18:24.772Z" }, + { url = "https://files.pythonhosted.org/packages/24/57/ccb20b6be5f1f52a053d1ba2a8f7a077edb9d918248b8490d7506c6832b3/regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1", size = 773559, upload-time = "2026-07-19T00:18:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/a3/82/f3b263cf8fad927dc102891da8502e718b7ff9d19af7a2a07c03865d7188/regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e", size = 857739, upload-time = "2026-07-19T00:18:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/1687bd1b6c2aed5e672ccf845fc11557821fe7366d921b50889ea5ce57bf/regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0", size = 804522, upload-time = "2026-07-19T00:18:31.362Z" }, + { url = "https://files.pythonhosted.org/packages/76/7c/cc4e7655181b2d9235b704f2c5e19d8eff002bbc437bae59baee0e381aca/regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4", size = 269141, upload-time = "2026-07-19T00:18:33.479Z" }, + { url = "https://files.pythonhosted.org/packages/bb/14/961b4c7b05a2391c32dbc85e27773076671ef8f97f36cec70fe414734c02/regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974", size = 280036, upload-time = "2026-07-19T00:18:35.419Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/795644550d788ddbb6dc458c95895f8009978ea6d6ea76b005eb3f45e8c9/regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d", size = 279394, upload-time = "2026-07-19T00:18:37.454Z" }, + { url = "https://files.pythonhosted.org/packages/d2/25/0c4c452f8ef3efe456745b2f33195f5904b573fb4c2ff3f0cb9ec188461e/regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd", size = 496750, upload-time = "2026-07-19T00:18:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/24/9e/b70ca6c1704f6c7cd32a9e143c86cc5968d10981eca284bad670c245ea7d/regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac", size = 297093, upload-time = "2026-07-19T00:18:41.583Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/0b692da2520d51fbff19c88b83d97e4c702909dd02386c585998b7e2dbed/regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5", size = 292043, upload-time = "2026-07-19T00:18:43.347Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a7/1d478e614016045a33feae57446215f9fd65b665a5ceb2f891fb3183bc52/regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3", size = 797214, upload-time = "2026-07-19T00:18:45.362Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ae/11b9c9411d92c30e3d2db32df5a31133e4a99a8fc397a604fd08f6c4bffb/regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053", size = 866433, upload-time = "2026-07-19T00:18:47.315Z" }, + { url = "https://files.pythonhosted.org/packages/b1/62/2b2efc4992f91d6d204b24c647c9f9412e85379d92b7c0ab9fdae622327e/regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b", size = 911360, upload-time = "2026-07-19T00:18:49.588Z" }, + { url = "https://files.pythonhosted.org/packages/14/71/986ceea9aa3da548bf1357cad89b63915ec6d21ec957c8113b29ece567df/regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a", size = 801275, upload-time = "2026-07-19T00:18:51.767Z" }, + { url = "https://files.pythonhosted.org/packages/15/be/ce9d9534b2cda96eab32c548261224b9b4e220a4126f098f60f42ae7b4cd/regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1", size = 777131, upload-time = "2026-07-19T00:18:54.053Z" }, + { url = "https://files.pythonhosted.org/packages/61/2b/58b5c710f2c3929515a25f3a1ca0dad0dcd4518d4fff3cf23bc7adb8dcd2/regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e", size = 785020, upload-time = "2026-07-19T00:18:56.579Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/5fe091935b74f15fe0f97998c215cae418d1c0413f6258c7d4d2e83aa37f/regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12", size = 861263, upload-time = "2026-07-19T00:18:58.64Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fa/d60bf82e10841eef62a9e32aac401468f05fddfbcb2942e342b1ba3d2433/regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2", size = 766199, upload-time = "2026-07-19T00:19:00.705Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5d/11e64d151b0662b81d6bf644c74dc118d461df85bdf2577fadbbf751788a/regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97", size = 851317, upload-time = "2026-07-19T00:19:03.015Z" }, + { url = "https://files.pythonhosted.org/packages/7c/34/532efb87488d90807bae6a443d357ee5e2728a478c597619c8aaa17cc0bd/regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4", size = 789557, upload-time = "2026-07-19T00:19:05.338Z" }, + { url = "https://files.pythonhosted.org/packages/d6/90/3a8d5ca977171ec3ae21a71207d2228b2663bde14d7f7ef0e6363ecf9290/regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa", size = 272531, upload-time = "2026-07-19T00:19:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/96/e1/8862885e70409de70e8c005f57fb2e7be8d9ef0317250d60f4c9660a300d/regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac", size = 280831, upload-time = "2026-07-19T00:19:09.46Z" }, + { url = "https://files.pythonhosted.org/packages/08/82/2693e53e29f9104d9de95d37ce4dd826bd32d5f9c0085d3aa6ac042675c4/regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459", size = 281099, upload-time = "2026-07-19T00:19:11.398Z" }, + { url = "https://files.pythonhosted.org/packages/92/b7/9a01aa16461a18cde9d7b9c3ab21e501db2ce33725f53014342b91df2b0a/regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3", size = 501121, upload-time = "2026-07-19T00:19:13.425Z" }, + { url = "https://files.pythonhosted.org/packages/f3/5e/bbaeca815dc9191c424c94a4fdc5c87c75748a64a6271821212ebdd4e1a3/regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518", size = 299415, upload-time = "2026-07-19T00:19:15.43Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d6/0dd1a321afaab95eb7ff44aa0f637301786f1dc71c6b797b9ed236ed8890/regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9", size = 294483, upload-time = "2026-07-19T00:19:17.879Z" }, + { url = "https://files.pythonhosted.org/packages/92/5f/40bacf91d0904f812e13bbbab3864604c463eced8afdc54aeaa50492ea95/regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435", size = 811833, upload-time = "2026-07-19T00:19:20.102Z" }, + { url = "https://files.pythonhosted.org/packages/94/7c/4902744261f775aeede8b5627314b38482da29cf49a57b66a6fb753246c5/regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0", size = 871270, upload-time = "2026-07-19T00:19:22.365Z" }, + { url = "https://files.pythonhosted.org/packages/16/70/6980c9be6bf21c0a60ed3e0aea39cf419ecf3b08d1d9947bc56e196ef186/regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a", size = 917534, upload-time = "2026-07-19T00:19:24.529Z" }, + { url = "https://files.pythonhosted.org/packages/52/92/8b2bd872782ce8c42691e39acb38eb8efe014e5ddb78ad7d943d6f197ce9/regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276", size = 816135, upload-time = "2026-07-19T00:19:26.919Z" }, + { url = "https://files.pythonhosted.org/packages/de/2d/33a602f657bdc4041f17d79f92ab18261d255d91a06117a6e29df023e5e2/regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c", size = 785492, upload-time = "2026-07-19T00:19:29.192Z" }, + { url = "https://files.pythonhosted.org/packages/9e/36/0987cf4cb271680064a70d24a475873775a151d0b7058698a006cb0cae4a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a", size = 800658, upload-time = "2026-07-19T00:19:31.392Z" }, + { url = "https://files.pythonhosted.org/packages/a8/24/c14f31c135e1ba55fa4f9a58ca98d0842512bf6188230763c31c8f449e3b/regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009", size = 865073, upload-time = "2026-07-19T00:19:33.485Z" }, + { url = "https://files.pythonhosted.org/packages/14/85/181a12211f22469f24d2de1ebddfe397d2396e2c29013b9a58134a91069a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218", size = 773684, upload-time = "2026-07-19T00:19:35.599Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/bd1a0c1a62251366f8d21f41b1ea3c76994962071b8b6ea42f72d505c0f0/regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966", size = 857769, upload-time = "2026-07-19T00:19:37.738Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4f/f7e2dad6756b2fe1fe75dd90a628c3b45f249d39f948dd90cd2476325417/regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44", size = 804546, upload-time = "2026-07-19T00:19:40.229Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d7/01d31d5bdb09bc026fab77f59a371fdf8f9b292e4810546c56182ca70498/regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78", size = 274526, upload-time = "2026-07-19T00:19:42.398Z" }, + { url = "https://files.pythonhosted.org/packages/52/0e/cea4ce73bc0a8247a0748228ae6669984c7e1f8134b6fa66e59c0572e0ea/regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2", size = 283763, upload-time = "2026-07-19T00:19:44.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b6/26e41975febae63b7a6e3e02f32cff6cff2e4f10d19c929082f56aebf7c6/regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547", size = 283451, upload-time = "2026-07-19T00:19:46.639Z" }, ] [[package]] From e125079a5f63898f53ec722dbba0aef9a35dc5ba Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 24 Jul 2026 19:33:11 +0000 Subject: [PATCH 28/82] Refresh Privacy Guard guidance and examples --- projects/privacy-guard/AGENTS.md | 126 ++--- projects/privacy-guard/BENCHMARKS.md | 89 ---- projects/privacy-guard/README.md | 450 +++++++----------- .../examples/email-scanner/.gitignore | 1 - .../examples/email-scanner/README.md | 206 -------- .../examples/email-scanner/gateway.toml | 12 - .../email-scanner/middleware_server.py | 50 -- .../examples/email-scanner/policy.yaml | 49 -- .../examples/regex-engine/README.md | 15 + .../examples/regex-engine/patterns.yaml | 11 + .../regex-engine/privacy-guard-config.yaml | 22 + .../examples/regex-scanner/.gitignore | 4 - .../examples/regex-scanner/README.md | 338 ------------- .../examples/regex-scanner/gateway.toml | 12 - .../regex-scanner/pi-models.template.json | 16 - .../regex-scanner/policy.pi.template.yaml | 42 -- .../examples/regex-scanner/policy.yaml | 49 -- .../examples/regex-scanner/regex-scanner.yaml | 10 - 18 files changed, 284 insertions(+), 1218 deletions(-) delete mode 100644 projects/privacy-guard/BENCHMARKS.md delete mode 100644 projects/privacy-guard/examples/email-scanner/.gitignore delete mode 100644 projects/privacy-guard/examples/email-scanner/README.md delete mode 100644 projects/privacy-guard/examples/email-scanner/gateway.toml delete mode 100644 projects/privacy-guard/examples/email-scanner/middleware_server.py delete mode 100644 projects/privacy-guard/examples/email-scanner/policy.yaml create mode 100644 projects/privacy-guard/examples/regex-engine/README.md create mode 100644 projects/privacy-guard/examples/regex-engine/patterns.yaml create mode 100644 projects/privacy-guard/examples/regex-engine/privacy-guard-config.yaml delete mode 100644 projects/privacy-guard/examples/regex-scanner/.gitignore delete mode 100644 projects/privacy-guard/examples/regex-scanner/README.md delete mode 100644 projects/privacy-guard/examples/regex-scanner/gateway.toml delete mode 100644 projects/privacy-guard/examples/regex-scanner/pi-models.template.json delete mode 100644 projects/privacy-guard/examples/regex-scanner/policy.pi.template.yaml delete mode 100644 projects/privacy-guard/examples/regex-scanner/policy.yaml delete mode 100644 projects/privacy-guard/examples/regex-scanner/regex-scanner.yaml diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md index 05fe35d..80f6b00 100644 --- a/projects/privacy-guard/AGENTS.md +++ b/projects/privacy-guard/AGENTS.md @@ -1,7 +1,8 @@ # Privacy Guard -Privacy Guard is OpenShell middleware that scans request bodies and applies -policy to detected sensitive data. +Privacy Guard is OpenShell middleware that runs an ordered pipeline of +entity-processing engines over one UTF-8 request body and applies a user-facing +detect, block, or replace action. ## Development commands @@ -10,96 +11,113 @@ Run commands from `projects/privacy-guard/`. - List targets: `make help` - Run all checks: `make check` - Check Python 3.11: `make check-py311` -- Run focused tests: `make test PYTEST_ARGS=tests/test_processor.py` -- Run the benchmark: `make benchmark` +- Run focused tests: `make test PYTEST_ARGS=tests/test_request_processor.py` -Run focused tests while working and `make check` before handoff. Benchmarks are -diagnostic and have no pass/fail threshold. +Run focused tests while working and `make check` before handoff. ## Engineering approach -- Do not preserve backward compatibility unless the user explicitly requests - it. Update callers, tests, examples, and docs with the change. +- Backwards compatibility is explicitly not a concern for the v0 redesign. Do + not restore legacy behavior, schemas, imports, names, tests, or examples. - Add defensive handling only for a concrete failure mode at the layer that owns it. Avoid speculative guards, duplicate validation, broad catches, retries, and fallbacks. +- Prefer explicit, domain-specific names. Avoid generic intermediate + abstractions that do not own behavior. +- Keep public declarations before private helper types, functions, methods, and + constants when dependency ordering permits. Put private implementation + details at the bottom of their module or class. ## Project map -- `src/privacy_guard/scanners/`: scanner contract, budgets, and built-ins -- `src/privacy_guard/request_body/`: normalization and reconstruction contracts -- `src/privacy_guard/processor.py`: request orchestration and policy +- `src/privacy_guard/engines/`: engine contract and built-in implementations +- `src/privacy_guard/config.py`: policy action and ordered stage configuration +- `src/privacy_guard/engine_registry.py`: registration and finalized config union +- `src/privacy_guard/request_processor.py`: stage execution and policy disposition +- `src/privacy_guard/base.py`: package-wide strict immutable domain-model base +- `src/privacy_guard/string_validators.py`: shared string validators and field types - `src/privacy_guard/service/`: gRPC lifecycle and protobuf adapter - `src/privacy_guard/bindings/`: generated protobuf files; never hand-edit - [`docs/architecture/`](docs/architecture/index.md): symlink to the canonical site sources under `../../docs/documentation/privacy-guard/architecture/` - `tests/`: tests that mirror source boundaries -- `tests/test_hardening.py`: cross-cutting security tests -- `examples/`: copyable deployments +- `examples/`: copyable policy-authoring examples -Keep each example's policy, configuration, commands, names, and tests in sync. -Before changing `processor.py`, `scanners/`, `request_body/`, or `service/`, -read the architecture overview and the matching topic page. -Architecture documentation changes follow +Before changing `request_processor.py`, `engines/`, or `service/`, read the +architecture overview and matching topic page. Architecture changes follow [`docs/development/index.md`](../../docs/development/index.md) and require its checks. ## Design boundaries -- Keep scanners separate from `RequestProcessor` and `service/`. A scanner - inspects one text block and returns block-relative findings. Do not make - scanners depend on request-body formats, policy, gRPC, or generated bindings. -- `RequestProcessor` orchestrates format handlers and scanners, applies policy, - attaches text-block paths, and enforces aggregate limits. +- One processor call receives one text string. Do not reintroduce request-body + codecs, format handlers, document regions, or JSON traversal. +- An `EntityProcessingEngine` receives engine configuration, a processing + strategy, and a shared `Timeout`. It never receives or infers the policy + action. +- `RequestProcessor` runs configured stages in order and owns detect, block, or + replace disposition. +- Engine configuration lives inside the OpenShell policy as the exact Pydantic + discriminated-union member registered for that engine. +- Deployment startup owns only installed engine implementations and operational + resources such as clients, endpoints, models, and credentials. +- Engine instances and injected resources serve concurrent requests. Do not + retain request content or mutable per-request state. - Outside generated `bindings/`, only `service/` may import gRPC or generated bindings. -- Public scanner and format-handler methods validate extension output. - `RequestProcessor` validates finding identities and offsets. Do not repeat - these checks elsewhere. -- `ScannerConfig.entity_types` lists every entity the scanner can emit; it is - independent of policy. Apply policy after scanning. -- Scanner and format-handler instances serve concurrent requests. Do not store - request content or mutable per-request state on them. -- Format handlers define path syntax. Treat paths as opaque everywhere else. +- The copied OpenShell `.proto` and generated bindings must be updated only + through `openshell-middleware-kit`; never edit them manually. ## Extension pattern -Define a concrete config type and implement `_scan`. Do not override `scan`. +Define a concrete `EngineConfig` and implement `_run`. Custom engines do not +define `__init__`; use optional `_initialize` for derived immutable state. +`@override` is not required. ```python -from pydantic import Field +from typing import Literal -from privacy_guard.scanners import Finding, ScanBudget, Scanner, ScannerConfig +from privacy_guard.engines import ( + EngineConfig, + EntityProcessingEngine, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.base import StrictDomainModel +from privacy_guard.timeout import Timeout -class KeywordScannerConfig(ScannerConfig): - keyword: str = Field(min_length=1) +class KeywordReplacement(StrictDomainModel): + strategy: Literal["token"] = "token" -class KeywordScanner(Scanner[KeywordScannerConfig]): - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - start = text_block.find(self.config.keyword) - if start < 0: - return () - return ( - Finding( - entity="keyword", - scanner_name=self.scanner_name, - start_offset=start, - end_offset=start + len(self.config.keyword), - ), - ) +class KeywordConfig(EngineConfig[KeywordReplacement]): + engine: Literal["keyword"] = "keyword" + keyword: str + + +class KeywordEngine(EntityProcessingEngine[KeywordConfig, None]): + supported_strategy = EntityProcessingStrategy.DETECT + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + timeout.raise_if_expired() + return TextProcessingResult(text=text, detections=()) ``` -Scanners that need custom initialization logic should implement `_initialize` -instead of `__init__`. The base constructor calls it last, after setting -validated configuration and scanner metadata. +The public `run` method validates extension output. Register every engine before +finalizing the registry so policy serialization retains its exact config type. ## Change limits - Add or update tests at the layer that owns the behavior. -- Ask before adding dependencies or changing the protobuf contract, stable error - codes, protocol limits, or fail-closed defaults. -- Do not remove or weaken tests to pass checks. +- Ask before adding dependencies or changing the OpenShell protobuf contract, + stable error codes, protocol limits, or fail-closed defaults. +- Do not remove or weaken relevant tests merely to pass checks. - Do not add casts, explicit `Any`, blanket ignores, or broad type suppressions to handwritten code. `tests/test_typing_policy.py` enforces this. diff --git a/projects/privacy-guard/BENCHMARKS.md b/projects/privacy-guard/BENCHMARKS.md deleted file mode 100644 index 85b2946..0000000 --- a/projects/privacy-guard/BENCHMARKS.md +++ /dev/null @@ -1,89 +0,0 @@ -# Diagnostic benchmark - -This is a manual development tool, not a contributor or release gate. It has no -pass/fail thresholds, and results should be compared only across equivalent, -controlled environments. The recorded table below is a one-time snapshot, not -an evergreen performance baseline. - -The benchmark invokes `RequestProcessor.process` and therefore covers JSON -normalization, scanner calls, scanner-output validation, contextual finding -validation, policy application, and body reconstruction. Input bodies are exact -1 KiB, 1 MiB, and 4 MiB JSON payloads. The maximum load is the request limit of -4,096 findings, distributed across 16 blocks; multiple-scanner cases use four -scanners. Replacement cases redact matched characters to an empty string so the -4 MiB input remains within the output limit. Each scanner constructs normal -strict `Finding` models during every measured scan; no finding output is reused -between requests. The full suite also contains a separate maximum-width numeric -array: exactly 4,194,303 bytes and 2,097,151 numeric elements, with zero text -blocks and findings. It exposes the complete duplicate-aware stdlib parse and -single cached strict Pydantic adapter path without large-string scanning costs. - -JSON normalization stores the adapter's `JsonValue` output directly in private -handler state. It does not build an immutable mirror tree. Incremental text-block -traversal retains state proportional to nesting depth, no-op reconstruction -returns the original bytes without walking the tree, and replacement -reconstruction performs one `deepcopy`. - -Run the recorded suite with: - -```bash -uv run --frozen python scripts/benchmark_privacy_guard.py --suite full -``` - -Wall-clock samples are collected without tracing. Peak allocations are measured -in separate runs with `tracemalloc`, preventing allocation tracing from changing -the wall-time result. Each reported value is the median of seven runs after one -verified warm-up. `--profile profile.out` adds one verified `cProfile` pass per -scenario without contaminating either measurement. - -Reference environment: CPython 3.11.9 on Apple arm64, macOS 26.5.2. Recorded -2026-07-22 with the command above. - -| Scenario | Input bytes | Findings | Scanners | Reconstruction | Median wall (ms) | Median peak traced allocation (bytes) | -| --- | ---: | ---: | ---: | --- | ---: | ---: | -| 1KiB-zero-one-noop | 1,024 | 0 | 1 | no-op | 0.075 | 7,043 | -| 1KiB-typical-multiple-replacement | 1,024 | 8 | 4 | replacement | 0.192 | 19,563 | -| 1MiB-typical-one-noop | 1,048,576 | 8 | 1 | no-op | 55.380 | 2,101,945 | -| 1MiB-max-one-replacement | 1,048,576 | 4,096 | 1 | replacement | 116.764 | 8,252,749 | -| 4MiB-zero-multiple-noop | 4,194,304 | 0 | 4 | no-op | 434.537 | 8,393,513 | -| 4MiB-max-multiple-replacement | 4,194,304 | 4,096 | 4 | replacement | 542.377 | 20,834,860 | -| 1KiB-typical-one-noop | 1,024 | 8 | 1 | no-op | 0.093 | 14,696 | -| 1MiB-zero-one-noop | 1,048,576 | 0 | 1 | no-op | 56.848 | 2,101,890 | -| 1MiB-typical-multiple-replacement | 1,048,576 | 8 | 4 | replacement | 129.166 | 4,209,498 | -| 1MiB-max-multiple-noop | 1,048,576 | 4,096 | 4 | no-op | 126.820 | 5,117,784 | -| 4MiB-typical-one-replacement | 4,194,304 | 8 | 1 | replacement | 303.447 | 16,792,456 | -| 4MiB-typical-multiple-noop | 4,194,304 | 8 | 4 | no-op | 437.978 | 8,393,401 | -| 4MiB-max-one-noop | 4,194,304 | 4,096 | 1 | no-op | 229.557 | 8,402,617 | -| 4194303B-2097151-elements-wide-numeric-zero-one-noop | 4,194,303 | 0 | 1 | no-op | 726.707 | 38,102,178 | - -The focused 100,000-element numeric-array regression uses a 200,001-byte body. -On the same interpreter, complete normalization peaked at 1,802,636 traced -bytes (below its committed 8 MiB ceiling), while walking the already validated -tree peaked at 1,067 traced bytes (below 64 KiB), demonstrating that walker -state does not grow with container width. - -`tracemalloc` reports traced Python allocations, not process RSS, allocator -arena retention, native allocations, or transient memory outside its tracing -domain. The maximum-width peak includes the unavoidable overlap while the -cached Pydantic adapter constructs its validated output from the stdlib parse -tree; the raw parse tree becomes unreachable when the parse helper returns, -before text-block materialization begins. - -The harness deliberately isolates processor overhead with a synthetic scanner. -It does not include protobuf conversion, gRPC transport, executor or semaphore -queuing, concurrent throughput, or whole-process memory. - -## Regex catalog scalability gate - -A separate one-shot gate measured the required 1,000 active entities and 10,000 -active patterns. The generated 749,790-byte single-profile YAML gave every -entity ten unique literal patterns. The representative sparse input was 46 -characters and produced zero matches. This deliberately measures catalog -evaluation rather than dense-result aggregation, which has independent limits. - -Reference environment: CPython 3.14.4 on Linux x86_64 in the Codex workspace. -Recorded 2026-07-22 using the stdlib `re` engine. Parsing, strict validation, -and compiling took 14.399 seconds with 132.1 MiB peak traced allocation. -Scanning took 9 ms, within the one-second default request budget. These are -reproducibility snapshots, not release thresholds; the shared deadline remains -the cooperative runtime CPU budget. diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 529f84a..702854b 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -1,331 +1,209 @@ # Privacy Guard -An OpenShell supervisor middleware. The supervisor calls it over gRPC (the -`SupervisorMiddleware` service in `bindings/`) to inspect provider-bound HTTP -requests *before* credentials are attached. Privacy Guard parses the request -body, scans its text for sensitive values, and returns an allow/deny decision -plus an optionally rewritten body that the supervisor forwards instead of the -original. +Privacy Guard is an OpenShell supervisor middleware that detects and optionally +replaces sensitive entities in provider-bound request text before credentials +are attached. + +This release is a clean-break redesign. It does not preserve the former +`Scanner`, `FormatHandler`, JSON traversal, `observe`, `redact`, startup catalog, +or scanner-profile APIs. A processor run accepts one UTF-8 text body and runs +the policy's entity-processing stages in order. + +## Policy experience + +The OpenShell policy owns entity behavior: ordered stages, each engine's exact +configuration, and the final `detect`, `block`, or `replace` action. + +```yaml +entity_processing: + stages: + - name: identifiers + config: + engine: regex + pattern_catalog: + entities: + - name: email + patterns: + - pattern: '(? Status: **hardened research project with a configurable regex scanner.** Strict -> policy and scanner configuration, request-level processing, bounded scanning, -> the generic JSON handler, safe gRPC adaptation, and a loopback server are -> implemented. +`entity_processing.stages` is ordered. In replace mode, each stage receives the +preceding stage's processed text. Detect and block run the same engines with the +detection-only strategy, so replacement recipes may remain configured but +dormant. + +Privacy Guard accepts a structured catalog, not a filesystem path. The +[regex engine example](examples/regex-engine/README.md) includes a reference +catalog to copy and adapt. Transparent catalog-file expansion belongs in +OpenShell's policy installation flow and is not yet supported by the current +protocol. + +## Architecture + +```text +OpenShell HttpRequestEvaluation + -> strict UTF-8 decode + -> finalized Pydantic policy union (config.engine discriminator) + -> canonical config fingerprint and bounded processor cache + -> RequestProcessor.process(one text string) + -> stage 1 engine.run(current text) + -> stage 2 engine.run(stage 1 text) + -> ... + -> policy action: detect, block, or replace + -> safe aggregated entity findings + -> OpenShell HttpRequestResult +``` -The self-contained -[built-in regex scanner walkthrough](examples/regex-scanner/README.md) provides -a scanner configuration, gateway registration, sandbox policy, and manual -Claude Code and custom-endpoint Pi workflows for a typical deployment. The -separate [email scanner example](examples/email-scanner/README.md) demonstrates -how to implement a custom scanner. +The policy action never crosses the engine boundary. Engines receive only +`EntityProcessingStrategy.DETECT` or `EntityProcessingStrategy.REPLACE`. +Blocking is a request-level disposition owned by `RequestProcessor`. -## Request flow +The copied `proto/supervisor_middleware.proto` and generated bindings are owned +by OpenShell. Update them only through the repository's middleware-kit workflow; +never hand-edit them. Today's protocol carries a `google.protobuf.Struct` +configuration on each evaluation, so Privacy Guard validates and caches it +internally. Large-catalog preparation RPCs, evaluation fingerprints, manifest +schema fields, and a dedicated finding-source field require a coordinated +change in the canonical OpenShell protocol rather than a private proto fork. -`RequestProcessor` orchestrates one complete request: +## Built-in engines -``` -proto HttpRequestEvaluation - -> payloads.InterceptedRequest proto-free capture of the request - -> request_body.FormatHandler.normalize() - -> request_body.RequestBody (TextBlocks) - -> scanners.Scanner.scan(text_block) -> scanners.Finding[] - -> policy via config.PolicyConfig -> per-block replacements - -> request_body.FormatHandler.reconstruct() - -> rewritten body (bytes) - -> payloads.ProcessingResult decision + replacement + findings - -> proto HttpRequestResult -``` +### RegexEngine -The servicer is the only seam that touches both the proto messages and the -domain types: it translates proto -> domain on the way in and domain -> proto on -the way out. Everything below the service layer is free of `bindings/` and -gRPC. OpenShell applies an allowed mutation and forwards the provider request; -Privacy Guard does not make the provider call itself. - -Scanners continue to operate on exactly one text block. Scanner calls run on a -dedicated four-thread executor rather than the gRPC event loop; scanners must be -thread-safe and must not retain request content. Format handlers decide -which text blocks are relevant and own their addressing scheme. The processor -scans every emitted text block but treats each block path as opaque, so adding a -new request format does not require provider-specific processor logic. - -## Scanner and JSON policy - -Findings carry scanner-owned `low`, `medium`, or `high` confidence. Each action -selects the findings it applies to by `entity_types` and `minimum_confidence`: - -```json -{ - "body_format": "json", - "on_finding": { - "action": "redact", - "entity_types": ["email"], - "minimum_confidence": "high", - "template": "[{entity}]" - } -} -``` +`RegexEngine` compiles configured patterns once and supports overlapping +detection and deterministic template replacement. It preserves numeric +backreferences by wrapping each configured pattern in a non-capturing group +followed by a private named marker. Pattern names are optional diagnostic +identities; `pattern` is the only field containing the regex string. -`PolicyConfig.on_finding` is a Pydantic discriminated union keyed by `action`. -Observe, block, and redact each carry their own finding criteria; only the -redact variant has a `template`. `entity_types: null` selects every emitted -entity type, while an empty list selects none. `minimum_confidence: null` -accepts every confidence level. - -Scanner configuration is independent of policy. A concrete `ScannerConfig` -subtype controls what and how its scanner detects; scanners can run without a -`PolicyConfig`. Action criteria filter the resulting `Finding` values for one -policy evaluation and never reconfigure the scanner. - -Every scanner config declares its complete `entity_types` catalog. When an -action selects entity types, every configured name must occur in the union of -the active scanner catalogs. Unknown names fail config validation instead of -silently producing no findings. - -Every JSON string value and every object key is scanned; JSON numbers, booleans, -and nulls are not. Keys are observable in `observe`, cause normal denial in -`block`, and cause a stable deny in `redact` because collision-safe key mutation -is not supported. Value findings may overlap for observe/block. Redaction picks -winners deterministically by confidence, span length, offsets, scanner identity, -and entity. A scanner sequence is passed to `RequestProcessor`; scanner names -must be unique and remain visible in aggregated findings. - -The `privacy-guard` command runs built-in scanners. Every built-in requires -`--config` and owns that file's schema and interpretation. -`RegexScanner` is available through the `regex` subcommand; its config path -selects a YAML catalog. Single-profile files contain a non-empty entity list. -Multi-profile files contain only a non-empty `profiles` mapping and require the -regex-specific `--profile` option. Use `privacy-guard --help` for the built-in -scanner list or `privacy-guard regex --help` for regex options. See -[the built-in regex scanner walkthrough](examples/regex-scanner/README.md) for a -complete manual deployment. +The third-party `regex` backend provides enforceable per-search timeouts. +Explicit `ignore_case`, `multiline`, `dot_all`, and `ascii` flags are supported; +inline flags and user-defined named groups are rejected to protect the wrapper +contract. -```bash -uv run privacy-guard \ - regex \ - --config examples/regex-scanner/regex-scanner.yaml \ - --listen 127.0.0.1:50051 -``` +Privacy Guard owns the catalog schema but maintains no authoritative patterns. -Each entity has a unique name and a non-empty `patterns` list. Patterns declare -`name`, `regex`, `confidence`, and optional `ignore_case`, `multiline`, -`dot_all`, and `ascii` booleans. Every match carries its configured pattern name -in general finding metadata and is reported as `entity/pattern-name` by the -service. Policy filtering remains at entity level. +## Custom engines -A scanner is a nominal extension: declare its strict configuration type and -implement `_scan`. Use `_initialize` for any custom initialization logic rather -than defining `__init__`; the base constructor runs the hook at the end of -initialization, after validating and retaining the configuration. The public -`scan` wrapper validates the returned tuple and each `Finding`. +Custom engines are a first-class extension point. Authors declare one typed +config, optional typed resources, `supported_strategy`, and `_run`. They do not +write `__init__`; `_initialize` is optional, and `@override` is not required. + +The first NeMo Anonymizer integration will be implemented as a custom engine, +not as a built-in or placeholder abstraction in Privacy Guard. ```python -from privacy_guard.scanners import Finding, ScanBudget, Scanner, ScannerConfig +from dataclasses import dataclass +from typing import Literal +from privacy_guard.engines import ( + EngineConfig, + EntityProcessingEngine, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.base import StrictDomainModel +from privacy_guard.timeout import Timeout -class ExampleScanner(Scanner[ScannerConfig]): - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - return () -``` -Applications construct the processor with a sequence, for example -`RequestProcessor([ExampleScanner(ScannerConfig(name="example", entity_types=frozenset()))])`. -The base -constructor infers and validates the scanner's declared config type. A scanner -returns block-relative -`Finding` values; the processor composes each one into a `RequestBodyFinding` -with the owning `TextBlock.path`. +class AcmeReplacement(StrictDomainModel): + strategy: Literal["token"] = "token" -Applications serving a custom scanner can use the high-level server API, which -owns processor, middleware, gRPC, and shutdown wiring: -```python -from privacy_guard.service import MiddlewareServer +class AcmeConfig(EngineConfig[AcmeReplacement]): + engine: Literal["acme-pii"] = "acme-pii" -scanner = ExampleScanner(ScannerConfig(name="example", entity_types=frozenset())) -server = MiddlewareServer(scanner=scanner) -server.serve() # Defaults to 127.0.0.1:50051 -``` -## Resource and failure behavior - -The service enforces the protocol's 4 MiB input and replacement-body maximum, -32 finding groups, and exact 4 KiB encoded limit for each aggregate finding. -The operator may configure a lower effective maximum; this project cannot observe -that value, so deployments must keep the registration aligned with the 4 MiB -manifest or add the lower limit to service configuration. - -JSON parsing is additionally bounded to 64 nesting levels, 4,096 text blocks, -and 4 MiB of scanned characters. Scanning is capped at 256 findings per block, -4,096 per request, four active scanner workers, and 16 concurrent gRPC calls. -One default one-second monotonic scan budget is shared across every block and -scanner in a request. `RegexScanner` checks it before and after each expression -evaluation and while consuming overlapping matches. Because the standard-library -regex engine cannot interrupt an active evaluation, one backtracking-heavy -expression may outlast the deadline. Test catalogs against representative -worst-case inputs before deployment and avoid expressions with pathological -backtracking behavior. -Shape excess is invalid input. Finding or outbound representation excess returns -a stable `privacy_guard_limit_exceeded` deny with no body or partial findings, -avoiding a failure-mode-dependent fail open. -Redacted text is projected against the body budget before it is rendered, which -bounds template and finding amplification. The authoritative serialized-body -limit is checked immediately after format reconstruction and again at the gRPC -boundary. - -Operational logs contain request ID, duration, action, finding count, and safe -error code only—never bodies, text blocks, or matches. A cancelled RPC does not -release its scanner slot until its synchronous worker really exits. - -## Module map - -| Module | Responsibility | -| --- | --- | -| `config` | Strict Pydantic `PolicyConfig` parsing at the untrusted config boundary | -| `constants` | Package-wide limits, service metadata, and stable protocol values | -| `errors` | Closed, content-safe error catalog shared by all components | -| `payloads` | Frozen `InterceptedRequest` and `ProcessingResult` domain records | -| `request_body` | Nominal `FormatHandler` ABC + `JsonHandler`; strict `RequestBody`, `TextBlock` models | -| `scanners` | `Scanner` ABC + strict `ScannerConfig`, `Finding`, and `RequestBodyFinding` models | -| `processor` | Proto-free request orchestration and policy application | -| `service` | High-level `MiddlewareServer`, gRPC lifecycle, and servicer adapter | -| `bindings` | generated protobuf stubs — do not edit by hand | +@dataclass(frozen=True) +class AcmeResources: + client: object -## Updating the OpenShell protocol -Privacy Guard uses -[`openshell-middleware-kit`](../openshell-middleware-kit/README.md) to keep its -checked-in OpenShell protocol and generated Python bindings aligned with a -released OpenShell version. Install the repository's local copy as the `omkit` -tool: +class AcmeEngine(EntityProcessingEngine[AcmeConfig, AcmeResources]): + supported_strategy = EntityProcessingStrategy.REPLACE -```bash -uv tool install --force ../openshell-middleware-kit + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + timeout.raise_if_expired() + return TextProcessingResult(text=text, detections=()) ``` -From this directory, update to the latest OpenShell release with: +Register engines before finalizing the policy schema: -```bash -omkit update +```python +from privacy_guard.engine_registry import EngineRegistry + +registry = EngineRegistry() +registry.register(AcmeEngine, resources=AcmeResources(client=client)) +registry.finalize() ``` -For a reproducible update to a specific release, pin the version: +The finalized registry builds a Pydantic discriminated union containing the +exact config type of every registered engine. `stage.config` therefore +round-trips without dropping engine-specific or replacement-variant fields. + +## CLI ```bash -omkit update --openshell-version v0.0.90 +uv run privacy-guard engines +uv run privacy-guard schema +uv run privacy-guard serve --listen 127.0.0.1:50051 ``` -The updater works on a validated temporary copy and replaces only -`proto/supervisor_middleware.proto`, `src/privacy_guard/bindings/`, `uv.lock`, -and `.openshell-middleware-manifest.json`. The manifest records the selected -OpenShell release, protocol source and checksum, and -`openshell-middleware-kit` version. Review all generated changes, then run -`make check`; never edit generated bindings by hand. - -## Notes for implementers - -- **Scanner metadata.** Applications pass an explicit `ScannerConfig` subtype to - the scanner constructor; `Scanner` infers and validates the declared generic - config type. The read-only `config` property preserves that concrete type, and - `supported_entity_types` returns its required `entity_types` catalog. -- **Finding types.** A scanner returns strict block-relative `scanners.Finding` - values. The processor creates `RequestBodyFinding` values with the owning text-block - path, and the servicer aggregates those into the protocol's count-based `Finding`. - A finding may include an immutable, bounded string metadata mapping for - scanner-specific attribution. Regex findings use its `pattern_name` key. -- **Scanner initialization.** Override `_initialize` only when validated config - must be compiled or transformed into reusable immutable scanner state. The - default hook does nothing. -- **Scan budgets.** The protected scanner hook receives a request-scoped - `ScanBudget`. Standalone `scan` calls create a safe default when no budget is - supplied. Potentially unbounded scanners must cooperate with the deadline. -- **Text-block paths.** `Finding` has no path state. A `Scanner` sees only one text - block; the processor attaches `TextBlock.path` by creating `RequestBodyFinding`. -- **Format selection.** `PolicyConfig.body_format` (default `json`) picks a handler - from the `format_handlers` mapping supplied to `RequestProcessor`. JSON is the - only built-in format; applications can supply mappings containing other formats. - Do not infer a provider from the request body or headers. -- **Format handlers.** A custom handler subclasses `FormatHandler`, calls - `super().__init__(format_name="...")`, and implements the protected - `_normalize` and `_reconstruct` hooks with `@override`. Return normally - constructed strict `RequestBody` and `TextBlock` models; handler instances are - reused concurrently and must retain no request content or mutable request state. - - ```python - from collections.abc import Mapping - - from typing_extensions import override - - from privacy_guard.request_body import FormatHandler, RequestBody - from privacy_guard.config import PolicyConfig - - - class CustomHandler(FormatHandler): - def __init__(self) -> None: - super().__init__(format_name="custom") - - @override - def _normalize(self, raw_body: bytes, policy_config: PolicyConfig) -> RequestBody: - return RequestBody( - text_blocks=(), parsed_value=None, original_bytes=raw_body - ) - - @override - def _reconstruct( - self, - request_body: RequestBody, - replacements_by_path: Mapping[str, str], - ) -> bytes: - return request_body.original_bytes - ``` -- **Log safety.** Raw bodies, parsed values, and text-block content use `repr=False` to - keep content out of routine domain representations; this does not sanitize - arbitrary tracebacks. Cataloged errors and gRPC status details are - content-safe, and caught collaborator exception chains must never be logged. - -## Development checks - -The project Makefile provides development shortcuts: +Entity behavior is supplied by OpenShell policy config, not server startup +flags. Deployment startup owns only installed engine implementations and +operator resources such as model profiles, endpoints, clients, and credentials. -```bash -make help -make test PYTEST_ARGS="tests/test_processor.py -k redact" -make fix -make check -``` +## Safety and limits + +- Input and replacement bodies are limited to 4 MiB. +- One monotonic `Timeout` is shared across every stage and result validation. +- Regex searches receive the remaining timeout and fail atomically. +- Intermediate text and detection cardinality are bounded. +- Detect and block never return a body mutation; replace returns final text. +- Findings expose entity, bounded confidence, count, and stage provenance, but + never matched text, surrounding text, offsets, patterns, or raw tool metadata. +- Engine instances and injected resources must be safe for concurrent requests. +- Cross-request entity memory is intentionally out of scope. -`make check` delegates to the authoritative complete local check: +## Updating the OpenShell protocol + +Privacy Guard uses +[`openshell-middleware-kit`](../openshell-middleware-kit/README.md) to keep its +copied protocol and generated Python bindings aligned with an OpenShell release. +Install the repository's local `omkit`, then update: ```bash -scripts/check.sh +uv tool install --force ../openshell-middleware-kit +omkit update --openshell-version v0.0.90 ``` -The top-level -`.github/workflows/privacy-guard.yml` workflow runs the same check in isolated -uv environments on the minimum and latest supported Python versions. +The updater replaces only the copied protocol, generated bindings, lockfile, and +`.openshell-middleware-manifest.json` from a validated temporary copy. Review +those generated changes and run `make check`. -To exercise the package with the minimum supported interpreter, run -`scripts/check.sh --python 3.11`. This committed script is the authoritative -project check used locally and in CI. It runs the full tests, formatting, lint, -curated `ty` rules, import smoke, and package build. -The AST policy test rejects cast operations and explicit dynamic typing in -handwritten `src`, `tests`, and `examples`; only generated protobuf/gRPC bindings -are excluded. +## Development validation -The optional diagnostic benchmark uses three samples per median and covers a -focused set of body sizes, finding loads, scanner shapes, and reconstruction -modes: +The project Makefile exposes the normal workflow: ```bash -uv run --frozen python scripts/benchmark_privacy_guard.py +make help +make test PYTEST_ARGS="tests/test_request_processor.py" +make fix +make check +make check-py311 ``` -Use it manually when changing performance-sensitive processing code; it is not -part of `scripts/check.sh` and does not enforce regression thresholds. For -the broader scenario set and seven samples per median, pass `--suite full`. -Pass `--profile profile.out` to either suite to record a `cProfile` artifact. -The harness reports median wall time and median peak traced allocation for the -synchronous normalize, synthetic scan, output validation, policy, and -reconstruction path. It does not measure a real PII scanner, gRPC adaptation, -executor queuing, concurrent throughput, or process RSS. Methodology and one -platform-specific development snapshot are in [BENCHMARKS.md](BENCHMARKS.md). +`make check` delegates to `scripts/check.sh`, the authoritative local and CI +gate. It runs tests, formatting, lint, `ty`, import smoke, and package builds. diff --git a/projects/privacy-guard/examples/email-scanner/.gitignore b/projects/privacy-guard/examples/email-scanner/.gitignore deleted file mode 100644 index b223234..0000000 --- a/projects/privacy-guard/examples/email-scanner/.gitignore +++ /dev/null @@ -1 +0,0 @@ -gateway.local.toml diff --git a/projects/privacy-guard/examples/email-scanner/README.md b/projects/privacy-guard/examples/email-scanner/README.md deleted file mode 100644 index b4a9ce7..0000000 --- a/projects/privacy-guard/examples/email-scanner/README.md +++ /dev/null @@ -1,206 +0,0 @@ -# Email scanner example - -This self-contained example supplies a deterministic custom email scanner, its -Privacy Guard server entry point, and the OpenShell gateway and sandbox policy -needed to try redaction with Claude Code. It temporarily runs the installed -OpenShell gateway with the config from this directory. It does not create or -modify `~/.config/openshell/gateway.toml` or create a project-local state -directory. It registers the standard local gateway endpoint with the OpenShell -CLI. - -The `EmailScanner` class in `middleware_server.py` demonstrates how to implement -the scanner extension contract directly. It detects email-shaped text in Claude -Code request bodies. The policy replaces matches with `[email]` before the -remote model receives the request. The implementation is intentionally small -and is not intended as comprehensive production PII detection. - -## Before you start - -- macOS with Docker Desktop running -- Python 3 and `uv` -- OpenShell installed with its recommended installer: - - ```bash - curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh - ``` - -The foreground gateway commands below target the Homebrew installation on -macOS. The scanner, gateway config, and sandbox policy also work on Linux, but -use the service and TLS paths from that OpenShell installation. - -Run all commands in this walkthrough from the example directory: - -```bash -cd projects/privacy-guard/examples/email-scanner -``` - -`uv` automatically discovers the Privacy Guard project in the parent -directories, so the commands do not need a `--project` option. - -Use terminal 1 for Privacy Guard, terminal 2 for the foreground gateway, -terminal 3 for Claude Code, and another terminal for finite log checks. - -## 1. Generate the local gateway config - -Enter the IPv4 address of the host's physical Ethernet or Wi-Fi interface after -`YOUR_HOST_IP=`, then run both lines: - -```bash -YOUR_HOST_IP= -sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml -grep grpc_endpoint gateway.local.toml -``` - -Do not use `127.0.0.1`, a VPN address, or `host.openshell.internal`. The address -must be reachable from both the host gateway and sandbox supervisor. -The final command lets you verify the generated endpoint before starting -anything. - -## 2. Start Privacy Guard - -This development server uses unauthenticated plaintext gRPC and receives request -bodies that may contain sensitive content. Restrict access to port 50051 with a -host firewall because sandboxes must be able to reach it. - -In terminal 1: - -```bash -uv run python middleware_server.py \ - --listen 0.0.0.0:50051 -``` - -Leave it running. - -## 3. Run the installed gateway with the example config - -In terminal 2: - -```bash -brew services stop openshell - -OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" \ -openshell-gateway --config "$PWD/gateway.local.toml" -``` - -The first command stops the background service so the foreground gateway can use -the standard port. The second command reuses the credentials and state created -by the recommended macOS installation, but loads this example's generated -`gateway.local.toml`. The gateway should stay in the foreground. TLS startup -followed by `Server listening` means it is ready. - -Middleware registration is static. After regenerating `gateway.local.toml`, -stop this foreground process with `Ctrl-C` and run the second command again. - -## 4. Create the sandbox and run Claude - -In terminal 3: - -```bash -openshell gateway add \ - https://127.0.0.1:17670 \ - --local \ - --name openshell - -openshell status - -openshell sandbox create \ - --name privacy-guard-email \ - --from base \ - --no-auto-providers \ - --policy "$PWD/policy.yaml" \ - -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude -``` - -`gateway add` saves the endpoint as the CLI's active gateway and refreshes its -package-managed mTLS credentials. The registration continues to target the -normal local gateway after this walkthrough. - -Choose Claude Code's subscription-account login and complete authentication. - -OpenShell can start a sandbox in a degraded state when its supervisor cannot -reach Privacy Guard. Before sending a prompt, use another terminal to inspect -the startup logs: - -```bash -openshell logs privacy-guard-email -n 100 -``` - -Do not continue if they contain `Middleware connect failed` or -`CONFIG:DEGRADED`; correct the host address and recreate the sandbox first. - -Then enter: - -```text -Tell me something that rhymes with my email wendy@gmail.com -``` - -Privacy Guard should replace the address with `[email]` before Anthropic -receives it. The reply must not be used to verify redaction because the model -may refer to the placeholder or invent an example address. - -In another terminal, inspect the finite recent log history: - -```bash -openshell logs privacy-guard-email -n 100 -``` - -Look for the `api.anthropic.com/v1/messages` request with `transformed:true` and -an email finding. - -## Troubleshooting - -- If gateway startup reports `middleware registration failed`, confirm Privacy - Guard is still running and that `gateway.local.toml` contains the host's - physical Ethernet or Wi-Fi IPv4 address. -- If a prompt returns `403 "middleware_failed"`, inspect the sandbox logs. A - `Middleware connect failed` or `binding_not_described` entry means the sandbox - supervisor could not reach the configured address. Regenerate - `gateway.local.toml`, restart the foreground gateway, and recreate the sandbox. -- If logs report `middleware_timeout`, make sure the config does not contain a - VPN address and that the host firewall permits sandbox traffic to port 50051. - -## Change the behavior - -Edit `policy.yaml` and change `on_finding.action` to `observe`, `block`, or -`redact`, then apply it without recreating the sandbox: - -```bash -openshell policy set privacy-guard-email \ - --policy "$PWD/policy.yaml" \ - --wait -``` - -- `redact` sends `[email]` to the remote model provider. -- `observe` records the finding but sends the original email. -- `block` denies the request. - -To reconnect later: - -```bash -openshell sandbox connect privacy-guard-email -``` - -Then run `env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude` inside the -sandbox. - -## Cleanup - -Exit Claude and the sandbox, then delete it: - -```bash -openshell sandbox delete privacy-guard-email -``` - -Stop the foreground gateway with `Ctrl-C` in terminal 2, then restore the normal -background gateway: - -```bash -brew services start openshell -``` - -Stop Privacy Guard with `Ctrl-C` in terminal 1. The example-specific gateway -configuration was never installed as the default configuration. - -This example uses Claude Code because subscription prompts are sent in inspectable -HTTP request bodies. ChatGPT-subscription Codex currently sends prompts in -WebSocket frames, which this HTTP middleware cannot inspect. diff --git a/projects/privacy-guard/examples/email-scanner/gateway.toml b/projects/privacy-guard/examples/email-scanner/gateway.toml deleted file mode 100644 index 510586c..0000000 --- a/projects/privacy-guard/examples/email-scanner/gateway.toml +++ /dev/null @@ -1,12 +0,0 @@ -# OpenShell gateway configuration for the email-scanner example. -# Pass this example-specific config directly to openshell-gateway. -# Do not copy it into ~/.config/openshell. - -[openshell] -version = 1 - -[[openshell.supervisor.middleware]] -name = "privacy-guard-email-scanner" -grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051" -max_body_bytes = 4194304 -timeout = "5s" diff --git a/projects/privacy-guard/examples/email-scanner/middleware_server.py b/projects/privacy-guard/examples/email-scanner/middleware_server.py deleted file mode 100644 index b964733..0000000 --- a/projects/privacy-guard/examples/email-scanner/middleware_server.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -"""Run Privacy Guard with the example deterministic email scanner.""" - -from __future__ import annotations - -import argparse -import re - -from privacy_guard.scanners import ( - Confidence, - Finding, - ScanBudget, - Scanner, - ScannerConfig, -) -from privacy_guard.service import MiddlewareServer - - -class EmailScanner(Scanner[ScannerConfig]): - """Detect common email-shaped text as a minimal custom scanner example.""" - - _EMAIL = re.compile( - r"(? tuple[Finding, ...]: - return tuple( - Finding( - entity="email", - scanner_name=self.scanner_name, - start_offset=match.start(), - end_offset=match.end(), - confidence=Confidence.HIGH, - ) - for match in self._EMAIL.finditer(text_block) - ) - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--listen", default="127.0.0.1:50051") - arguments = parser.parse_args() - scanner = EmailScanner( - ScannerConfig(name="example_regex", entity_types=frozenset({"email"})) - ) - MiddlewareServer(scanner=scanner).serve(arguments.listen) - - -if __name__ == "__main__": - main() diff --git a/projects/privacy-guard/examples/email-scanner/policy.yaml b/projects/privacy-guard/examples/email-scanner/policy.yaml deleted file mode 100644 index b305eb5..0000000 --- a/projects/privacy-guard/examples/email-scanner/policy.yaml +++ /dev/null @@ -1,49 +0,0 @@ -version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - claude_code: - name: Claude Code subscription access - endpoints: - - host: api.anthropic.com - port: 443 - protocol: rest - enforcement: enforce - access: full - - host: platform.claude.com - port: 443 - protocol: rest - enforcement: enforce - access: full - - host: claude.ai - port: 443 - - { host: statsig.anthropic.com, port: 443 } - - { host: sentry.io, port: 443 } - binaries: - - { path: /usr/local/bin/claude } - - { path: /usr/bin/node } - -network_middlewares: - privacy_guard_redaction: - name: Redact deterministic example emails - middleware: privacy-guard-email-scanner - order: 0 - config: - body_format: json - on_finding: - action: redact - entity_types: [email] - minimum_confidence: high - on_error: fail_closed - endpoints: - include: - - api.anthropic.com diff --git a/projects/privacy-guard/examples/regex-engine/README.md b/projects/privacy-guard/examples/regex-engine/README.md new file mode 100644 index 0000000..d40851d --- /dev/null +++ b/projects/privacy-guard/examples/regex-engine/README.md @@ -0,0 +1,15 @@ +# RegexEngine policy example + +This example keeps entity behavior in the OpenShell policy configuration. +`privacy-guard-config.yaml` contains the complete structured +`RegexPatternCatalog` accepted by Privacy Guard. + +Privacy Guard does not ship authoritative regex presets. Copy and adapt this +example and the larger `patterns.yaml` reference catalog for the data you +actually need to identify, and test it against representative worst-case inputs +before deployment. + +The current OpenShell policy flow does not expand catalog file paths. A catalog +must therefore be inline before it is passed to Privacy Guard. Transparent file +expansion and larger prepared catalogs require coordinated upstream OpenShell +support; this project does not fork the copied `.proto`. diff --git a/projects/privacy-guard/examples/regex-engine/patterns.yaml b/projects/privacy-guard/examples/regex-engine/patterns.yaml new file mode 100644 index 0000000..b333bdf --- /dev/null +++ b/projects/privacy-guard/examples/regex-engine/patterns.yaml @@ -0,0 +1,11 @@ +entities: + - name: email + patterns: + - name: conventional-email + pattern: '(? gateway.local.toml -grep grpc_endpoint gateway.local.toml -``` - -Do not use `127.0.0.1`, a VPN address, or `host.openshell.internal`. The address -must be reachable from both the host gateway and sandbox supervisor. -The final command lets you verify the generated endpoint before starting -anything. - -## 3. Start Privacy Guard - -This development server uses unauthenticated plaintext gRPC and receives request -bodies that may contain sensitive content. Restrict access to port 50051 with a -host firewall because sandboxes must be able to reach it. - -In terminal 1: - -```bash -uv run privacy-guard regex \ - --config regex-scanner.yaml \ - --listen 0.0.0.0:50051 -``` - -Leave it running. Privacy Guard loads and compiles the scanner configuration -before it binds the port. - -## 4. Run the installed gateway with the example config - -In terminal 2: - -```bash -brew services stop openshell - -OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" \ -openshell-gateway --config "$PWD/gateway.local.toml" -``` - -The first command stops the background service so the foreground gateway can use -the standard port. The second command reuses the credentials and state created -by the recommended macOS installation, but loads this example's generated -`gateway.local.toml`. Keep the gateway in the foreground. TLS startup followed -by `Server listening` means it is ready. - -Middleware registration is static. After regenerating `gateway.local.toml`, -stop this foreground process with `Ctrl-C` and run the second command again. - -## 5. Create the sandbox and run an agent - -In terminal 3: - -```bash -openshell gateway add \ - https://127.0.0.1:17670 \ - --local \ - --name openshell - -openshell status -``` - -`gateway add` saves the endpoint as the CLI's active gateway and refreshes its -package-managed mTLS credentials. The registration continues to target the -normal local gateway after this walkthrough. - -Choose either of the following paths. Both use the same sandbox name, so run -only one create command. - -### Path A: Claude Code - -```bash -openshell sandbox create \ - --name privacy-guard-regex \ - --from base \ - --no-auto-providers \ - --policy "$PWD/policy.yaml" \ - -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude -``` - -Disabling nonessential Claude Code traffic keeps telemetry and error-reporting -batches out of this focused redaction exercise. Privacy Guard still processes -the model requests sent to `api.anthropic.com`. - -Choose Claude Code's subscription-account login and complete authentication. - -### Path B: Pi - -This path supports a user-supplied OpenAI-compatible HTTPS endpoint on port 443. -Enter your endpoint URL, model ID, and API key after the three `=` signs: - -```bash -PI_MODEL_ENDPOINT= -PI_MODEL_ID= -export PI_MODEL_API_KEY= -``` - -Keep a suffix such as `/v1` in `PI_MODEL_ENDPOINT` when the provider requires -it. The setup below derives the policy hostname from that URL. - -Generate ignored local model and policy files: - -```bash -: "${PI_MODEL_ENDPOINT:?Set PI_MODEL_ENDPOINT above}" -: "${PI_MODEL_ID:?Set PI_MODEL_ID above}" -: "${PI_MODEL_API_KEY:?Set PI_MODEL_API_KEY above}" - -PI_MODEL_HOST=${PI_MODEL_ENDPOINT#*://} -PI_MODEL_HOST=${PI_MODEL_HOST%%/*} - -sed \ - -e "s|REPLACE_WITH_MODEL_ENDPOINT|$PI_MODEL_ENDPOINT|" \ - -e "s|REPLACE_WITH_MODEL_ID|$PI_MODEL_ID|g" \ - pi-models.template.json > pi-models.local.json - -sed "s/REPLACE_WITH_MODEL_HOST/$PI_MODEL_HOST/g" \ - policy.pi.template.yaml > policy.local.yaml -``` - -Create an OpenShell credential provider once. Passing the bare environment -variable name reads the value from the host without putting the key on the -command line: - -```bash -openshell provider create \ - --name privacy-guard-model \ - --type generic \ - --credential PI_MODEL_API_KEY -``` - -If `privacy-guard-model` already exists, skip `provider create` and confirm it -with `openshell provider get privacy-guard-model`. - -Create the sandbox with the generated model configuration and policy: - -```bash -openshell sandbox create \ - --name privacy-guard-regex \ - --from pi \ - --provider privacy-guard-model \ - --no-auto-providers \ - --policy "$PWD/policy.local.yaml" \ - --no-git-ignore \ - --upload pi-models.local.json:/sandbox/.pi/agent/models.json \ - -- pi \ - --provider custom \ - --model "$PI_MODEL_ID" -``` - -The `pi` community sandbox contains the Pi coding agent in addition to the base -sandbox tools. These options open its interactive TUI with the custom model -already selected. OpenShell injects an opaque credential placeholder into the -sandbox and substitutes the real key only in the TLS-terminated request to -the configured endpoint. `privacy-guard-model` is the OpenShell credential -provider; `custom` is the separate provider name in Pi's model configuration. - -### Confirm middleware startup - -OpenShell can start a sandbox in a degraded state when its supervisor cannot -reach Privacy Guard. Before sending a prompt, use another terminal to inspect -the startup logs: - -```bash -openshell logs privacy-guard-regex -n 100 -``` - -Do not continue if they contain `Middleware connect failed` or -`CONFIG:DEGRADED`; correct the host address and recreate the sandbox first. - -### Exercise the scanner - -In the agent you chose, enter: - -```text -Tell me something that rhymes with my email wendy@gmail.com and remember that my customer ID is CUST-12345678. -``` - -Privacy Guard should replace the values with `[email]` and `[customer-id]` -before the remote model provider receives the request. The reply must not be -used to verify redaction: models may refer to the placeholders or invent -example values. - -In another terminal, inspect the finite recent log history: - -```bash -openshell logs privacy-guard-regex -n 100 -``` - -For Claude Code, look for the `api.anthropic.com/v1/messages` request. For Pi, -look for a request to the hostname assigned to `PI_MODEL_HOST`. The request -should have `transformed:true` and findings for both configured entities. - -## Troubleshooting - -- If gateway startup reports `middleware registration failed`, confirm Privacy - Guard is still running and that `gateway.local.toml` contains the host's - physical Ethernet or Wi-Fi IPv4 address. -- If a prompt returns `403 "middleware_failed"`, inspect the sandbox logs. A - `Middleware connect failed` or `binding_not_described` entry means the sandbox - supervisor could not reach the configured address. Regenerate - `gateway.local.toml`, restart the foreground gateway, and recreate the sandbox. -- If logs report `middleware_timeout`, make sure the config does not contain a - VPN address and that the host firewall permits sandbox traffic to port 50051. - -To investigate latency without logging request content, restart Privacy Guard -in terminal 1 with `--debug`: - -```bash -uv run privacy-guard --debug regex \ - --config regex-scanner.yaml \ - --listen 0.0.0.0:50051 -``` - -To compare the body received by Privacy Guard with the body it returns, use -`--debug-log-content` instead. This mode logs secrets by design: - -```bash -uv run privacy-guard --debug-log-content regex \ - --config regex-scanner.yaml \ - --listen 0.0.0.0:50051 -``` - -Match `stage=received` and `stage=forwarded` entries by `request_id`, and disable -content logging after capturing the failing request. - -## Change the behavior - -To change enforcement, edit `policy.yaml` for Claude Code or -`policy.pi.template.yaml` for Pi and set `on_finding.action` to `observe`, -`block`, or `redact`. For Pi, regenerate `policy.local.yaml` with the command in -Path B. Then apply the selected policy without recreating the sandbox: - -```bash -POLICY_FILE=policy.yaml # Use policy.local.yaml for Pi. -openshell policy set privacy-guard-regex \ - --policy "$PWD/$POLICY_FILE" \ - --wait -``` - -- `redact` sends `[email]` and `[customer-id]` to the remote model provider. -- `observe` records findings but sends the original values. -- `block` denies a request containing either configured entity. - -To change detection, edit `regex-scanner.yaml`, stop Privacy Guard with `Ctrl-C`, -and run the terminal 1 command again. Scanner configuration is loaded only at -startup. - -To reconnect later: - -```bash -openshell sandbox connect privacy-guard-regex -``` - -Then run the same launch command you chose earlier: - -```bash -env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude -# or, in a Pi sandbox: -PI_MODEL_ID= -pi \ - --provider custom \ - --model "$PI_MODEL_ID" -``` - -## Cleanup - -Exit the agent and the sandbox, then delete it: - -```bash -openshell sandbox delete privacy-guard-regex -``` - -Stop the foreground gateway with `Ctrl-C` in terminal 2, then restore the normal -background gateway: - -```bash -brew services start openshell -``` - -Stop Privacy Guard with `Ctrl-C` in terminal 1. The example-specific gateway -configuration was never installed as the default configuration. - -This example uses Claude Code with Anthropic or Pi with an OpenAI-compatible -endpoint because their prompts are sent in inspectable HTTP request bodies. -ChatGPT-subscription Codex currently sends prompts in WebSocket frames, which -this HTTP middleware cannot inspect. diff --git a/projects/privacy-guard/examples/regex-scanner/gateway.toml b/projects/privacy-guard/examples/regex-scanner/gateway.toml deleted file mode 100644 index 8a6aaf3..0000000 --- a/projects/privacy-guard/examples/regex-scanner/gateway.toml +++ /dev/null @@ -1,12 +0,0 @@ -# OpenShell gateway configuration for the built-in regex-scanner example. -# Pass this example-specific config directly to openshell-gateway. -# Do not copy it into ~/.config/openshell. - -[openshell] -version = 1 - -[[openshell.supervisor.middleware]] -name = "privacy-guard-regex-scanner" -grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051" -max_body_bytes = 4194304 -timeout = "5s" diff --git a/projects/privacy-guard/examples/regex-scanner/pi-models.template.json b/projects/privacy-guard/examples/regex-scanner/pi-models.template.json deleted file mode 100644 index dc71f19..0000000 --- a/projects/privacy-guard/examples/regex-scanner/pi-models.template.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "providers": { - "custom": { - "baseUrl": "REPLACE_WITH_MODEL_ENDPOINT", - "api": "openai-completions", - "apiKey": "$PI_MODEL_API_KEY", - "authHeader": true, - "models": [ - { - "id": "REPLACE_WITH_MODEL_ID", - "name": "REPLACE_WITH_MODEL_ID" - } - ] - } - } -} diff --git a/projects/privacy-guard/examples/regex-scanner/policy.pi.template.yaml b/projects/privacy-guard/examples/regex-scanner/policy.pi.template.yaml deleted file mode 100644 index 96a1d7b..0000000 --- a/projects/privacy-guard/examples/regex-scanner/policy.pi.template.yaml +++ /dev/null @@ -1,42 +0,0 @@ -version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - pi: - name: Pi access to the configured model endpoint - endpoints: - - host: REPLACE_WITH_MODEL_HOST - port: 443 - protocol: rest - enforcement: enforce - access: full - - { host: pi.dev, port: 443 } - binaries: - - { path: /usr/bin/pi } - - { path: /usr/lib/node_modules/@earendil-works/pi-coding-agent/dist/cli.js } - - { path: /usr/bin/node } - -network_middlewares: - privacy_guard_redaction: - name: Redact configured email addresses and customer IDs - middleware: privacy-guard-regex-scanner - order: 0 - config: - body_format: json - on_finding: - action: redact - entity_types: [email, customer-id] - minimum_confidence: high - on_error: fail_closed - endpoints: - include: - - REPLACE_WITH_MODEL_HOST diff --git a/projects/privacy-guard/examples/regex-scanner/policy.yaml b/projects/privacy-guard/examples/regex-scanner/policy.yaml deleted file mode 100644 index 4dc7675..0000000 --- a/projects/privacy-guard/examples/regex-scanner/policy.yaml +++ /dev/null @@ -1,49 +0,0 @@ -version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - claude_code: - name: Claude Code subscription access - endpoints: - - host: api.anthropic.com - port: 443 - protocol: rest - enforcement: enforce - access: full - - host: claude.ai - port: 443 - - host: platform.claude.com - port: 443 - protocol: rest - enforcement: enforce - access: full - - { host: statsig.anthropic.com, port: 443 } - - { host: sentry.io, port: 443 } - binaries: - - { path: /usr/local/bin/claude } - - { path: /usr/bin/node } - -network_middlewares: - privacy_guard_redaction: - name: Redact configured email addresses and customer IDs - middleware: privacy-guard-regex-scanner - order: 0 - config: - body_format: json - on_finding: - action: redact - entity_types: [email, customer-id] - minimum_confidence: high - on_error: fail_closed - endpoints: - include: - - api.anthropic.com diff --git a/projects/privacy-guard/examples/regex-scanner/regex-scanner.yaml b/projects/privacy-guard/examples/regex-scanner/regex-scanner.yaml deleted file mode 100644 index 3508588..0000000 --- a/projects/privacy-guard/examples/regex-scanner/regex-scanner.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- name: email - patterns: - - name: common-email - regex: '(? Date: Fri, 24 Jul 2026 19:33:30 +0000 Subject: [PATCH 29/82] Rewrite Privacy Guard architecture documentation --- .../architecture/configuration.md | 124 ++++++++ .../architecture/entity-processing-engines.md | 267 ++++++++++++++++++ .../architecture/format-handlers.md | 193 ------------- .../privacy-guard/architecture/index.md | 160 +++++++---- .../architecture/request-lifecycle.md | 225 ++++++++------- .../architecture/safety-and-limits.md | 258 +++++++++-------- .../privacy-guard/architecture/scanners.md | 175 ------------ .../architecture/service-boundary.md | 216 +++++++++----- zensical.toml | 4 +- 9 files changed, 907 insertions(+), 715 deletions(-) create mode 100644 docs/documentation/privacy-guard/architecture/configuration.md create mode 100644 docs/documentation/privacy-guard/architecture/entity-processing-engines.md delete mode 100644 docs/documentation/privacy-guard/architecture/format-handlers.md delete mode 100644 docs/documentation/privacy-guard/architecture/scanners.md diff --git a/docs/documentation/privacy-guard/architecture/configuration.md b/docs/documentation/privacy-guard/architecture/configuration.md new file mode 100644 index 0000000..df4ae0a --- /dev/null +++ b/docs/documentation/privacy-guard/architecture/configuration.md @@ -0,0 +1,124 @@ +--- +title: Configuration and text boundary +description: Runtime configuration ownership and the single-text processing contract. +agent_markdown: true +--- + +# Configuration and text boundary + +Privacy Guard processes the complete request body as one text string. Its +runtime configuration is self-contained structured data supplied by OpenShell +for each evaluation. + +## Bytes and text + +The service owns the transport boundary: + +- it validates the advertised 4 MiB request-body limit +- it allows an empty body without invoking an engine +- it decodes each non-empty body as strict UTF-8 +- it passes exactly one `str` to `RequestProcessor` +- detect and block return no body mutation +- replace UTF-8 encodes the final processed text + +Because detect and block return no mutation, OpenShell retains the exact +original bytes. Replace returns the final text even when it happens to equal +the input. + +Headers and media types do not change this behavior. Privacy Guard does not +parse JSON, select nested values, create document regions, or reconstruct +structured payloads. + +## Policy configuration + +The OpenShell policy owns: + +- ordered entity-processing stages +- each stage's exact engine configuration +- entity definitions and detection settings +- engine-specific replacement recipes +- the final detect, block, or replace action + +For example: + +```yaml +entity_processing: + stages: + - name: identifiers + config: + engine: regex + pattern_catalog: + entities: + - name: customer-id + patterns: + - name: prefixed-eight-digit-id + pattern: '\bCUST-[0-9]{8}\b' + confidence: high + replacement: + strategy: template + template: "[{entity}]" +on_detection: + action: replace +``` + +`config.engine` is the Pydantic discriminator. `EngineRegistry.finalize()` +builds the complete policy model from the exact config type registered for each +engine. Engine-specific fields therefore validate and serialize without a +generic mapping layer. + +Deployment startup owns operational resources rather than privacy behavior: +installed engine implementations, approved model profiles, clients, endpoints, +credentials, and data-egress constraints. + +## Regex catalogs + +`RegexEngineConfig.pattern_catalog` is always a structured +`RegexPatternCatalog`. Privacy Guard maintains its schema and safety limits but +does not ship an authoritative pattern set. + +The repository may publish reference catalog YAML for users to copy and adapt. +Those files are examples, not presets, runtime defaults, or a second +configuration source. + +The current OpenShell policy flow does not expand: + +```yaml +pattern_catalog: ./patterns.yaml +``` + +Privacy Guard cannot resolve that path because the middleware process does not +own the policy bundle. Accepting middleware-local paths would also make policy +behavior depend on deployment filesystem state. + +Transparent catalog-file support belongs in OpenShell's policy installation +flow. OpenShell would resolve the path relative to the policy bundle, validate +the referenced value, retain the expanded configuration, and send only the +self-contained mapping to Privacy Guard. Until that upstream feature exists, +catalogs must be inline in the configuration sent to the middleware. + +## Current transport constraint + +The copied OpenShell protocol carries policy configuration in a +per-evaluation `google.protobuf.Struct` limited to 64 KiB. This bounds the +catalog size that can reach Privacy Guard today. + +The service validates the complete configuration, computes its canonical +fingerprint, and uses a bounded internal `RequestProcessor` cache. Caching +avoids repeated engine initialization but does not increase the transport +limit. + +Supporting larger catalogs requires an upstream OpenShell contract for +preparing expanded configuration and referring to it during evaluation. +Privacy Guard must not create a private protocol fork. + +## Configuration identity + +Canonical serialization includes every concrete engine field and nested +replacement variant. Mapping keys are sorted, compact JSON encoding is used, +and the SHA-256 fingerprint is computed over the resulting UTF-8 bytes. + +Equivalent structured configurations therefore share a processor cache entry. +Cache state is only an optimization; eviction or restart reconstructs the +processor from configuration supplied by a later evaluation. + +[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md new file mode 100644 index 0000000..58d0528 --- /dev/null +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -0,0 +1,267 @@ +--- +title: Entity-processing engines +description: Engine configuration, lifecycle, output contract, and extension rules. +agent_markdown: true +--- + +# Entity-processing engines + +An `EntityProcessingEngine` detects sensitive entities in one text string and +may replace them when requested. It has no access to the user-facing policy +action, request metadata, gRPC, or protobuf messages. + +Each configured stage owns one engine instance. The same instance may serve +concurrent requests, while separate stages may use the same implementation +with different configurations. + +## Engine interface + +An engine: + +1. declares concrete configuration and resource types through generics +2. declares its maximum `supported_strategy` +3. receives validated configuration and operator-owned resources through the + base constructor +4. optionally derives immutable reusable state in `_initialize()` +5. implements `_run(text, strategy, timeout)` +6. returns one `TextProcessingResult` + +The public `run()` method validates input, strategy support, the timeout, and +the complete output contract. Custom engines implement `_run()` and do not +override `run()` or define `__init__`. `_initialize()` is optional, and +`@override` is not required. + +## Configuration + +Every concrete config subclasses `EngineConfig` and declares exactly one +literal `engine` discriminator: + +```python +class AcmeEngineConfig(EngineConfig[AcmeReplacement]): + engine: Literal["acme-pii"] = "acme-pii" +``` + +The object under `EntityProcessingStage.config` is this exact concrete model. +It is validated and serialized as a member of the registry-built Pydantic +discriminated union, then passed unchanged to the engine constructor. + +`EngineConfig` provides the optional location for an engine-specific +replacement recipe. Engines with multiple replacement algorithms define their +own nested discriminated union, using fields such as +`replacement.strategy`. There is no closed Privacy Guard enum of all possible +replacement algorithms. + +Configuration contains privacy behavior. Runtime resources contain operational +details such as model clients, credentials, endpoints, and approved profiles. +Resources are registered by the operator and never serialized into policy. +Engines without resources declare `None` as their resource type. + +## Invocation strategy + +```python +class EntityProcessingStrategy(StrEnum): + DETECT = "detect" + REPLACE = "replace" +``` + +Every engine supports detection. A detection-only engine declares: + +```python +supported_strategy = EntityProcessingStrategy.DETECT +``` + +A replacement-capable engine declares: + +```python +supported_strategy = EntityProcessingStrategy.REPLACE +``` + +`REPLACE` includes detection support, so `supported_strategy` is a singular +maximum rather than a capability collection. Blocking does not appear here: +the processor runs engines with `DETECT` and applies the block disposition +afterward. + +## Result contract + +`TextProcessingResult` contains: + +| Field | Meaning | +| --- | --- | +| `text` | Authoritative text returned by this stage | +| `detections` | Tuple of all `EntityDetection` occurrences | + +Each `EntityDetection` contains: + +| Field | Meaning | +| --- | --- | +| `entity` | Bounded entity label | +| `start` | Inclusive Unicode code-point offset in the stage input | +| `end` | Exclusive Unicode code-point offset in the stage input | +| `confidence` | Optional `low`, `medium`, `high`, or strict value from 0 through 1 | +| `metadata` | Optional bounded engine-specific attribution retained inside the processing boundary | + +A detection span is non-empty and must fall within the stage input. The public +wrapper also enforces stage detection and output-size limits. + +For `DETECT`, output text must exactly equal input text. For `REPLACE`, a +successful return is the engine's authoritative completed result. Text may not +change without at least one detection. Engines must raise on native partial +failure instead of returning partial output. + +Confidence remains in the representation supplied by the engine. Privacy Guard +does not invent numeric values for categorical confidence or compare confidence +across engines. + +## Custom engine example + +```python +from typing import Literal + +from pydantic import Field + +from privacy_guard.engines import ( + EngineConfig, + EngineConfigurationError, + EntityDetection, + EntityProcessingEngine, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.timeout import Timeout +from privacy_guard.base import StrictDomainModel + + +class KeywordReplacement(StrictDomainModel): + strategy: Literal["token"] = "token" + token: str = "[keyword]" + + +class KeywordEngineConfig(EngineConfig[KeywordReplacement]): + engine: Literal["keyword"] = "keyword" + keyword: str = Field(min_length=1) + + +class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig, None]): + supported_strategy = EntityProcessingStrategy.REPLACE + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + timeout.raise_if_expired() + start = text.find(self.config.keyword) + if start < 0: + return TextProcessingResult(text=text, detections=()) + + end = start + len(self.config.keyword) + detection = EntityDetection( + entity="keyword", + start=start, + end=end, + ) + if strategy is EntityProcessingStrategy.DETECT: + output = text + else: + replacement = self.config.replacement + if replacement is None: + raise EngineConfigurationError( + "keyword replacement configuration is required" + ) + output = text[:start] + replacement.token + text[end:] + return TextProcessingResult(text=output, detections=(detection,)) +``` + +Register the implementation and its resources before finalizing the registry: + +```python +registry.register(KeywordEngine) +registry.finalize() +``` + +Finalization freezes registration and constructs the exact policy config type, +JSON Schema, and engine discovery metadata. + +## Timeout and concurrency + +One `Timeout` is created for the processor run and passed through every stage. +An engine must not create a fresh per-stage duration. It should call: + +```python +timeout.raise_if_expired() +remaining = timeout.remaining_seconds() +``` + +When a delegated API accepts a timeout, pass the remaining duration. Operations +that cannot be interrupted must be documented and bounded independently. + +Engine configuration, derived state, and injected resources are shared across +concurrent requests. Keep request text, detections, and counters local to +`_run()`. Resources and engines must be concurrency-safe. + +## RegexEngine + +`RegexEngine` owns both regular-expression detection and deterministic template +replacement. Its `RegexPatternCatalog` contains structured entities and ordered +patterns; Privacy Guard maintains the schema and limits but no authoritative +pattern set. + +Important properties: + +- patterns compile once during configuration validation and initialization +- a non-capturing wrapper plus a private trailing named marker preserves + numeric backreferences and proves the configured match completed +- user-defined named groups and inline flags are rejected +- `ignore_case`, `multiline`, `dot_all`, and `ascii` are explicit fields +- detection retains overlapping matches within and across patterns +- each backend search receives the shared remaining timeout +- pattern names are optional; unnamed patterns receive deterministic + diagnostic identities without changing serialized configuration +- replacement resolves overlaps by categorical confidence, span length, + offsets, entity, and pattern identity +- templates allow literal text and `{entity}` only +- replacement size is projected before output allocation + +The third-party `regex` backend is used because it can interrupt an individual +search when the timeout expires. + +## Tool-specific custom engines + +Tool integrations belong in custom engines until they have a complete, +production-backed implementation. Privacy Guard does not ship placeholder +engine types or runtime protocols that merely resemble a third-party tool. + +The first NeMo Anonymizer integration will be a custom engine. Its configuration +and replacement types should preserve Anonymizer's native concepts, while the +engine itself owns all translation to and from the actual Anonymizer SDK. + +## State + +Cross-request entity memory is intentionally out of scope. The engine API has +no state argument, session identifier, persistent replacement map, or +placeholder storage interface. + +Future memory support requires separate decisions about tenant isolation, +retention, deletion, consistency, and policy ownership. + +## Testing an engine + +Test engines without gRPC: + +- valid and invalid exact configuration and resource types +- detection-only immutability +- replacement behavior and native partial failure +- Unicode offsets and invalid spans +- categorical and numeric confidence +- output and detection limits +- timeout propagation and expiration +- deterministic ordering where relevant +- concurrent calls over immutable initialized state +- content-safe errors that omit input and engine secrets + +Processor tests should cover only ordered multi-stage behavior, shared timeout, +policy disposition, aggregate limits, and stage-qualified findings. + +[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/format-handlers.md b/docs/documentation/privacy-guard/architecture/format-handlers.md deleted file mode 100644 index e229901..0000000 --- a/docs/documentation/privacy-guard/architecture/format-handlers.md +++ /dev/null @@ -1,193 +0,0 @@ ---- -title: Format handlers -description: Request-body normalization, opaque paths, and reconstruction contracts. -agent_markdown: true ---- - -# Format handlers - -A format handler translates raw body bytes into text blocks and reconstructs the -body after redaction. It owns all format-specific parsing and path syntax. - -The processor uses the same interface for every format and never parses a path. - -## Format handler interface - -A format handler implements two protected methods: - -| Method | Responsibility | -| --- | --- | -| `_normalize(raw_body, policy_config)` | Parse bytes and return a `RequestBody` | -| `_reconstruct(request_body, replacements_by_path)` | Apply replacements and return bytes | - -Applications call the public `normalize` and `reconstruct` methods. These -wrappers validate extension output. - -The handler constructor declares a stable `format_name`. The processor registry -key must match it. - -## Normalized body - -`RequestBody` contains: - -| Field | Purpose | -| --- | --- | -| `text_blocks` | `TextBlock` values selected for scanning | -| `parsed_value` | Opaque handler-owned reconstruction state | -| `original_bytes` | Exact input bytes | - -`TextBlock` contains: - -| Field | Purpose | -| --- | --- | -| `path` | Opaque handler-defined address | -| `text` | String passed to scanners | -| `replaceable` | Whether redaction may rewrite this block | - -The processor requires unique paths. It may compare paths and use them as -mapping keys, but it must not parse them. - -The handler must not mutate `parsed_value` during reconstruction. It may copy -the value before applying replacements. - -## Unchanged and rewritten bodies - -When `replacements_by_path` is empty, return `original_bytes` exactly. - -When replacements exist: - -- replace only the addressed text blocks -- reject invalid paths or replacement types -- preserve untouched values semantically -- return bytes - -A rewritten body does not need to preserve whitespace, object formatting, or -other serialization details. - -## JSON handler - -`JsonHandler` is the only built-in format handler. It accepts strict UTF-8 JSON -and rejects: - -- invalid UTF-8 -- duplicate object keys -- non-finite numbers -- invalid Unicode scalar values -- excessive nesting or text shape - -It scans: - -- every JSON string value -- every object key - -It does not scan numbers, booleans, or null. - -### Paths - -String values use JSON Pointer paths: - -```text -{"items": [{"email": "a@example.com"}]} -``` - -```text -/items/0/email -``` - -The root string uses the empty path. Pointer tokens escape `~` as `~0` and `/` -as `~1`. - -Object keys use an internal `#key:` prefix: - -```text -#key:/items/0/email -``` - -Key paths are not reconstruction paths. Their text blocks set -`replaceable=False`. - -### Reconstruction - -`JsonHandler` copies its parsed tree, resolves each JSON Pointer, verifies the -target is a string, applies the replacement, and serializes compact UTF-8 JSON. - -Object keys are never rewritten. If redact policy selects a finding in a key, -the processor denies the request. - -## Minimal plain-text handler - -```python -from collections.abc import Mapping - -from privacy_guard.config import PolicyConfig -from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.request_body import FormatHandler, RequestBody, TextBlock - - -class PlainTextHandler(FormatHandler): - def __init__(self) -> None: - super().__init__(format_name="text") - - def _normalize( - self, - raw_body: bytes, - policy_config: PolicyConfig, - ) -> RequestBody: - try: - text = raw_body.decode("utf-8") - except UnicodeDecodeError: - raise PrivacyGuardError(ErrorCode.BODY_ENCODING_INVALID) from None - return RequestBody( - text_blocks=(TextBlock(path="", text=text),), - parsed_value=None, - original_bytes=raw_body, - ) - - def _reconstruct( - self, - request_body: RequestBody, - replacements_by_path: Mapping[str, str], - ) -> bytes: - if not replacements_by_path: - return request_body.original_bytes - if set(replacements_by_path) != {""}: - raise PrivacyGuardError(ErrorCode.BODY_RECONSTRUCTION_INVALID) - return replacements_by_path[""].encode("utf-8") -``` - -Register it when constructing the processor: - -```python -processor = RequestProcessor( - scanners=[scanner], - format_handlers={"text": PlainTextHandler()}, -) -``` - -## Concurrency - -One handler instance serves concurrent requests. Keep all parsed request state -inside the returned `RequestBody`. - -Do not store request bytes, parsed bodies, text blocks, paths, or replacements -on the handler instance. - -## Testing a handler - -Test: - -- valid and invalid input encoding -- parser edge cases specific to the format -- stable and unique text-block paths -- text selection and `replaceable` flags -- exact no-op byte preservation -- replacement of every supported path shape -- invalid and stale replacement paths -- reconstruction does not mutate parsed state -- shape limits -- concurrent calls - -Processor tests should cover cross-handler rules such as path uniqueness, -original-byte equality, and aggregate text limits. - -[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/index.md b/docs/documentation/privacy-guard/architecture/index.md index c2954ed..3e5a929 100644 --- a/docs/documentation/privacy-guard/architecture/index.md +++ b/docs/documentation/privacy-guard/architecture/index.md @@ -6,12 +6,17 @@ agent_markdown: true # Privacy Guard architecture -Privacy Guard is OpenShell middleware that scans provider-bound HTTP request -bodies and applies policy to detected sensitive data. OpenShell calls it before -adding provider credentials. +Privacy Guard is OpenShell middleware that detects and optionally replaces +sensitive entities in provider-bound HTTP request text before OpenShell adds +provider credentials. -Privacy Guard can allow the request unchanged, allow it with a replacement body, -or deny it. It never sends the provider request itself. +This architecture is a clean break from the earlier implementation. A processor +run receives one UTF-8 text value and invokes an ordered pipeline of +entity-processing engines. Structured-body parsing and compatibility with the +previous extension and policy APIs are intentionally out of scope. + +Privacy Guard can allow the original body, allow a replacement body, or deny +the request. It never sends the provider request itself. ## Where Privacy Guard runs @@ -26,7 +31,7 @@ OpenShell supervisor v Privacy Guard | - | allow, replace body, or deny + | allow original body, allow replacement body, or deny v OpenShell supervisor | @@ -35,25 +40,39 @@ OpenShell supervisor Provider ``` -Privacy Guard implements the `SupervisorMiddleware` gRPC service. Its manifest -registers one binding: HTTP requests in the pre-credentials phase. +Privacy Guard implements the OpenShell-owned `SupervisorMiddleware` gRPC +service and advertises one HTTP-request binding in the pre-credentials phase. +The checked-in protocol and generated bindings are canonical copies from +OpenShell and must not be edited locally. ## Component boundaries Source paths on these pages are relative to `projects/privacy-guard/src/privacy_guard/`. -- `service/` owns gRPC, protobuf conversion, worker scheduling, and finding - aggregation. It does not scan or apply policy. Outside generated `bindings/`, - no other package imports gRPC or generated bindings. -- `processor.py` coordinates format handlers and scanners, applies policy, and - redacts bodies. It does not import gRPC or protobuf. -- `request_body/` parses and reconstructs bodies. It does not scan or apply - policy. -- `scanners/` detects values within one text block. It does not depend on - request formats, policy, gRPC, or protobuf. -- `config.py` defines policy configuration, not scanner configuration. -- `payloads/` defines transport-independent request and result records. +- `service/` owns gRPC, protobuf conversion, UTF-8 decoding and encoding, + bounded worker scheduling, processor caching, and finding serialization. + Outside generated `bindings/`, no other package imports gRPC or generated + bindings. +- `request_processor.py` runs configured stages over one text value, shares one + timeout across them, aggregates detections, and applies the user-facing + policy action. It does not import gRPC or implement an engine's algorithms. +- `engines/` defines the custom-engine contract and the built-in Regex + implementation. Each engine owns its detection and replacement algorithms. +- `engine_registry.py` registers engine implementations and operator-owned + resources, builds the exact Pydantic discriminated union, validates + configurations, and constructs configured engines. +- `config.py` defines ordered stages, the required policy action, canonical + configuration serialization, and fingerprints. +- top-level `base.py` defines the package-wide strict immutable domain-model + base. +- `string_validators.py` defines shared string validators and field types. + +The OpenShell policy is the single source of privacy behavior: stage order, +each stage's exact engine configuration, entity definitions, replacement +recipes, and the final `detect`, `block`, or `replace` action. Deployment +configuration registers implementations and injects operational resources such +as model profiles, endpoints, clients, and credentials. ## Request flow @@ -62,69 +81,86 @@ HttpRequestEvaluation protobuf | v PrivacyGuardMiddleware - validates transport input - parses policy - creates InterceptedRequest + validates phase and body size + validates expanded policy configuration + resolves or builds a cached RequestProcessor + decodes a non-empty body as strict UTF-8 | v -RequestProcessor - selects FormatHandler +RequestProcessor.process(text) + derives DETECT or REPLACE engine strategy + creates one shared Timeout | v -FormatHandler.normalize - returns RequestBody + TextBlock values +stage 1 engine.run(current text) | v -Scanner.scan - returns block-relative Finding values +stage 2 engine.run(stage 1 text) + | + ... | v RequestProcessor - validates and filters findings - observes, blocks, or redacts + aggregates stage-qualified detections + applies detect, block, or replace + | + v +RequestProcessingResult | - +-- policy deny or pre-reconstruction limit --> ProcessingResult + v +PrivacyGuardMiddleware + serializes bounded findings + encodes replacement text only for replace | - +-- continue - | - v - FormatHandler.reconstruct - | - v - output-size check - | - v - ProcessingResult + v +HttpRequestResult protobuf ``` -Both paths return a `ProcessingResult` to `PrivacyGuardMiddleware`, which -aggregates findings and creates the `HttpRequestResult` protobuf. +The processor passes only `EntityProcessingStrategy.DETECT` or +`EntityProcessingStrategy.REPLACE` to engines. Blocking is a request +disposition and never crosses the engine boundary. The processor is synchronous. The service runs it in a dedicated thread pool -so scanner work does not block the gRPC event loop. +so engine work does not block the gRPC event loop. ## Core data types | Type | Meaning | | --- | --- | -| `PolicyConfig` | Body format and action to apply to selected findings | -| `InterceptedRequest` | Protobuf-free request body, request ID, content type, and parsed policy | -| `RequestBody` | Handler-owned parsed state, original bytes, and text blocks | -| `TextBlock` | One string to scan, with an opaque path and replacement flag | -| `Finding` | One scanner result with block-relative offsets | -| `RequestBodyFinding` | A finding paired with its text-block path | -| `ProcessingResult` | Allow or deny decision, optional replacement body, findings, and reason code | - -Domain model fields are strict and frozen. `RequestBody.parsed_value` is -handler-owned and may contain mutable objects; reconstruction must not mutate -it. Fields containing request content are excluded from normal representations. +| `PrivacyGuardConfig` | Ordered entity-processing stages and the required action on detection | +| `EntityProcessingStage` | One configured engine invocation with an optional diagnostic name | +| `EngineConfig` | Generic base for an engine's exact policy configuration and optional replacement recipe | +| `EntityProcessingStrategy` | Per-run engine selection: detect or replace | +| `EntityDetection` | One occurrence with stage-input offsets and optional confidence | +| `TextProcessingResult` | One engine's authoritative output text and detections | +| `Timeout` | One monotonic deadline shared across all stages | +| `EntityDetectionSummary` | Bounded stage/entity/confidence aggregate for audit output | +| `RequestProcessingResult` | Allow or deny decision, detection summaries, and replacement text when requested | + +Pydantic domain models are strict, frozen, reject unknown fields, hide rejected +input from validation errors, and suppress sensitive fields from normal +representations. + +## Deliberate omissions + +- Cross-request entity memory is not part of v0. +- Engines do not receive transport metadata or the user-facing policy action. +- There is no parallel execution-plan model; preparation constructs a + `RequestProcessor` directly. +- There is no generic replacement-strategy enum. Each engine owns the + discriminated replacement recipes appropriate to its underlying algorithm. +- Runtime policy models do not accept catalog paths. Transparent file expansion + requires an upstream OpenShell policy-authoring feature. ## Read next -- [Request lifecycle](request-lifecycle.md) explains normalization, scanning, - policy filtering, and redaction. -- [Scanners](scanners.md) defines the scanner extension contract. -- [Format handlers](format-handlers.md) defines body parsing and reconstruction. -- [Service boundary](service-boundary.md) covers gRPC adaptation and concurrency. -- [Safety and limits](safety-and-limits.md) records failure behavior and resource - bounds. +- [Request lifecycle](request-lifecycle.md) explains configuration resolution, + ordered execution, actions, and output behavior. +- [Entity-processing engines](entity-processing-engines.md) defines the + extension contract and built-in engines. +- [Configuration and text boundary](configuration.md) covers the one-text + contract, configuration ownership, and current catalog limits. +- [Service boundary](service-boundary.md) covers gRPC adaptation, caching, and + concurrency. +- [Safety and limits](safety-and-limits.md) records failure behavior and + resource bounds. diff --git a/docs/documentation/privacy-guard/architecture/request-lifecycle.md b/docs/documentation/privacy-guard/architecture/request-lifecycle.md index d9a30ed..f0a13ea 100644 --- a/docs/documentation/privacy-guard/architecture/request-lifecycle.md +++ b/docs/documentation/privacy-guard/architecture/request-lifecycle.md @@ -1,146 +1,175 @@ --- title: Request lifecycle -description: How Privacy Guard normalizes, scans, filters, and transforms one request. +description: How Privacy Guard prepares policy, runs ordered engines, and applies one request action. agent_markdown: true --- # Request lifecycle -`RequestProcessor` owns the complete protobuf-free request flow. It composes -format handlers and scanners, applies policy, and returns a `ProcessingResult`. +`RequestProcessor` owns the complete protobuf-free flow for one text value. It +runs configured entity-processing stages in order, aggregates detections, and +applies the policy action. + +The gRPC service owns the surrounding bytes/text and configuration transport +boundaries. + +## Policy configuration -## Inputs +The policy supplies: + +```yaml +entity_processing: + stages: + - name: optional-diagnostic-name + config: + engine: regex + pattern_catalog: + entities: + - name: email + patterns: + - pattern: '...' + confidence: high + replacement: + strategy: template + template: "[{entity}]" +on_detection: + action: detect +``` -The processor receives an `InterceptedRequest` containing: +`entity_processing` is an object so concrete pipeline-wide settings can be +added later without changing the stage-list shape. V0 defines only `stages`. +The list must be non-empty. -- the exact request body bytes -- a parsed `PolicyConfig` -- the request ID -- the original content type, retained as context but not used for format - selection +Each `EntityProcessingStage` contains: -Policy selects the format through `body_format`; the processor does not infer it -from request content or metadata. +- optional `name`, used only as bounded diagnostic provenance +- required `config`, which is the exact concrete configuration model owned by + the selected engine -After validating the configured format and entity filter, the processor allows -an empty body without normalization or scanning. +`config.engine` is a Pydantic discriminator. After every implementation is +registered, `EngineRegistry.finalize()` constructs a real discriminated union +of their concrete config models. Engine-specific fields and nested replacement +variants therefore validate, serialize, and appear in JSON Schema without a +generic mapping or translation layer. -## Processing stages +When a stage name is omitted, Privacy Guard derives a deterministic one-based +label such as `regex[1]`. All resulting diagnostic names must be unique. -### 1. Select the format handler +## Configuration resolution and preparation -The processor looks up `PolicyConfig.body_format` in its registered format -handlers. The handler's own `format_name` must match its registry key. +For each evaluation under the current OpenShell protocol, the service: -It also verifies that every entity named by policy appears in at least one -active scanner's `ScannerConfig.entity_types`. An unknown format or entity is -invalid configuration. +1. converts the protobuf `Struct` to a mapping +2. validates it through the finalized registry-backed Pydantic model +3. validates each concrete config against its registered implementation and + injected resources +4. validates the action/replacement compatibility +5. computes a SHA-256 fingerprint of canonical expanded configuration +6. resolves a cached `RequestProcessor`, or constructs the engines and + processor and adds it to the bounded cache -### 2. Normalize the body +`ValidateConfig` performs the validation steps without populating this cache. +Preparation is repeatable; cache state is an optimization and never required +for correctness. -The selected handler parses the original bytes and returns a `RequestBody`. -The processor then verifies: +There is no separate execution-plan abstraction. The validated stage order +already contains the necessary policy structure, and the prepared processor +privately retains the corresponding ordered engine instances. -- `RequestBody.original_bytes` exactly matches the input -- every `TextBlock.path` is unique -- the number of text blocks is within the request limit -- the total text length is within the scan limit +## Text input -The processor does not parse text-block paths. Paths belong to the format -handler. +The service validates the pre-credentials phase and the request body byte +limit before processing. It still validates configuration for an empty body, +then immediately allows that body without invoking an engine. -### 3. Scan and filter findings +A non-empty body must decode as strict UTF-8. The decoded `str` is the only +request input passed to `RequestProcessor`; headers, content type, request ID, +target, and protobuf messages do not cross that boundary. -The processor creates one request-wide `ScanBudget`. It passes every text block -to every configured scanner in registration order. It validates each result, -then selects findings by: +The processor validates both character and encoded-byte bounds before running +the pipeline. -- `entity_types` -- `minimum_confidence` +## Ordered stage execution -Policy never changes scanner behavior. `entity_types: null` selects all entity -types. An empty list selects none. -`minimum_confidence: null` accepts every confidence level. +The processor derives one invocation strategy for the whole pipeline: -`Scanner.scan` validates the returned tuple and finding models. -`RequestProcessor` validates scanner identity, declared entities, offsets, and -selected finding totals. See -[Validation ownership](safety-and-limits.md#validation-ownership). +| Policy action | Engine strategy | +| --- | --- | +| `detect` | `DETECT` | +| `block` | `DETECT` | +| `replace` | `REPLACE` | -### 4. Apply the action +`PolicyAction` is never passed to an engine. -| Action | No selected findings | Selected findings | -| --- | --- | --- | -| `observe` | Allow unchanged | Allow unchanged and report findings | -| `block` | Allow unchanged | Deny and report findings | -| `redact` | Allow unchanged | Replace selected spans, or deny if safe replacement is impossible | - -The processor scans all text blocks for every action. The action controls how -selected findings affect the result. +The processor then: -### 5. Reconstruct the body +1. creates one monotonic `Timeout` +2. calls each stage exactly once in policy order +3. passes the current text, invocation strategy, and shared timeout to + `engine.run()` +4. validates intermediate character, byte, and detection limits +5. passes the returned text to the next stage +6. checks the same timeout after the final result -For redaction, the processor creates replacement text for each affected, -replaceable block. It passes a path-to-text mapping back to the same format -handler that normalized the body. +In detect and block mode, the public engine contract requires returned text to +equal that stage's input. In replace mode, each later stage sees the preceding +stage's processed text. -A selected block action, non-replaceable redaction, or limit result returns -before reconstruction. Every other non-empty request is reconstructed once. -Observe and block-without-findings pass an empty replacement map. +Detection offsets always refer to the input revision seen by the producing +stage. Privacy Guard does not reinterpret earlier offsets after a later stage +changes the text. -With an empty replacement map, a handler returns the original bytes exactly. -After a rewrite, untouched values must remain semantically equivalent, but -byte formatting may change. +If a stage times out, exceeds an execution limit, or fails, its partial text and +detections are discarded. No later stage runs. -The processor returns a replacement body only when reconstructed bytes differ -from the original. +## Detection aggregation -## Redaction behavior +After all stages succeed, the processor aggregates detections by: -Findings may overlap. Every action retains all selected findings for reporting. -Redaction resolves overlaps only when choosing spans to replace; losing overlaps -remain in the result. +```text +source stage + entity + confidence representation +``` -Redaction chooses non-overlapping replacement spans in this order: +It does not deduplicate across stages. Two stages may have inspected different +text revisions, and confidence values from different tools are not assumed to +be calibrated. -1. higher confidence -2. longer span -3. earlier start offset -4. earlier end offset -5. scanner name -6. entity name -7. pattern name +The aggregate `EntityDetectionSummary` intentionally omits matched text, +surrounding text, offsets, patterns, and raw engine metadata. -The stable ordering makes redaction independent of incidental scanner return -order. +## Applying the policy action -The redaction template accepts static text and `{entity}`. Formatting options, -conversions, and other fields are invalid. +The processor owns the final disposition: -Before allocating replacement text, the processor projects its UTF-8 size -against the body limit. It checks the serialized body again after -reconstruction. - -## Non-replaceable text +| Action | No detections | One or more detections | +| --- | --- | --- | +| `detect` | Allow original body | Allow original body and report detection summaries | +| `block` | Allow original body | Deny with `privacy_guard_blocked` and report detection summaries | +| `replace` | Allow final processed text | Allow final processed text and report detection summaries | -A format handler may expose text that can be observed but not safely rewritten. -It marks that `TextBlock` with `replaceable=False`. +For `replace`, configuration validation requires every stage to advertise +replacement support and to include its valid engine-specific replacement +recipe. A recipe may remain configured but dormant when the action is changed +to detect or block. -The JSON handler uses this for object keys. Observe reports findings in keys, -and block denies normally. Redact denies the request when a selected finding is -in a key because changing a key could create a collision. +Replacement behavior belongs to each engine. For example, `RegexEngine` +selects deterministic non-overlapping matches and renders its constrained +template. A custom engine backed by another tool owns that tool's native +replacement operation. `RequestProcessor` does not reproduce either algorithm. ## Output -The processor returns a `ProcessingResult`: +`RequestProcessor.process()` returns a `RequestProcessingResult`: -- `ALLOW` with no replacement for an unchanged request -- `ALLOW` with replacement bytes for a successful redaction -- `DENY` with `privacy_guard_blocked` for a policy block -- `DENY` with `privacy_guard_limit_exceeded` when safe bounded output cannot be - produced +- detect and block-without-detections return `ALLOW` without replacement text +- block-with-detections returns `DENY`, detection summaries, and + `privacy_guard_blocked` +- replace returns `ALLOW`, the final text, and detection summaries +- timeout or execution-limit exhaustion returns `DENY` with + `privacy_guard_limit_exceeded` and no partial summaries or replacement -Findings remain path-aware in the domain result. The service later aggregates -them for the protobuf response. +The service leaves the original request bytes untouched for detect and block. +For replace it UTF-8 encodes the final text and sets `has_body=true`, including +when the final text happens to equal the input. [Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 3bf8eb8..50716a4 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -1,14 +1,14 @@ --- title: Safety and limits -description: Validation ownership, bounded processing, failure behavior, and logging rules. +description: Validation ownership, bounded entity processing, failure behavior, and logging rules. agent_markdown: true --- # Safety and limits -Privacy Guard bounds input shape, scanning work, findings, replacement size, and -transport output. Limits are part of the architecture, not optional tuning -defaults. +Privacy Guard bounds request text, engine output, detections, regex execution, +transport output, and concurrency. Limits are part of the architecture, not +optional tuning defaults. Package-wide values live in `constants.py`. @@ -18,171 +18,209 @@ Each layer validates facts it can establish: | Layer | Validates | | --- | --- | -| Policy and domain models | Strict field types, shape, and local field rules | -| Scanner public wrapper | Scanner return container and finding models | -| Format-handler public wrappers | Normalized body and reconstructed byte types | -| JSON handler | Early JSON nesting, text-block, and character limits | -| Processor | Registry consistency, original bytes, paths, finding identity, offsets, and authoritative request-wide limits | -| Service | Phase, transport body size, protobuf conversion, response size, and finding aggregation | +| Policy and engine config models | Strict types, exact fields, discriminators, local field rules, catalog bounds, and regex validity | +| `EngineRegistry` | Registration contracts, exact config and resource types, pure resource-backed config checks, and action/replacement compatibility | +| `EntityProcessingEngine.run()` | Text and invocation types, supported strategy, timeout, result model, spans, detection-only immutability, mutation attribution, and stage output bounds | +| Concrete engine | Algorithm-specific execution, replacement completeness, backend timeout propagation, and native failure normalization | +| `RequestProcessor` | Request text bounds, shared timeout, intermediate output, request-wide detection count, aggregation, and policy disposition | +| Service | Phase, transport body size, strict UTF-8, protobuf conversion, replacement encoding, finding representation, and response limits | -Keep validation at these owners. Early checks may bound allocation, while the -next trust boundary repeats authoritative checks for extension output. Do not -move request-wide checks into a scanner or scanner semantics into the service. +Keep request-wide policy decisions in the processor and algorithm-specific +behavior in the concrete engine. The service adapts transport but does not +reimplement either. -## Request limits +## Text, processing, and transport limits | Limit | Constant | Value | Owner | | --- | --- | ---: | --- | | Incoming request body | `MAX_BODY_BYTES` | 4 MiB | Service | -| Projected or reconstructed replacement body | `MAX_BODY_BYTES` | 4 MiB | Processor | -| Serialized replacement body | `MAX_BODY_BYTES` | 4 MiB | Service | +| Input or intermediate text characters | `MAX_SCANNED_CHARACTERS` | 4,194,304 | Processor | +| Input or intermediate UTF-8 bytes | `MAX_BODY_BYTES` | 4 MiB | Processor | +| Engine output UTF-8 bytes | `MAX_BODY_BYTES` | 4 MiB | Engine wrapper | +| Encoded replacement body | `MAX_BODY_BYTES` | 4 MiB | Service | | gRPC receive message | `MAX_RECEIVE_MESSAGE_BYTES` | 5 MiB | gRPC server | -| JSON nesting | `MAX_JSON_NESTING` | 64 levels | JSON handler | -| Text blocks per request | `MAX_TEXT_BLOCKS` | 4,096 | JSON handler + processor | -| Characters scanned per request | `MAX_SCANNED_CHARACTERS` | 4,194,304 | JSON handler + processor | -| Findings returned by one scanner call | `MAX_FINDINGS_PER_BLOCK` | 256 | Scanner wrapper | -| Selected findings per text block | `MAX_FINDINGS_PER_BLOCK` | 256 | Processor | -| Selected findings per request | `MAX_FINDINGS_PER_REQUEST` | 4,096 | Processor | -| Default request scan budget | `DEFAULT_SCAN_TIMEOUT_SECONDS` | 1 second | Processor | -| Maximum configured scan budget | `MAX_SCAN_TIMEOUT_SECONDS` | 30 seconds | Processor | -| Active scanner workers | `MAX_CONCURRENT_SCANS` | 4 | Service | +| Detections returned by one stage | `MAX_DETECTIONS_PER_STAGE` | 256 | Engine wrapper | +| Detections across one request | `MAX_DETECTIONS_PER_REQUEST` | 4,096 | Processor | +| Default shared timeout | `DEFAULT_TIMEOUT_SECONDS` | 1 second | Processor | +| Maximum configured timeout | `MAX_TIMEOUT_SECONDS` | 30 seconds | Processor and `Timeout` | +| Active processor workers | `MAX_CONCURRENT_PROCESSING` | 4 | Service | | Concurrent gRPC calls | `MAX_CONCURRENT_RPCS` | 16 | gRPC server | -The gRPC receive limit includes 1 MiB beyond the body limit for the protobuf -envelope. The service still enforces the advertised 4 MiB body limit. +The gRPC receive allowance is 1 MiB larger than the request body limit for the +protobuf envelope. The service still enforces the advertised 4 MiB body bound. -The JSON handler checks text-block and character limits while parsing. The -processor enforces them authoritatively for every handler. +One monotonic `Timeout` is shared by all stages and final result validation. +`RegexEngine` passes the remaining duration into every backend search. +Third-party calls that accept a timeout should receive the same remaining +duration; calls that cannot be interrupted continue to occupy a service worker +until they exit. -The scan budget is shared across all scanners and text blocks in a request. It -is cooperative: a scanner that calls `remaining_seconds()` after the deadline -causes a limit deny. The processor cannot detect or interrupt an opaque scanner -operation that does not check the budget. - -## Name, finding, and response limits +## Diagnostic and result limits | Limit | Constant | Value | | --- | --- | ---: | -| Entity, scanner, and metadata strings | `MAX_SCANNER_METADATA_BYTES` | 1,024 UTF-8 bytes | -| Metadata entries per finding | `MAX_FINDING_METADATA_ENTRIES` | 32 | +| Bounded diagnostic string | `MAX_DIAGNOSTIC_TEXT_BYTES` | 1,024 UTF-8 bytes | +| Metadata entries per detection | `MAX_FINDING_METADATA_ENTRIES` | 32 | | Aggregated protobuf finding groups | `MAX_PROTO_FINDING_GROUPS` | 32 | | Encoded size per protobuf finding | `MAX_PROTO_FINDING_BYTES` | 4 KiB | -The string limit covers policy entity filters, finding entities, finding scanner -names, and finding metadata keys and values. +The diagnostic-string bound applies to stage names, entity names, metadata +keys and values, model-profile names, and other audit-safe identifiers built +from the shared domain field type. Regex entity and supplied pattern names have a +stricter ASCII grammar and limit described below. -The service aggregates findings before applying protobuf limits. If a safe -result cannot be represented, it returns a limit deny with no partial findings. +The processor aggregates occurrences before the service applies protobuf +limits. If a safe result cannot be represented, the service returns a limit +deny with no partial findings. -## Regex configuration limits +## Regex and catalog limits | Limit | Constant | Value | | --- | --- | ---: | -| YAML file size | `MAX_SCANNER_CONFIG_BYTES` | 16 MiB | -| YAML nesting | `MAX_SCANNER_CONFIG_NESTING` | 16 | -| YAML nodes | `MAX_SCANNER_CONFIG_NODES` | 250,000 | -| YAML scalar | `MAX_SCANNER_CONFIG_SCALAR_BYTES` | 16 KiB | -| Name length | `MAX_REGEX_NAME_BYTES` | 128 ASCII bytes | -| Profiles | `MAX_REGEX_PROFILES` | 32 | -| Entities per profile | `MAX_REGEX_ENTITIES_PER_PROFILE` | 2,000 | -| Patterns per profile | `MAX_REGEX_PATTERNS_PER_PROFILE` | 10,000 | -| Entities across the file | `MAX_REGEX_ENTITIES_TOTAL` | 10,000 | -| Patterns across the file | `MAX_REGEX_PATTERNS_TOTAL` | 50,000 | -| Regex pattern | `MAX_REGEX_PATTERN_BYTES` | 16 KiB | -| Matches per pattern and text block | `MAX_MATCHES_PER_PATTERN` | 256 | - -These catalog limits are separate from request finding limits. A large catalog -may still produce only a few findings for a request. +| Regex entity or supplied pattern name | `MAX_REGEX_NAME_BYTES` | 128 ASCII bytes | +| Entities per catalog | `MAX_REGEX_ENTITIES_PER_CATALOG` | 2,000 | +| Patterns per catalog | `MAX_REGEX_PATTERNS_PER_CATALOG` | 10,000 | +| Pattern string | `MAX_REGEX_PATTERN_BYTES` | 16 KiB | +| Matches per pattern | `MAX_MATCHES_PER_PATTERN` | 256 | + +Entity and pattern names use +`[A-Za-z_][A-Za-z0-9_-]*`. Pattern names are optional; their deterministic +derived diagnostic identities are not serialized back into configuration. + +The per-evaluation config `Struct` is limited to 64 KiB. A catalog may satisfy +Privacy Guard's engine limits yet remain too large for the current upstream +protocol. Internal caching does not raise that transport ceiling. + +## Regex execution safety + +`RegexEngine` validates and compiles the complete catalog before accepting +configuration. It: + +- rejects invalid and empty pattern strings +- rejects patterns that match empty input +- rejects user-defined named groups and inline flags +- verifies its private trailing marker on every match +- treats a context-dependent zero-width match as an atomic configuration + failure +- evaluates patterns independently to retain overlaps +- uses the timeout-capable third-party `regex` backend +- caps matches per pattern and detections per stage +- projects exact UTF-8 replacement size before rendering + +No timeout, limit, or pattern failure returns partial stage detections or +mutated text. ## When a limit is exceeded -Privacy Guard returns a successful deny result with -`privacy_guard_limit_exceeded` when: +The processor returns a successful deny with +`privacy_guard_limit_exceeded` and no partial findings or replacement when: + +- the shared timeout expires +- an engine exceeds its detection or output limit +- intermediate text exceeds the character or byte limit +- aggregate request detections exceed the limit + +The service returns the same bounded deny when: + +- findings exceed the protobuf group or encoded-size limit +- replacement text cannot be encoded within the body limit +- a deny reason code is not safely representable -- a scanner observes an expired scan budget -- findings exceed block or request limits -- projected redaction exceeds the body limit -- a reconstructed body exceeds the body limit -- aggregated findings exceed protobuf limits -- a deny reason code cannot be represented safely +Malformed input and engine contract or execution failures instead abort the RPC +with a cataloged error. OpenShell then applies the middleware registration's +failure behavior. -The limit result contains no replacement body and no partial findings. +The distinction is deliberate: -During evaluation, malformed input and internal failures instead abort the RPC -with a cataloged gRPC error. OpenShell then applies the middleware -registration's failure mode. +- an exhausted declared processing bound is a closed, successful deny +- invalid user input is an `INVALID_ARGUMENT` RPC failure +- an engine or middleware defect is an `INTERNAL` RPC failure ## Error catalog -Production failures use `PrivacyGuardError` and a stable `ErrorCode`. Each error -spec defines: +Production failures use `PrivacyGuardError` and a stable `ErrorCode`. Each +error spec defines: -- whether the failure is invalid input or internal -- the responsible component -- the failed operation -- a content-safe summary -- a remediation hint +- invalid-input or internal classification +- responsible component +- failed operation +- content-safe summary +- remediation hint -Caught extension exceptions are translated at their boundary. Their messages -and exception chains are not exposed. +Caught extension and third-party exceptions are translated at their trust +boundary. Raw exception messages and chains are not exposed. Configuration +errors do not include rejected pattern strings, paths, credentials, provider +endpoints, or request text. -Use a new catalog entry for a new externally observable failure. Do not return -raw parser, scanner, Pydantic, or protobuf exceptions. +Use a new catalog entry for a genuinely new externally observable failure. Do +not return raw Pydantic, regex-backend, engine, or protobuf exceptions. ## Logging Default operational logs include: - request ID -- duration -- action -- finding count +- evaluation duration +- allow or deny action +- aggregate finding count - stable error code -- phase timing and aggregate sizes at debug level +- stage source and invocation strategy at debug level They do not include: -- request or replacement bodies -- text blocks -- matches -- paths -- headers or credentials -- scanner configuration +- request or replacement text +- matched or surrounding text +- offsets +- patterns or catalog contents +- headers, targets, or credentials +- model endpoints - arbitrary exception text -Sensitive model fields use `repr=False` to reduce accidental exposure. This is -not a substitute for safe exception handling. +Sensitive domain fields use `repr=False` to reduce accidental exposure. This +does not replace safe exception handling. -`--debug` enables content-safe phase diagnostics. `--debug-log-content` -deliberately logs complete bodies and emits a warning; use it only in a -controlled development environment. +`--debug` enables content-safe processing diagnostics. +`--debug-log-content` deliberately logs complete input and processed text and +emits a warning; use it only in a controlled development environment. ## Extension requirements -Scanner and format-handler extensions should: - -- return the types declared by the extension hook -- keep request data in local variables or returned request models -- translate expected parse and configuration failures to `PrivacyGuardError` -- leave extension-output validation to the public wrapper -- check applicable limits before allocating output proportional to input or - findings +Custom engines should: + +- use the base constructor and public `run()` wrapper +- return the exact `TextProcessingResult` contract +- keep request data local to `_run()` +- keep initialized state immutable and make resources concurrency-safe +- pass the shared remaining timeout to delegated APIs +- fail the stage on native partial failure +- check applicable bounds before allocating output proportional to input or + detections +- translate expected collaborator failures to the content-safe engine + exception hierarchy - avoid logging input or caught exception text -Do not add retries, fallbacks, or extra validation without a concrete failure -mode and a clear owning layer. +Do not add retries, fallback providers, persistent request artifacts, or extra +validation without a concrete failure mode and a clear owning layer. + +## State and retention + +Cross-request entity memory is not implemented. Privacy Guard retains validated +configuration and immutable engine state in its processor cache, but never +retains request text, detections, or replacement mappings there. ## Changing a limit -A limit change may affect the middleware manifest, processor behavior, protobuf -serialization, tests, examples, and deployment configuration. +A limit change may affect policy schema, processor behavior, engine contracts, +protobuf serialization, tests, examples, and the OpenShell middleware +registration. Before changing a limit: 1. identify every layer that enforces or advertises it 2. update exact-boundary tests -3. check failure behavior above the boundary -4. run `make check` -5. benchmark changes that affect request processing +3. check the failure result immediately above the boundary +4. run the complete project validation +5. benchmark changes that affect regex compilation or request processing +6. coordinate upstream when the copied OpenShell protocol is involved [Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/scanners.md b/docs/documentation/privacy-guard/architecture/scanners.md deleted file mode 100644 index 62e9192..0000000 --- a/docs/documentation/privacy-guard/architecture/scanners.md +++ /dev/null @@ -1,175 +0,0 @@ ---- -title: Scanners -description: Scanner configuration, lifecycle, output contract, and extension rules. -agent_markdown: true ---- - -# Scanners - -A scanner detects sensitive data in one text block. It has no access to the -request format, text-block path, active policy, gRPC request, or protobuf -messages. - -## Scanner interface - -A scanner: - -1. declares a concrete `ScannerConfig` type -2. receives validated configuration at construction -3. receives one text block and a `ScanBudget` in `_scan` -4. returns a tuple of block-relative `Finding` values - -The public `scan` method owns output validation. Extensions implement `_scan` -and do not override `scan`. - -## Configuration - -`ScannerConfig` contains: - -| Field | Purpose | -| --- | --- | -| `name` | Stable scanner identity copied into findings | -| `entity_types` | Complete set of entity names the scanner can emit | - -Custom scanners subclass `ScannerConfig` to add detection-specific fields. -Configuration fields are strict and frozen. Use immutable types for custom -fields because nested values are not deep-frozen. Scanner configuration remains -independent of policy. - -The base constructor infers the concrete config type from -`Scanner[ConcreteConfig]`, validates the supplied model, stores scanner metadata, -and then calls `_initialize`. - -Use `_initialize` for reusable derived state such as compiled expressions. Do -not define a scanner `__init__` for custom initialization. - -## Minimal scanner - -```python -from pydantic import Field - -from privacy_guard.scanners import Finding, ScanBudget, Scanner, ScannerConfig - - -class KeywordScannerConfig(ScannerConfig): - keyword: str = Field(min_length=1) - - -class KeywordScanner(Scanner[KeywordScannerConfig]): - def _scan(self, text_block: str, budget: ScanBudget) -> tuple[Finding, ...]: - start = text_block.find(self.config.keyword) - if start < 0: - return () - return ( - Finding( - entity="keyword", - scanner_name=self.scanner_name, - start_offset=start, - end_offset=start + len(self.config.keyword), - ), - ) -``` - -Construct it with an explicit entity catalog: - -```python -scanner = KeywordScanner( - KeywordScannerConfig( - name="keywords", - entity_types=frozenset({"keyword"}), - keyword="secret", - ) -) -``` - -## Finding model - -A `Finding` contains: - -| Field | Meaning | -| --- | --- | -| `entity` | Entity name declared in `ScannerConfig.entity_types` | -| `scanner_name` | Name of the scanner that produced the finding | -| `start_offset` | Inclusive character offset within the text block | -| `end_offset` | Exclusive character offset within the text block | -| `confidence` | `low`, `medium`, or `high` | -| `metadata` | Optional bounded, immutable scanner attribution | - -Offsets count Python string characters, not UTF-8 bytes. A finding must cover a -non-empty span. - -The processor adds the text-block path later by creating a -`RequestBodyFinding`. Scanners never create or interpret paths. - -## Output validation - -`Scanner.scan` validates the returned tuple and findings. `RequestProcessor` -validates scanner identity, declared entities, offsets, and aggregate limits. -The processor validates every finding, then enforces block and request totals on -findings selected by policy. - -See [Validation ownership](safety-and-limits.md#validation-ownership). - -## Budgets and blocking work - -After normalization, the processor creates one monotonic budget for the scan -phase and shares it across every scanner and text block. The budget is -cooperative. - -Iterative scanners should call `budget.remaining_seconds()` before and after -each bounded unit of work. The method returns the remaining time or raises -`ScanBudgetExceeded`. - -A budget cannot interrupt a blocking operation already in progress. For -example, Python's standard regex engine cannot stop an expression while it is -evaluating. Avoid operations without a tested worst-case bound. - -## Concurrency - -One scanner instance serves concurrent requests through the service worker -pool. Scanner state must be immutable after initialization. - -Do not store: - -- text blocks -- matches from a request -- request IDs -- mutable per-request counters - -Keep per-request data local to `_scan`; do not mutate shared objects from it. - -## Built-in regex scanner - -`RegexScanner` loads a bounded YAML catalog. It compiles configured patterns in -`_initialize` and stores immutable compiled rules. - -Each regex finding includes `pattern_name` metadata. The processor uses entity -names for policy filtering. The service renders the audit label as -`entity/pattern-name`. - -The regex scanner: - -- rejects empty matches -- rejects named groups because it reserves an internal marker group -- rejects inline flags in favor of explicit config fields -- supports overlapping matches by advancing from the previous match start -- limits matches per pattern and findings per block -- checks the shared budget around each regex evaluation - -## Testing a scanner - -Test scanners without gRPC: - -- valid and invalid configuration -- no-match and match cases -- Unicode offsets -- declared entity and scanner identity -- output ordering when relevant -- finding limits -- budget exhaustion for iterative work -- concurrent calls when the scanner keeps derived state - -Processor tests should cover only behavior that requires multiple scanners, -text-block paths, policy, or request-wide limits. - -[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index 10dd18b..fdd3e0a 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -1,6 +1,6 @@ --- title: Service boundary -description: gRPC adaptation, worker scheduling, result aggregation, and server lifecycle. +description: gRPC adaptation, configuration caching, worker scheduling, and server lifecycle. agent_markdown: true --- @@ -8,81 +8,112 @@ agent_markdown: true Outside the generated `bindings/` package, `service/` is the only layer that imports gRPC or generated protobuf bindings. It translates between OpenShell's -transport contract and Privacy Guard's domain types. +transport contract and Privacy Guard's one-text domain contract. + +The `.proto` file and generated bindings checked into this project are copied +from OpenShell. Protocol changes must land upstream and be adopted from a new +canonical copy; Privacy Guard must not create a private fork. ## gRPC methods -Privacy Guard implements three `SupervisorMiddleware` methods: +Privacy Guard implements the three methods currently defined by +`SupervisorMiddleware`: | RPC | Behavior | | --- | --- | -| `Describe` | Advertises the service name, version, pre-credentials HTTP binding, and body limit | -| `ValidateConfig` | Parses policy and asks the processor to validate referenced formats and entities | -| `EvaluateHttpRequest` | Validates transport input, runs the processor, and returns a decision | +| `Describe` | Advertises service identity, one pre-credentials HTTP binding, and the 4 MiB body limit | +| `ValidateConfig` | Purely validates expanded policy configuration and registered resources | +| `EvaluateHttpRequest` | Validates transport input, resolves a processor, processes text, and returns a decision | `Describe` advertises only -`SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS`. The service rejects evaluations -for any other phase. +`SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS`. Evaluations at any other phase +are invalid input. -## Incoming requests +The current manifest message has no field for the finalized policy schema or +engine discovery metadata. Those are available through the local +`privacy-guard schema` and `privacy-guard engines` commands. + +## Configuration lifecycle + +The current protocol carries a complete `google.protobuf.Struct` on both +validation and every request evaluation. + +`ValidateConfig`: + +1. converts the `Struct` to a mapping +2. validates the registry-built discriminated policy model +3. validates each exact engine config against registered resources +4. validates replacement support for a replace action +5. returns `valid=true`, or a content-safe reason -For each evaluation, the servicer: +It does not construct engines, populate the processor cache, contact model +providers, download resources, or write artifacts. -1. validates the phase and body size -2. converts the protobuf `Struct` into a strict `PolicyConfig` -3. extracts the request ID, content type, and body -4. constructs an `InterceptedRequest` -5. passes that domain record to `RequestProcessor` +During evaluation, the service validates the same expanded config, computes its +canonical SHA-256 fingerprint, and resolves a configured processor from a +bounded 128-entry LRU cache. A cache miss constructs engines directly from the +exact stage configs and operator-injected resources, then constructs the +`RequestProcessor`. -Transport-only fields that processing does not need remain at the service -boundary. +The cache is protected for concurrent access and is not correctness-relevant. +Eviction or process restart simply causes reconstruction from a later +evaluation's expanded config. -`content_type` is retained as request context, but the processor selects the -format from policy. It does not infer format from this header. +## Incoming requests + +For each evaluation, the service: + +1. validates the pre-credentials phase +2. validates the transport body byte limit +3. validates and resolves the expanded policy config +4. allows an empty body without invoking an engine +5. decodes a non-empty body as strict UTF-8 +6. schedules `RequestProcessor.process(text)` in the bounded worker pool + +Request context, target, headers, middleware name, and protobuf values remain at +the service boundary. The request ID is used only for content-safe operational +logging. ## Outgoing results -The processor returns `ProcessingResult`. The servicer converts it to +The processor returns `RequestProcessingResult`. The service maps it to `HttpRequestResult`: | Domain result | Protobuf result | | --- | --- | -| Allow without replacement | `DECISION_ALLOW`, `has_body=false` | -| Allow with replacement | `DECISION_ALLOW`, `has_body=true`, replacement bytes | -| Deny | `DECISION_DENY`, no body, stable reason code | - -The service checks replacement size again before serialization. - -### Finding aggregation +| Detect allow | `DECISION_ALLOW`, `has_body=false` | +| Block allow with no detections | `DECISION_ALLOW`, `has_body=false` | +| Replace allow | `DECISION_ALLOW`, `has_body=true`, final UTF-8 body | +| Policy block | `DECISION_DENY`, `privacy_guard_blocked`, no body | +| Limit deny | `DECISION_DENY`, `privacy_guard_limit_exceeded`, no body or partial findings | -Domain findings include text-block paths and exact offsets. The protobuf result -contains count-based audit findings. +The service checks the encoded replacement size again before serialization. -The service groups findings by: +### Findings -- scanner name -- entity -- optional `pattern_name` metadata -- confidence +The processor has already aggregated occurrences by source stage, entity, and +confidence. For each `EntityDetectionSummary`, the service emits: -It emits: +- `type`: `detected_entity` +- `label`: `entity (source-stage)` +- `confidence`: the categorical value, a bounded numeric representation, or + empty when absent +- `count`: aggregate occurrence count -- `type`: scanner name -- `label`: entity or `entity/pattern-name` -- `confidence` -- `count` - -Paths, offsets, and matched request content never cross the protobuf boundary. +The current OpenShell `Finding` message has no dedicated source field. +Stage provenance is therefore secondary text in the bounded label while the +entity remains primary. Matched content, offsets, patterns, and raw engine +metadata never cross the protobuf boundary. ## Concurrency model -The gRPC server allows up to 16 concurrent RPCs. Synchronous processing runs in -a dedicated pool with four workers. +The gRPC server accepts at most 16 concurrent RPCs. Synchronous processing uses +a dedicated executor and semaphore with four active slots: ```text gRPC event loop | - | acquire scan slot + | acquire processing slot v 4-slot semaphore | @@ -91,61 +122,68 @@ gRPC event loop | v RequestProcessor.process + | + v +ordered engine pipeline ``` -This prevents scanner work from blocking the async event loop and bounds active -synchronous scans. +This keeps synchronous engine work off the async event loop and bounds the +number of active processor runs. -Scanner and format-handler instances are shared by those workers. They must be -safe for concurrent use. +Cached processors, engine instances, and injected resources may be used by +multiple worker threads. They must retain no mutable per-request state and must +be safe for concurrent access. ## Cancellation -Cancelling an async RPC cannot stop Python code already running in a worker -thread. The service shields the worker future and keeps its semaphore slot until -the worker actually exits. +Cancelling an async RPC cannot stop Python code already running in its worker +thread. The service shields the worker bridge and releases the semaphore slot +only after that worker actually finishes. -This rule prevents cancellation from creating more than four active scans. +Cancellation therefore cannot create more than four active processor runs. +An engine should pass the remaining shared timeout to any delegated API that +supports bounded execution. A non-preemptible call continues to occupy its slot +until it exits. -During shutdown, the service stops gRPC and waits for the scan executor. +During shutdown, the server stops gRPC and waits for active executor work. ## Error mapping `PrivacyGuardError` classifies each cataloged failure as invalid input or -internal failure. +internal failure: | Error kind | gRPC status | | --- | --- | | `invalid_input` | `INVALID_ARGUMENT` | | `internal` | `INTERNAL` | -This mapping applies to `EvaluateHttpRequest`. `ValidateConfig` reports failures -in `ValidateConfigResponse` with `valid=false` and a safe reason. +This mapping applies to evaluation. `ValidateConfig` instead returns +`valid=false` with a content-safe reason. -Unexpected exceptions become the content-safe -`unexpected_service_failure` error. The service does not return arbitrary -exception text. +Unexpected failures become `unexpected_service_failure`. The service does not +return caught collaborator messages or exception chains. -A gRPC error is different from a policy deny. OpenShell applies its configured -middleware failure mode when the RPC fails. A policy deny is a successful RPC -whose result explicitly stops the request. +A gRPC failure is distinct from a successful policy deny. OpenShell applies the +middleware registration's failure behavior when an RPC fails. A policy deny is +a successful RPC result that explicitly stops the request. -## Server lifecycle +## Server lifecycle and discovery -`MiddlewareServer` is the high-level API for custom scanners. It constructs: +`MiddlewareServer` is the high-level API. It owns: ```text -Scanner - -> RequestProcessor +EngineRegistry -> PrivacyGuardMiddleware -> gRPC server ``` -The CLI uses the same API for built-in scanners. +The default registry includes `RegexEngine`. Operators register custom engines +and resource-backed tool integrations before registry finalization, then pass +that registry to `MiddlewareServer`. The server: -1. creates an unstarted async gRPC server +1. creates an unstarted async gRPC server with receive and concurrency limits 2. binds the configured address 3. starts and waits for termination 4. stops gRPC @@ -153,21 +191,49 @@ The server: A bind failure becomes the stable `server_bind_failed` error. +The CLI exposes: + +```bash +privacy-guard engines +privacy-guard schema +privacy-guard serve --listen 127.0.0.1:50051 +``` + +Entity behavior comes from policy configuration, not server startup flags. + +## Upstream protocol work + +The intended large-catalog and discovery experience requires coordinated +OpenShell changes for: + +- a preparation operation that accepts expanded configuration larger than the + current 64 KiB evaluation limit +- canonical configuration fingerprints on evaluations +- typed cache-miss recovery +- finalized policy schema and engine discovery in the manifest +- a dedicated finding source field + +These are protocol evolution items, not local implementation hooks. Until they +land upstream, Privacy Guard continues to validate the per-evaluation config, +repopulate its local cache when needed, expose discovery through the CLI, and +render stage provenance inside the finding label. + ## Testing the boundary Service tests should cover: - protobuf/domain translation -- manifest contents -- policy validation RPCs -- phase and body-size validation -- allow, replacement, and deny serialization -- finding aggregation and encoded limits +- manifest fields supported by the current protocol +- pure policy validation +- phase, body-size, and UTF-8 validation +- cache hit and reconstruction behavior +- detect, replacement, block, and limit serialization +- finding encoding and limits - gRPC status mapping - worker-slot behavior under cancellation - startup, bind failure, and shutdown -Keep scanner detection and processor policy cases out of service tests unless -the case requires transport translation. +Engine algorithms and ordered policy semantics belong in engine and processor +tests unless transport translation is essential to the case. [Back to the architecture overview](index.md) diff --git a/zensical.toml b/zensical.toml index 5f5bf4a..b6a4ac8 100644 --- a/zensical.toml +++ b/zensical.toml @@ -26,8 +26,8 @@ nav = [ {"Architecture" = [ "documentation/privacy-guard/architecture/index.md", {"Request lifecycle" = "documentation/privacy-guard/architecture/request-lifecycle.md"}, - {"Scanners" = "documentation/privacy-guard/architecture/scanners.md"}, - {"Format handlers" = "documentation/privacy-guard/architecture/format-handlers.md"}, + {"Entity-processing engines" = "documentation/privacy-guard/architecture/entity-processing-engines.md"}, + {"Configuration and text boundary" = "documentation/privacy-guard/architecture/configuration.md"}, {"Service boundary" = "documentation/privacy-guard/architecture/service-boundary.md"}, {"Safety and limits" = "documentation/privacy-guard/architecture/safety-and-limits.md"} ]} From 389e1a09b32320e1965342a9a077af296bf2f95b Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 24 Jul 2026 19:47:26 +0000 Subject: [PATCH 30/82] Model engine strategies as independent support --- .../architecture/entity-processing-engines.md | 26 +++++--- projects/privacy-guard/AGENTS.md | 2 +- projects/privacy-guard/README.md | 9 ++- .../src/privacy_guard/engine_registry.py | 40 +++++++----- .../src/privacy_guard/engines/base.py | 21 ++++--- .../src/privacy_guard/engines/regex.py | 7 ++- .../src/privacy_guard/service/server.py | 12 ++-- .../privacy-guard/tests/engines/test_base.py | 43 +++++++++++-- .../tests/service/test_server.py | 9 ++- .../tests/test_engine_registry.py | 61 +++++++++++++++++-- 10 files changed, 177 insertions(+), 53 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md index 58d0528..aebe9d8 100644 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -19,7 +19,7 @@ with different configurations. An engine: 1. declares concrete configuration and resource types through generics -2. declares its maximum `supported_strategy` +2. declares its non-empty set of `supported_strategies` 3. receives validated configuration and operator-owned resources through the base constructor 4. optionally derives immutable reusable state in `_initialize()` @@ -64,22 +64,23 @@ class EntityProcessingStrategy(StrEnum): REPLACE = "replace" ``` -Every engine supports detection. A detection-only engine declares: +An engine declares exactly the invocation strategies it supports. A +detection-only engine declares: ```python -supported_strategy = EntityProcessingStrategy.DETECT +supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) ``` -A replacement-capable engine declares: +A replacement-only engine declares: ```python -supported_strategy = EntityProcessingStrategy.REPLACE +supported_strategies = frozenset({EntityProcessingStrategy.REPLACE}) ``` -`REPLACE` includes detection support, so `supported_strategy` is a singular -maximum rather than a capability collection. Blocking does not appear here: -the processor runs engines with `DETECT` and applies the block disposition -afterward. +An engine that exposes both operations includes both enum values. Supporting +`REPLACE` does not imply that the engine exposes `DETECT`, even when replacement +requires internal detection. Blocking does not appear here: the processor runs +engines with `DETECT` and applies the block disposition afterward. ## Result contract @@ -142,7 +143,12 @@ class KeywordEngineConfig(EngineConfig[KeywordReplacement]): class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig, None]): - supported_strategy = EntityProcessingStrategy.REPLACE + supported_strategies = frozenset( + { + EntityProcessingStrategy.DETECT, + EntityProcessingStrategy.REPLACE, + } + ) def _run( self, diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md index 80f6b00..a7e3f6a 100644 --- a/projects/privacy-guard/AGENTS.md +++ b/projects/privacy-guard/AGENTS.md @@ -97,7 +97,7 @@ class KeywordConfig(EngineConfig[KeywordReplacement]): class KeywordEngine(EntityProcessingEngine[KeywordConfig, None]): - supported_strategy = EntityProcessingStrategy.DETECT + supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) def _run( self, diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 702854b..3d4ff6d 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -92,7 +92,7 @@ Privacy Guard owns the catalog schema but maintains no authoritative patterns. ## Custom engines Custom engines are a first-class extension point. Authors declare one typed -config, optional typed resources, `supported_strategy`, and `_run`. They do not +config, optional typed resources, `supported_strategies`, and `_run`. They do not write `__init__`; `_initialize` is optional, and `@override` is not required. The first NeMo Anonymizer integration will be implemented as a custom engine, @@ -126,7 +126,12 @@ class AcmeResources: class AcmeEngine(EntityProcessingEngine[AcmeConfig, AcmeResources]): - supported_strategy = EntityProcessingStrategy.REPLACE + supported_strategies = frozenset( + { + EntityProcessingStrategy.DETECT, + EntityProcessingStrategy.REPLACE, + } + ) def _run( self, diff --git a/projects/privacy-guard/src/privacy_guard/engine_registry.py b/projects/privacy-guard/src/privacy_guard/engine_registry.py index 4dd39f0..0ccc5f6 100644 --- a/projects/privacy-guard/src/privacy_guard/engine_registry.py +++ b/projects/privacy-guard/src/privacy_guard/engine_registry.py @@ -38,7 +38,7 @@ class EngineDescription: engine: str description: str - supported_strategy: EntityProcessingStrategy + supported_strategies: frozenset[EntityProcessingStrategy] configuration_schema: dict[str, object] @@ -106,9 +106,16 @@ def register( ): raise EngineRegistryError("engine config type is already registered") - supported_strategy = getattr(engine_type, "supported_strategy", None) - if not isinstance(supported_strategy, EntityProcessingStrategy): - raise EngineRegistryError("engine supported strategy is invalid") + supported_strategies = getattr(engine_type, "supported_strategies", None) + if ( + not isinstance(supported_strategies, frozenset) + or not supported_strategies + or any( + not isinstance(strategy, EntityProcessingStrategy) + for strategy in supported_strategies + ) + ): + raise EngineRegistryError("engine supported strategies are invalid") if resources_runtime_type is NoneType: if resources is not None: raise EngineRegistryError("resource-free engine received resources") @@ -122,7 +129,7 @@ def register( config_type=config_type, resources_type=resources_type, resources=resources, - supported_strategy=supported_strategy, + supported_strategies=supported_strategies, ) def finalize(self) -> FinalizedPrivacyGuardConfigType: @@ -147,6 +154,11 @@ def finalize(self) -> FinalizedPrivacyGuardConfigType: def validate_config(self, values: object) -> FinalizedPrivacyGuardConfig: """Purely parse and validate an expanded Privacy Guard configuration.""" config = parse_privacy_guard_config(self.config_adapter, values) + required_strategy = ( + EntityProcessingStrategy.REPLACE + if config.on_detection.action is PolicyAction.REPLACE + else EntityProcessingStrategy.DETECT + ) for stage in config.entity_processing.stages: registration = self._resolve_registration(stage.config) engine_type = registration.engine_type @@ -160,13 +172,13 @@ def validate_config(self, values: object) -> FinalizedPrivacyGuardConfig: ) except EngineConfigurationError: raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None - if config.on_detection.action is PolicyAction.REPLACE: - if ( - registration.supported_strategy - is not EntityProcessingStrategy.REPLACE - or stage.config.replacement is None - ): - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) + if required_strategy not in registration.supported_strategies: + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) + if ( + required_strategy is EntityProcessingStrategy.REPLACE + and stage.config.replacement is None + ): + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) return config def create_engine( @@ -189,7 +201,7 @@ def describe_engines(self) -> tuple[EngineDescription, ...]: EngineDescription( engine=engine, description=_engine_description(registration.engine_type), - supported_strategy=registration.supported_strategy, + supported_strategies=registration.supported_strategies, configuration_schema=registration.config_type.model_json_schema(), ) for engine, registration in self._registrations.items() @@ -217,7 +229,7 @@ class _Registration: config_type: type[EngineConfig[StrictDomainModel]] resources_type: object resources: object - supported_strategy: EntityProcessingStrategy + supported_strategies: frozenset[EntityProcessingStrategy] def _engine_discriminator( diff --git a/projects/privacy-guard/src/privacy_guard/engines/base.py b/projects/privacy-guard/src/privacy_guard/engines/base.py index f9e21ef..c904fb8 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/base.py +++ b/projects/privacy-guard/src/privacy_guard/engines/base.py @@ -158,7 +158,7 @@ class EngineLimitExceeded(EntityProcessingError): class EntityProcessingEngine(ABC, Generic[_ConfigT, _ResourcesT]): """Nominal, typed extension point for processing one text string.""" - supported_strategy: ClassVar[EntityProcessingStrategy] + supported_strategies: ClassVar[frozenset[EntityProcessingStrategy]] def __init__(self, config: _ConfigT, resources: _ResourcesT) -> None: """Validate typed configuration/resources and initialize reusable state.""" @@ -209,10 +209,16 @@ def get_resources_type(cls) -> object: @classmethod def _validate_class_contract(cls) -> None: - if not isinstance( - getattr(cls, "supported_strategy", None), EntityProcessingStrategy + supported_strategies = getattr(cls, "supported_strategies", None) + if ( + not isinstance(supported_strategies, frozenset) + or not supported_strategies + or any( + not isinstance(strategy, EntityProcessingStrategy) + for strategy in supported_strategies + ) ): - raise EngineConfigurationError("engine supported strategy is invalid") + raise EngineConfigurationError("engine supported strategies are invalid") cls.get_config_type() cls.get_resources_type() @@ -245,11 +251,8 @@ def run( raise EngineContractError("engine processing strategy is invalid") if not isinstance(timeout, Timeout): raise EngineContractError("engine timeout is invalid") - if ( - strategy is EntityProcessingStrategy.REPLACE - and self.supported_strategy is EntityProcessingStrategy.DETECT - ): - raise EngineContractError("engine does not support replacement") + if strategy not in self.supported_strategies: + raise EngineContractError("engine does not support the requested strategy") timeout.raise_if_expired() result: object = self._run( validated_text, diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index 36c6127..31a1668 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -173,7 +173,12 @@ def _patterns_are_valid(self) -> Self: class RegexEngine(EntityProcessingEngine[RegexEngineConfig, None]): """Detect overlapping regex matches and optionally replace deterministic winners.""" - supported_strategy = EntityProcessingStrategy.REPLACE + supported_strategies = frozenset( + { + EntityProcessingStrategy.DETECT, + EntityProcessingStrategy.REPLACE, + } + ) def _initialize(self) -> None: try: diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 7eb84a4..0975608 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -13,7 +13,7 @@ from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES from privacy_guard.engine_registry import EngineRegistry -from privacy_guard.engines import RegexEngine +from privacy_guard.engines import EntityProcessingStrategy, RegexEngine from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.service.servicer import PrivacyGuardMiddleware @@ -140,12 +140,14 @@ def show_schema() -> None: @app.command("engines") def show_engines() -> None: - """List installed engines and their maximum processing strategy.""" + """List installed engines and every supported processing strategy.""" for description in create_default_registry().describe_engines(): - typer.echo( - f"{description.engine}\t{description.supported_strategy.value}" - f"\t{description.description}" + strategies = ",".join( + strategy.value + for strategy in EntityProcessingStrategy + if strategy in description.supported_strategies ) + typer.echo(f"{description.engine}\t{strategies}\t{description.description}") _LOGGER = logging.getLogger(__name__) diff --git a/projects/privacy-guard/tests/engines/test_base.py b/projects/privacy-guard/tests/engines/test_base.py index 0f6bee9..13bd749 100644 --- a/projects/privacy-guard/tests/engines/test_base.py +++ b/projects/privacy-guard/tests/engines/test_base.py @@ -34,7 +34,12 @@ class _Resources: class _CustomEngine(EntityProcessingEngine[_Config, _Resources]): - supported_strategy = EntityProcessingStrategy.REPLACE + supported_strategies = frozenset( + { + EntityProcessingStrategy.DETECT, + EntityProcessingStrategy.REPLACE, + } + ) def _run( self, @@ -116,7 +121,7 @@ def test_detection_confidence_and_metadata_are_strict_bounded_values() -> None: class _DetectOnlyEngine(EntityProcessingEngine[_Config, None]): - supported_strategy = EntityProcessingStrategy.DETECT + supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) def _run( self, @@ -139,8 +144,38 @@ def test_detect_only_engine_rejects_replacement_before_running() -> None: ) +class _ReplaceOnlyEngine(EntityProcessingEngine[_Config, None]): + supported_strategies = frozenset({EntityProcessingStrategy.REPLACE}) + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + del strategy, timeout + return TextProcessingResult(text=text, detections=()) + + +def test_replace_only_engine_rejects_detection_before_running() -> None: + engine = _ReplaceOnlyEngine(_Config(replacement=_Replacement()), None) + + with pytest.raises(EngineContractError): + engine.run( + "text", + strategy=EntityProcessingStrategy.DETECT, + timeout=Timeout.from_seconds(1), + ) + + class _MutatingDetectEngine(EntityProcessingEngine[_Config, None]): - supported_strategy = EntityProcessingStrategy.REPLACE + supported_strategies = frozenset( + { + EntityProcessingStrategy.DETECT, + EntityProcessingStrategy.REPLACE, + } + ) def _run( self, @@ -167,7 +202,7 @@ def test_detection_strategy_rejects_mutated_text() -> None: class _InvalidSpanEngine(EntityProcessingEngine[_Config, None]): - supported_strategy = EntityProcessingStrategy.DETECT + supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) def _run( self, diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index af2a9d5..6166edf 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -80,7 +80,12 @@ def test_default_registry_contains_the_builtin_regex_engine() -> None: assert registry.engine_names == ("regex",) description = registry.describe_engines()[0] assert description.engine == "regex" - assert description.supported_strategy is EntityProcessingStrategy.REPLACE + assert description.supported_strategies == frozenset( + { + EntityProcessingStrategy.DETECT, + EntityProcessingStrategy.REPLACE, + } + ) def test_middleware_server_uses_an_injected_registry_and_default_address( @@ -165,7 +170,7 @@ def test_cli_engines_describes_the_installed_engine() -> None: result = CliRunner().invoke(app, ["engines"]) assert result.exit_code == 0 - assert result.output.startswith("regex\treplace\t") + assert result.output.startswith("regex\tdetect,replace\t") assert "Detect overlapping regex matches" in result.output diff --git a/projects/privacy-guard/tests/test_engine_registry.py b/projects/privacy-guard/tests/test_engine_registry.py index 2d3cdb6..816bf53 100644 --- a/projects/privacy-guard/tests/test_engine_registry.py +++ b/projects/privacy-guard/tests/test_engine_registry.py @@ -18,6 +18,7 @@ RegexEngine, TextProcessingResult, ) +from privacy_guard.errors import PrivacyGuardError from privacy_guard.timeout import Timeout @@ -43,7 +44,12 @@ class AcmeResources: class AcmeEngine(EntityProcessingEngine[AcmeConfig, AcmeResources]): - supported_strategy = EntityProcessingStrategy.REPLACE + supported_strategies = frozenset( + { + EntityProcessingStrategy.DETECT, + EntityProcessingStrategy.REPLACE, + } + ) def _run( self, @@ -61,7 +67,7 @@ class DetectConfig(EngineConfig[AcmeReplacement]): class DetectEngine(EntityProcessingEngine[DetectConfig, None]): - supported_strategy = EntityProcessingStrategy.DETECT + supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) def _run( self, @@ -117,10 +123,53 @@ def test_detection_only_engine_is_rejected_for_replace_action() -> None: "on_detection": {"action": "replace"}, } - with pytest.raises(Exception): + with pytest.raises(PrivacyGuardError): registry.validate_config(values) +class ReplaceOnlyConfig(EngineConfig[AcmeReplacement]): + engine: Literal["replace-only"] = "replace-only" + + +class ReplaceOnlyEngine(EntityProcessingEngine[ReplaceOnlyConfig, None]): + supported_strategies = frozenset({EntityProcessingStrategy.REPLACE}) + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + del strategy, timeout + return TextProcessingResult(text=text, detections=()) + + +def test_replacement_only_engine_is_rejected_for_detect_action() -> None: + registry = EngineRegistry() + registry.register(ReplaceOnlyEngine) + registry.finalize() + values = { + "entity_processing": { + "stages": [ + { + "config": { + "engine": "replace-only", + "replacement": {"strategy": "token"}, + } + } + ] + }, + "on_detection": {"action": "detect"}, + } + + with pytest.raises(PrivacyGuardError): + registry.validate_config(values) + + values["on_detection"] = {"action": "replace"} + registry.validate_config(values) + + def test_registry_is_frozen_after_finalize_and_finalize_is_idempotent() -> None: registry = EngineRegistry() registry.register(RegexEngine) @@ -145,7 +194,7 @@ def test_registry_rejects_duplicate_discriminators_and_resource_mismatch() -> No def test_describe_does_not_construct_an_engine() -> None: class CountingEngine(EntityProcessingEngine[DetectConfig, None]): - supported_strategy = EntityProcessingStrategy.DETECT + supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) initialized = 0 def _initialize(self) -> None: @@ -169,7 +218,9 @@ def _run( assert CountingEngine.initialized == 0 assert descriptions[0].engine == "detect-only" - assert descriptions[0].supported_strategy is EntityProcessingStrategy.DETECT + assert descriptions[0].supported_strategies == frozenset( + {EntityProcessingStrategy.DETECT} + ) properties_value = descriptions[0].configuration_schema["properties"] assert isinstance(properties_value, Mapping) properties = { From 8df3bce082d744e651f35bc92cc1ff248abb31c0 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 24 Jul 2026 20:35:56 +0000 Subject: [PATCH 31/82] Make custom entity processing engines first-class --- .../architecture/entity-processing-engines.md | 98 ++++++++++-- .../privacy-guard/architecture/index.md | 7 +- .../architecture/service-boundary.md | 24 ++- projects/privacy-guard/AGENTS.md | 17 +- projects/privacy-guard/README.md | 85 +++++----- .../examples/custom-engine/.gitignore | 1 + .../examples/custom-engine/README.md | 130 +++++++++++++++ .../examples/custom-engine/custom_engine.py | 149 ++++++++++++++++++ .../examples/custom-engine/gateway.toml | 11 ++ .../examples/custom-engine/policy.yaml | 56 +++++++ .../custom-engine/privacy-guard-config.yaml | 12 ++ .../privacy-guard/src/privacy_guard/config.py | 14 +- .../src/privacy_guard/engine_registry.py | 62 +++++--- .../src/privacy_guard/engines/__init__.py | 2 + .../src/privacy_guard/engines/base.py | 116 +++++++++----- .../src/privacy_guard/engines/regex.py | 19 ++- .../src/privacy_guard/service/server.py | 98 ++++++++++-- .../src/privacy_guard/service/servicer.py | 4 +- .../privacy-guard/tests/engines/test_base.py | 16 +- .../tests/examples/test_custom_engine.py | 128 +++++++++++++++ .../tests/service/test_server.py | 90 ++++++++++- .../tests/service/test_servicer.py | 10 +- projects/privacy-guard/tests/test_config.py | 4 +- .../tests/test_engine_registry.py | 63 ++++++-- 24 files changed, 1026 insertions(+), 190 deletions(-) create mode 100644 projects/privacy-guard/examples/custom-engine/.gitignore create mode 100644 projects/privacy-guard/examples/custom-engine/README.md create mode 100644 projects/privacy-guard/examples/custom-engine/custom_engine.py create mode 100644 projects/privacy-guard/examples/custom-engine/gateway.toml create mode 100644 projects/privacy-guard/examples/custom-engine/policy.yaml create mode 100644 projects/privacy-guard/examples/custom-engine/privacy-guard-config.yaml create mode 100644 projects/privacy-guard/tests/examples/test_custom_engine.py diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md index aebe9d8..437b07b 100644 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -18,7 +18,8 @@ with different configurations. An engine: -1. declares concrete configuration and resource types through generics +1. declares a concrete configuration type and, when needed, a resource type + through generics 2. declares its non-empty set of `supported_strategies` 3. receives validated configuration and operator-owned resources through the base constructor @@ -37,24 +38,60 @@ Every concrete config subclasses `EngineConfig` and declares exactly one literal `engine` discriminator: ```python -class AcmeEngineConfig(EngineConfig[AcmeReplacement]): +class AcmeEngineConfig(EngineConfig): engine: Literal["acme-pii"] = "acme-pii" + replacement: AcmeReplacement | None = None ``` The object under `EntityProcessingStage.config` is this exact concrete model. It is validated and serialized as a member of the registry-built Pydantic discriminated union, then passed unchanged to the engine constructor. -`EngineConfig` provides the optional location for an engine-specific -replacement recipe. Engines with multiple replacement algorithms define their -own nested discriminated union, using fields such as -`replacement.strategy`. There is no closed Privacy Guard enum of all possible -replacement algorithms. - -Configuration contains privacy behavior. Runtime resources contain operational -details such as model clients, credentials, endpoints, and approved profiles. +`EngineConfig` is a nominal, strict base model. It does not prescribe a +`replacement` field or any other algorithm-specific setting. An engine that +needs a replacement recipe declares that field on its concrete config. Engines +with multiple replacement algorithms may define a nested discriminated union, +using fields such as `replacement.strategy`. An engine whose underlying tool +has intrinsic replacement behavior may support `REPLACE` without declaring a +replacement field at all. + +Configuration contains privacy behavior. An optional `EngineResources` object +contains operator-owned runtime dependencies such as initialized model clients, +SDK adapters, endpoints, credential providers, or approved model profiles. Resources are registered by the operator and never serialized into policy. -Engines without resources declare `None` as their resource type. + +`EngineResources` is a nominal contract. A concrete resource bundle subclasses +it, is typed as the engine's second generic argument, and must: + +- contain operational dependencies rather than policy behavior +- retain no request text or mutable per-request state +- expose only dependencies that are safe for concurrent engine calls +- avoid relying on construction or mutation during request processing + +For example: + +```python +from dataclasses import dataclass + +from privacy_guard.engines import EngineResources + + +@dataclass(frozen=True) +class AcmeResources(EngineResources): + client: AcmeClient + + +class AcmeEngine(EntityProcessingEngine[AcmeEngineConfig, AcmeResources]): + ... +``` + +Resources are optional. A resource-free engine omits the second generic +argument and receives `None` from the base class: + +```python +class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig]): + ... +``` ## Invocation strategy @@ -82,6 +119,12 @@ An engine that exposes both operations includes both enum values. Supporting requires internal detection. Blocking does not appear here: the processor runs engines with `DETECT` and applies the block disposition afterward. +The registry calls `validate_run_config()` with the strategy derived from the +policy action. The base implementation verifies strategy support. An engine +may add technique-specific requirements through `_validate_run_config()`, such +as requiring a template only when `REPLACE` is requested. The engine receives +the strategy, never the user-facing `PolicyAction`. + ## Result contract `TextProcessingResult` contains: @@ -137,12 +180,13 @@ class KeywordReplacement(StrictDomainModel): token: str = "[keyword]" -class KeywordEngineConfig(EngineConfig[KeywordReplacement]): +class KeywordEngineConfig(EngineConfig): engine: Literal["keyword"] = "keyword" keyword: str = Field(min_length=1) + replacement: KeywordReplacement | None = None -class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig, None]): +class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig]): supported_strategies = frozenset( { EntityProcessingStrategy.DETECT, @@ -150,6 +194,22 @@ class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig, None]): } ) + @classmethod + def _validate_run_config( + cls, + config: KeywordEngineConfig, + resources: None, + *, + strategy: EntityProcessingStrategy, + ) -> None: + if ( + strategy is EntityProcessingStrategy.REPLACE + and config.replacement is None + ): + raise EngineConfigurationError( + "keyword replacement configuration is required" + ) + def _run( self, text: str, @@ -180,16 +240,24 @@ class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig, None]): return TextProcessingResult(text=output, detections=(detection,)) ``` -Register the implementation and its resources before finalizing the registry: +Register the implementation and, when required, its resources before finalizing +the registry: ```python +registry = EngineRegistry() registry.register(KeywordEngine) -registry.finalize() +registry = registry.finalize() ``` Finalization freezes registration and constructs the exact policy config type, JSON Schema, and engine discovery metadata. +The complete example at +`projects/privacy-guard/examples/custom-engine/README.md` adapts an +operator-provided analysis tool, injects it as a typed resource, and runs its +finalized registry through discovery, schema generation, serving, and an +OpenShell policy. + ## Timeout and concurrency One `Timeout` is created for the processor run and passed through every stage. diff --git a/docs/documentation/privacy-guard/architecture/index.md b/docs/documentation/privacy-guard/architecture/index.md index 3e5a929..a1dc274 100644 --- a/docs/documentation/privacy-guard/architecture/index.md +++ b/docs/documentation/privacy-guard/architecture/index.md @@ -129,7 +129,7 @@ so engine work does not block the gRPC event loop. | --- | --- | | `PrivacyGuardConfig` | Ordered entity-processing stages and the required action on detection | | `EntityProcessingStage` | One configured engine invocation with an optional diagnostic name | -| `EngineConfig` | Generic base for an engine's exact policy configuration and optional replacement recipe | +| `EngineConfig` | Nominal strict base for an engine's exact policy configuration | | `EntityProcessingStrategy` | Per-run engine selection: detect or replace | | `EntityDetection` | One occurrence with stage-input offsets and optional confidence | | `TextProcessingResult` | One engine's authoritative output text and detections | @@ -147,8 +147,9 @@ representations. - Engines do not receive transport metadata or the user-facing policy action. - There is no parallel execution-plan model; preparation constructs a `RequestProcessor` directly. -- There is no generic replacement-strategy enum. Each engine owns the - discriminated replacement recipes appropriate to its underlying algorithm. +- There is no generic replacement field or replacement-strategy enum. Each + engine owns any replacement settings appropriate to its underlying + algorithm. - Runtime policy models do not accept catalog paths. Transparent file expansion requires an upstream OpenShell policy-authoring feature. diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index fdd3e0a..1e79268 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -177,10 +177,30 @@ EngineRegistry -> gRPC server ``` -The default registry includes `RegexEngine`. Operators register custom engines +The built-in registry includes `RegexEngine`. Operators register custom engines and resource-backed tool integrations before registry finalization, then pass that registry to `MiddlewareServer`. +The registry is an explicit application-scoped dependency, not a global +singleton. `MiddlewareServer` and `PrivacyGuardMiddleware` reject unfinalized +registries. A deployment creates and finalizes one registry during startup; +cached processors then construct configured stage engines from that registry. +Different middleware applications in the same process may intentionally use +different engine inventories or runtime resources. + +The CLI accepts an operator registry factory in `module:factory` form. The +factory is invoked once, must return a finalized `EngineRegistry`, and supplies +the same engine inventory to discovery, schema generation, or serving: + +```bash +privacy-guard --registry-factory my_engines:create_registry engines +privacy-guard --registry-factory my_engines:create_registry schema +privacy-guard --registry-factory my_engines:create_registry serve +``` + +The factory is trusted operator code imported into the Privacy Guard process. +It is not a policy-controlled plugin hook. + The server: 1. creates an unstarted async gRPC server with receive and concurrency limits @@ -200,6 +220,8 @@ privacy-guard serve --listen 127.0.0.1:50051 ``` Entity behavior comes from policy configuration, not server startup flags. +Engine implementations and operational resources come from the selected +application registry. ## Upstream protocol work diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md index a7e3f6a..c11d802 100644 --- a/projects/privacy-guard/AGENTS.md +++ b/projects/privacy-guard/AGENTS.md @@ -74,6 +74,11 @@ Define a concrete `EngineConfig` and implement `_run`. Custom engines do not define `__init__`; use optional `_initialize` for derived immutable state. `@override` is not required. +Resource-free engines omit the second generic argument. Resource-backed engines +declare an `EngineResources` subclass as that argument; the bundle contains +only operator-owned, concurrency-safe runtime dependencies and no policy +behavior or per-request state. + ```python from typing import Literal @@ -83,20 +88,15 @@ from privacy_guard.engines import ( EntityProcessingStrategy, TextProcessingResult, ) -from privacy_guard.base import StrictDomainModel from privacy_guard.timeout import Timeout -class KeywordReplacement(StrictDomainModel): - strategy: Literal["token"] = "token" - - -class KeywordConfig(EngineConfig[KeywordReplacement]): +class KeywordConfig(EngineConfig): engine: Literal["keyword"] = "keyword" keyword: str -class KeywordEngine(EntityProcessingEngine[KeywordConfig, None]): +class KeywordEngine(EntityProcessingEngine[KeywordConfig]): supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) def _run( @@ -112,6 +112,9 @@ class KeywordEngine(EntityProcessingEngine[KeywordConfig, None]): The public `run` method validates extension output. Register every engine before finalizing the registry so policy serialization retains its exact config type. +Custom deployments expose a `module:factory` callable that returns the +application-scoped finalized registry and pass it to the CLI with +`--registry-factory`. ## Change limits diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 3d4ff6d..27f56dc 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -95,69 +95,58 @@ Custom engines are a first-class extension point. Authors declare one typed config, optional typed resources, `supported_strategies`, and `_run`. They do not write `__init__`; `_initialize` is optional, and `@override` is not required. +Resource-backed engines define an `EngineResources` subclass containing their +operator-owned runtime dependencies. Resource bundles may contain initialized +tool clients, SDK adapters, models, endpoints, or credential providers, but +must contain no policy behavior or per-request state and must be safe for +concurrent use. Resource-free engines omit the second +`EntityProcessingEngine` generic argument entirely. + The first NeMo Anonymizer integration will be implemented as a custom engine, not as a built-in or placeholder abstraction in Privacy Guard. -```python -from dataclasses import dataclass -from typing import Literal - -from privacy_guard.engines import ( - EngineConfig, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, -) -from privacy_guard.base import StrictDomainModel -from privacy_guard.timeout import Timeout - - -class AcmeReplacement(StrictDomainModel): - strategy: Literal["token"] = "token" +Application startup registers engines and operator-owned resources, then +returns one finalized registry: +```python +from privacy_guard.engine_registry import EngineRegistry -class AcmeConfig(EngineConfig[AcmeReplacement]): - engine: Literal["acme-pii"] = "acme-pii" +def create_registry() -> EngineRegistry: + registry = EngineRegistry() + registry.register(AcmeEngine, resources=AcmeResources(client=client)) + return registry.finalize() +``` -@dataclass(frozen=True) -class AcmeResources: - client: object +Pass that factory to every CLI operation so discovery, schema generation, and +the running server use the same engine inventory: +```bash +uv run privacy-guard --registry-factory my_engines:create_registry engines +uv run privacy-guard --registry-factory my_engines:create_registry schema +uv run privacy-guard --registry-factory my_engines:create_registry serve +``` -class AcmeEngine(EntityProcessingEngine[AcmeConfig, AcmeResources]): - supported_strategies = frozenset( - { - EntityProcessingStrategy.DETECT, - EntityProcessingStrategy.REPLACE, - } - ) +The [custom engine end-to-end example](examples/custom-engine/README.md) +contains a complete tool adapter, typed policy and replacement configuration, +runtime resource registration, registry factory, OpenShell policy, and +walkthrough. - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - timeout.raise_if_expired() - return TextProcessingResult(text=text, detections=()) -``` +The registry is application-scoped, not a process-global singleton. A +`MiddlewareServer` requires an explicit finalized registry. The finalized +registry builds a Pydantic discriminated union containing the exact config type +of every registered engine, so `stage.config` round-trips without dropping +engine-specific or replacement-variant fields. -Register engines before finalizing the policy schema: +The base installation has an explicit built-in registry containing +`RegexEngine`: ```python -from privacy_guard.engine_registry import EngineRegistry +from privacy_guard.service.server import create_builtin_registry -registry = EngineRegistry() -registry.register(AcmeEngine, resources=AcmeResources(client=client)) -registry.finalize() +registry = create_builtin_registry() ``` -The finalized registry builds a Pydantic discriminated union containing the -exact config type of every registered engine. `stage.config` therefore -round-trips without dropping engine-specific or replacement-variant fields. - ## CLI ```bash @@ -169,6 +158,8 @@ uv run privacy-guard serve --listen 127.0.0.1:50051 Entity behavior is supplied by OpenShell policy config, not server startup flags. Deployment startup owns only installed engine implementations and operator resources such as model profiles, endpoints, clients, and credentials. +Use `--registry-factory module:factory` for a custom engine installation. +Registry factories execute operator Python code; load only trusted modules. ## Safety and limits diff --git a/projects/privacy-guard/examples/custom-engine/.gitignore b/projects/privacy-guard/examples/custom-engine/.gitignore new file mode 100644 index 0000000..b223234 --- /dev/null +++ b/projects/privacy-guard/examples/custom-engine/.gitignore @@ -0,0 +1 @@ +gateway.local.toml diff --git a/projects/privacy-guard/examples/custom-engine/README.md b/projects/privacy-guard/examples/custom-engine/README.md new file mode 100644 index 0000000..9f4b847 --- /dev/null +++ b/projects/privacy-guard/examples/custom-engine/README.md @@ -0,0 +1,130 @@ +# Custom engine end-to-end example + +This example implements `KeywordEngine`, registers an operator-owned +`KeywordAnalysisTool`, runs that custom registry through the standard Privacy +Guard CLI, and supplies an OpenShell policy for an end-to-end replacement. + +The analyzer is intentionally small. Its purpose is to show the complete +integration boundary that a production adapter for a library or service such as +NeMo Anonymizer would use: + +- `KeywordEngineConfig` and `TokenReplacement` contain policy-owned behavior. +- `KeywordAnalysisTool` is an operator-created dependency held by the + `KeywordEngineResources` bundle and never appears in policy. +- `KeywordEngine` translates tool matches into `EntityDetection` objects and + implements detection and replacement. +- `create_registry()` registers the implementation and resource, finalizes the + exact Pydantic policy union, and returns the application-scoped registry. + +Run commands from this directory: + +```bash +cd projects/privacy-guard/examples/custom-engine +``` + +## Inspect the custom installation + +The registry-factory option uses the same custom registry for discovery, +schema generation, and serving: + +```bash +uv run privacy-guard \ + --registry-factory custom_engine:create_registry \ + engines + +uv run privacy-guard \ + --registry-factory custom_engine:create_registry \ + schema +``` + +The engine listing should contain `keyword-tool`, and the schema should contain +its exact `KeywordEngineConfig` fields. Registry factories execute operator +Python code in the Privacy Guard process; use only trusted modules. + +## Run Privacy Guard + +In terminal 1: + +```bash +uv run privacy-guard \ + --registry-factory custom_engine:create_registry \ + serve \ + --listen 0.0.0.0:50051 +``` + +The development server uses unauthenticated plaintext gRPC and receives +potentially sensitive request bodies. Restrict port 50051 to the host and +OpenShell sandbox network. + +## Connect OpenShell + +Enter the host's physical Ethernet or Wi-Fi IPv4 address, then generate the +example-specific gateway configuration: + +```bash +YOUR_HOST_IP= +sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml +grep grpc_endpoint gateway.local.toml +``` + +Do not use `127.0.0.1`, a VPN address, or `host.openshell.internal`; the sandbox +supervisor must be able to reach the middleware. + +In terminal 2, stop the background gateway and run the installed gateway with +the example configuration. These paths target the recommended macOS Homebrew +installation: + +```bash +brew services stop openshell + +OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" \ +openshell-gateway --config "$PWD/gateway.local.toml" +``` + +In terminal 3, register that local gateway and create the sandbox: + +```bash +openshell gateway add \ + https://127.0.0.1:17670 \ + --local \ + --name openshell + +openshell sandbox create \ + --name privacy-guard-custom-engine \ + --from base \ + --no-auto-providers \ + --policy "$PWD/policy.yaml" \ + -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude +``` + +After authenticating Claude Code, enter: + +```text +Tell me something that rhymes with the confidential name Project Cobalt +``` + +Privacy Guard should send `[confidential-project]` instead of `Project Cobalt` +to the provider. Verify the middleware result from finite recent logs, rather +than relying on the model's reply: + +```bash +openshell logs privacy-guard-custom-engine -n 100 +``` + +Look for the `api.anthropic.com/v1/messages` request with `transformed:true` and +a `confidential-project (project-names)` finding. + +## Cleanup + +Exit Claude and delete the sandbox: + +```bash +openshell sandbox delete privacy-guard-custom-engine +``` + +Stop the foreground gateway and Privacy Guard with `Ctrl-C`, then restore the +normal gateway: + +```bash +brew services start openshell +``` diff --git a/projects/privacy-guard/examples/custom-engine/custom_engine.py b/projects/privacy-guard/examples/custom-engine/custom_engine.py new file mode 100644 index 0000000..5858044 --- /dev/null +++ b/projects/privacy-guard/examples/custom-engine/custom_engine.py @@ -0,0 +1,149 @@ +"""Example custom entity-processing engine and application registry.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from pydantic import Field + +from privacy_guard.base import StrictDomainModel +from privacy_guard.engine_registry import EngineRegistry +from privacy_guard.engines import ( + ConfidenceLevel, + EngineConfig, + EngineConfigurationError, + EngineResources, + EntityDetection, + EntityName, + EntityProcessingEngine, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.timeout import Timeout + + +class TokenReplacement(StrictDomainModel): + """Replace every detected keyword with one configured token.""" + + strategy: Literal["token"] = "token" + token: str = Field(min_length=1, max_length=256) + + +class KeywordEngineConfig(EngineConfig): + """Policy-owned behavior for the example keyword-analysis tool.""" + + engine: Literal["keyword-tool"] = "keyword-tool" + entity: EntityName + keyword: str = Field(min_length=1, max_length=256, repr=False) + replacement: TokenReplacement | None = None + + +@dataclass(frozen=True) +class KeywordMatch: + """One match returned by the example third-party-style tool.""" + + start: int + end: int + + +class KeywordAnalysisTool: + """Small stand-in for an operator-provided entity-analysis library or client.""" + + def find_matches(self, text: str, keyword: str) -> tuple[KeywordMatch, ...]: + matches: list[KeywordMatch] = [] + start = 0 + while True: + start = text.find(keyword, start) + if start < 0: + return tuple(matches) + end = start + len(keyword) + matches.append(KeywordMatch(start=start, end=end)) + start = end + + +@dataclass(frozen=True) +class KeywordEngineResources(EngineResources): + """Operator-owned dependencies injected into every configured engine.""" + + analysis_tool: KeywordAnalysisTool + + +class KeywordEngine( + EntityProcessingEngine[KeywordEngineConfig, KeywordEngineResources], +): + """Adapt KeywordAnalysisTool results to the Privacy Guard engine contract.""" + + supported_strategies = frozenset( + { + EntityProcessingStrategy.DETECT, + EntityProcessingStrategy.REPLACE, + } + ) + + @classmethod + def _validate_run_config( + cls, + config: KeywordEngineConfig, + resources: KeywordEngineResources, + *, + strategy: EntityProcessingStrategy, + ) -> None: + del cls, resources + if strategy is EntityProcessingStrategy.REPLACE and config.replacement is None: + raise EngineConfigurationError( + "keyword replacement configuration is required" + ) + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + timeout.raise_if_expired() + matches = self.resources.analysis_tool.find_matches( + text, + self.config.keyword, + ) + detections = tuple( + EntityDetection( + entity=self.config.entity, + start=match.start, + end=match.end, + confidence=ConfidenceLevel.HIGH, + ) + for match in matches + ) + if strategy is EntityProcessingStrategy.DETECT or not matches: + return TextProcessingResult(text=text, detections=detections) + + replacement = self.config.replacement + if replacement is None: + raise EngineConfigurationError( + "keyword replacement configuration is required" + ) + output = _replace_matches(text, matches, replacement.token) + return TextProcessingResult(text=output, detections=detections) + + +def create_registry() -> EngineRegistry: + """Create the finalized application registry consumed by Privacy Guard.""" + registry = EngineRegistry() + registry.register( + KeywordEngine, + resources=KeywordEngineResources(analysis_tool=KeywordAnalysisTool()), + ) + return registry.finalize() + + +def _replace_matches( + text: str, + matches: tuple[KeywordMatch, ...], + replacement: str, +) -> str: + output = text + for match in reversed(matches): + output = output[: match.start] + replacement + output[match.end :] + return output diff --git a/projects/privacy-guard/examples/custom-engine/gateway.toml b/projects/privacy-guard/examples/custom-engine/gateway.toml new file mode 100644 index 0000000..c1bef4e --- /dev/null +++ b/projects/privacy-guard/examples/custom-engine/gateway.toml @@ -0,0 +1,11 @@ +# OpenShell gateway configuration for the custom-engine example. +# Pass this file directly to openshell-gateway after replacing the host IP. + +[openshell] +version = 1 + +[[openshell.supervisor.middleware]] +name = "privacy-guard-custom-engine" +grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051" +max_body_bytes = 4194304 +timeout = "5s" diff --git a/projects/privacy-guard/examples/custom-engine/policy.yaml b/projects/privacy-guard/examples/custom-engine/policy.yaml new file mode 100644 index 0000000..f3ab891 --- /dev/null +++ b/projects/privacy-guard/examples/custom-engine/policy.yaml @@ -0,0 +1,56 @@ +version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + claude_code: + name: Claude Code subscription access + endpoints: + - host: api.anthropic.com + port: 443 + protocol: rest + enforcement: enforce + access: full + - host: platform.claude.com + port: 443 + protocol: rest + enforcement: enforce + access: full + - host: claude.ai + port: 443 + - { host: statsig.anthropic.com, port: 443 } + - { host: sentry.io, port: 443 } + binaries: + - { path: /usr/local/bin/claude } + - { path: /usr/bin/node } + +network_middlewares: + privacy_guard_replace: + name: Replace confidential project names + middleware: privacy-guard-custom-engine + order: 0 + config: + entity_processing: + stages: + - name: project-names + config: + engine: keyword-tool + entity: confidential-project + keyword: Project Cobalt + replacement: + strategy: token + token: "[confidential-project]" + on_detection: + action: replace + on_error: fail_closed + endpoints: + include: + - api.anthropic.com diff --git a/projects/privacy-guard/examples/custom-engine/privacy-guard-config.yaml b/projects/privacy-guard/examples/custom-engine/privacy-guard-config.yaml new file mode 100644 index 0000000..af4925b --- /dev/null +++ b/projects/privacy-guard/examples/custom-engine/privacy-guard-config.yaml @@ -0,0 +1,12 @@ +entity_processing: + stages: + - name: project-names + config: + engine: keyword-tool + entity: confidential-project + keyword: Project Cobalt + replacement: + strategy: token + token: "[confidential-project]" +on_detection: + action: replace diff --git a/projects/privacy-guard/src/privacy_guard/config.py b/projects/privacy-guard/src/privacy_guard/config.py index 76a5a19..e2888ba 100644 --- a/projects/privacy-guard/src/privacy_guard/config.py +++ b/projects/privacy-guard/src/privacy_guard/config.py @@ -55,7 +55,7 @@ def _parse_action(cls, value: object) -> PolicyAction: _EngineConfigT = TypeVar( "_EngineConfigT", - bound=EngineConfig[StrictDomainModel], + bound=EngineConfig, ) @@ -116,14 +116,12 @@ class PrivacyGuardConfig( on_detection: OnDetection = Field(repr=False) -FinalizedPrivacyGuardConfig: TypeAlias = PrivacyGuardConfig[ - EngineConfig[StrictDomainModel] -] +FinalizedPrivacyGuardConfig: TypeAlias = PrivacyGuardConfig[EngineConfig] FinalizedPrivacyGuardConfigType = type[FinalizedPrivacyGuardConfig] def build_privacy_guard_config_type( - config_types: Sequence[type[EngineConfig[StrictDomainModel]]], + config_types: Sequence[type[EngineConfig]], ) -> FinalizedPrivacyGuardConfigType: """Build the exact registry-dependent discriminated policy model.""" if not config_types: @@ -144,7 +142,7 @@ def build_privacy_guard_config_type( def build_privacy_guard_config_adapter( - config_types: Sequence[type[EngineConfig[StrictDomainModel]]], + config_types: Sequence[type[EngineConfig]], ) -> TypeAdapter[FinalizedPrivacyGuardConfig]: """Build the registry-dependent adapter used for validation and schemas.""" config_type = build_privacy_guard_config_type(config_types) @@ -165,7 +163,7 @@ def parse_privacy_guard_config( def canonical_config_json( - config: PrivacyGuardConfig[EngineConfig[StrictDomainModel]], + config: PrivacyGuardConfig[EngineConfig], ) -> bytes: """Serialize every concrete engine field deterministically for hashing.""" return json.dumps( @@ -178,7 +176,7 @@ def canonical_config_json( def configuration_fingerprint( - config: PrivacyGuardConfig[EngineConfig[StrictDomainModel]], + config: PrivacyGuardConfig[EngineConfig], ) -> str: """Return the canonical SHA-256 fingerprint of an expanded policy config.""" return sha256(canonical_config_json(config)).hexdigest() diff --git a/projects/privacy-guard/src/privacy_guard/engine_registry.py b/projects/privacy-guard/src/privacy_guard/engine_registry.py index 0ccc5f6..3fe510d 100644 --- a/projects/privacy-guard/src/privacy_guard/engine_registry.py +++ b/projects/privacy-guard/src/privacy_guard/engine_registry.py @@ -6,12 +6,11 @@ import re from dataclasses import dataclass from types import NoneType -from typing import Literal, get_args, get_origin +from typing import Literal, Self, get_args, get_origin from pydantic import TypeAdapter from pydantic_core import PydanticUndefined -from privacy_guard.base import StrictDomainModel from privacy_guard.config import ( FinalizedPrivacyGuardConfig, FinalizedPrivacyGuardConfigType, @@ -22,6 +21,7 @@ from privacy_guard.engines import ( EngineConfig, EngineConfigurationError, + EngineResources, EntityProcessingEngine, EntityProcessingStrategy, ) @@ -93,9 +93,20 @@ def register( config_type, EngineConfig ): raise EngineRegistryError("engine config type is invalid") - resources_runtime_type = get_origin(resources_type) or resources_type + resources_runtime_type = ( + NoneType + if resources_type is None + else get_origin(resources_type) or resources_type + ) if not isinstance(resources_runtime_type, type): raise EngineRegistryError("engine resources type is invalid") + if resources_runtime_type is not NoneType and not issubclass( + resources_runtime_type, + EngineResources, + ): + raise EngineRegistryError( + "engine resources type must extend EngineResources" + ) engine_name = _engine_discriminator(config_type) if engine_name in self._registrations: @@ -119,10 +130,15 @@ def register( if resources_runtime_type is NoneType: if resources is not None: raise EngineRegistryError("resource-free engine received resources") - elif resources is None or not isinstance(resources, resources_runtime_type): - raise EngineRegistryError( - "engine resources do not match their declared type" - ) + else: + if resources is not None and not isinstance(resources, EngineResources): + raise EngineRegistryError( + "engine resources must extend EngineResources" + ) + if resources is None or not isinstance(resources, resources_runtime_type): + raise EngineRegistryError( + "engine resources do not match their declared type" + ) self._registrations[engine_name] = _Registration( engine_type=engine_type, @@ -132,10 +148,10 @@ def register( supported_strategies=supported_strategies, ) - def finalize(self) -> FinalizedPrivacyGuardConfigType: - """Freeze registrations and build the Pydantic discriminated union.""" + def finalize(self) -> Self: + """Freeze registrations, build the policy union, and return this registry.""" if self.is_finalized: - return self.config_type + return self try: config_type = build_privacy_guard_config_type( tuple( @@ -149,7 +165,7 @@ def finalize(self) -> FinalizedPrivacyGuardConfigType: ) from None self._config_type = config_type self._config_adapter = TypeAdapter(config_type) - return config_type + return self def validate_config(self, values: object) -> FinalizedPrivacyGuardConfig: """Purely parse and validate an expanded Privacy Guard configuration.""" @@ -165,26 +181,20 @@ def validate_config(self, values: object) -> FinalizedPrivacyGuardConfig: if not issubclass(engine_type, EntityProcessingEngine): raise EngineRegistryError("registered engine type is invalid") try: - validate_config = getattr(engine_type, "validate_config") - validate_config( + validate_run_config = getattr(engine_type, "validate_run_config") + validate_run_config( stage.config, registration.resources, + strategy=required_strategy, ) except EngineConfigurationError: raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None - if required_strategy not in registration.supported_strategies: - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) - if ( - required_strategy is EntityProcessingStrategy.REPLACE - and stage.config.replacement is None - ): - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) return config def create_engine( self, - config: EngineConfig[StrictDomainModel], - ) -> EntityProcessingEngine[EngineConfig[StrictDomainModel], object]: + config: EngineConfig, + ) -> EntityProcessingEngine[EngineConfig, EngineResources | None]: """Construct an initialized engine from its exact validated config.""" registration = self._resolve_registration(config) if type(config) is not registration.config_type: @@ -209,7 +219,7 @@ def describe_engines(self) -> tuple[EngineDescription, ...]: def _resolve_registration( self, - config: EngineConfig[StrictDomainModel], + config: EngineConfig, ) -> _Registration: if not self.is_finalized: raise EngineRegistryError("engine registry is not finalized") @@ -226,14 +236,14 @@ def _resolve_registration( @dataclass(frozen=True) class _Registration: engine_type: type[object] - config_type: type[EngineConfig[StrictDomainModel]] + config_type: type[EngineConfig] resources_type: object - resources: object + resources: EngineResources | None supported_strategies: frozenset[EntityProcessingStrategy] def _engine_discriminator( - config_type: type[EngineConfig[StrictDomainModel]], + config_type: type[EngineConfig], ) -> str: field = config_type.model_fields.get("engine") if field is None: diff --git a/projects/privacy-guard/src/privacy_guard/engines/__init__.py b/projects/privacy-guard/src/privacy_guard/engines/__init__.py index ba8bc18..4edc6d8 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/__init__.py +++ b/projects/privacy-guard/src/privacy_guard/engines/__init__.py @@ -11,6 +11,7 @@ EngineContractError, EngineExecutionError, EngineLimitExceeded, + EngineResources, EntityDetection, EntityName, EntityProcessingEngine, @@ -37,6 +38,7 @@ "EngineContractError", "EngineExecutionError", "EngineLimitExceeded", + "EngineResources", "EntityDetection", "EntityName", "EntityProcessingEngine", diff --git a/projects/privacy-guard/src/privacy_guard/engines/base.py b/projects/privacy-guard/src/privacy_guard/engines/base.py index c904fb8..c881d07 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/base.py +++ b/projects/privacy-guard/src/privacy_guard/engines/base.py @@ -11,7 +11,6 @@ ClassVar, Generic, TypeAlias, - TypeVar, get_args, get_origin, ) @@ -23,6 +22,7 @@ field_validator, model_validator, ) +from typing_extensions import TypeVar from privacy_guard.base import StrictDomainModel from privacy_guard.constants import ( @@ -118,17 +118,8 @@ def _detections_are_a_tuple(cls, value: object) -> object: return value -_ReplacementConfigT = TypeVar( - "_ReplacementConfigT", - bound=StrictDomainModel, - covariant=True, -) - - -class EngineConfig(StrictDomainModel, Generic[_ReplacementConfigT]): - """Common typed location for an engine-specific replacement recipe.""" - - replacement: _ReplacementConfigT | None = None +class EngineConfig(StrictDomainModel): + """Nominal base for an engine's exact policy configuration.""" class EntityProcessingError(Exception): @@ -151,8 +142,24 @@ class EngineLimitExceeded(EntityProcessingError): """An engine exceeded a bounded configuration or output limit.""" -_ConfigT = TypeVar("_ConfigT", bound=EngineConfig[StrictDomainModel]) -_ResourcesT = TypeVar("_ResourcesT") +class EngineResources: + """Optional operator-owned runtime dependencies shared by engine instances. + + Resource objects contain initialized operational dependencies such as model + clients, SDK adapters, endpoints, or credential providers. They must not + contain policy behavior or mutable per-request state, and everything they + expose to an engine must be safe for concurrent use. + """ + + __slots__ = () + + +_ConfigT = TypeVar("_ConfigT", bound=EngineConfig) +_ResourcesT = TypeVar( + "_ResourcesT", + bound=EngineResources | None, + default=None, +) class EntityProcessingEngine(ABC, Generic[_ConfigT, _ResourcesT]): @@ -160,7 +167,11 @@ class EntityProcessingEngine(ABC, Generic[_ConfigT, _ResourcesT]): supported_strategies: ClassVar[frozenset[EntityProcessingStrategy]] - def __init__(self, config: _ConfigT, resources: _ResourcesT) -> None: + def __init__( + self, + config: _ConfigT, + resources: _ResourcesT, + ) -> None: """Validate typed configuration/resources and initialize reusable state.""" self.validate_config(config, resources) self.__config = config @@ -188,16 +199,26 @@ def validate_config( cls._validate_config(config, resources) @classmethod - def _validate_config( + def validate_run_config( cls, config: _ConfigT, resources: _ResourcesT, + *, + strategy: EntityProcessingStrategy, ) -> None: - """Optionally validate resource-backed config without side effects.""" + """Validate that one config can execute the requested strategy.""" + if not isinstance(strategy, EntityProcessingStrategy): + raise EngineConfigurationError("engine processing strategy is invalid") + cls.validate_config(config, resources) + if strategy not in cls.supported_strategies: + raise EngineConfigurationError( + "engine does not support the requested strategy" + ) + cls._validate_run_config(config, resources, strategy=strategy) @classmethod - def get_config_type(cls) -> type[EngineConfig[StrictDomainModel]]: - """Return the concrete ``EngineConfig`` generic argument.""" + def get_config_type(cls) -> type[EngineConfig]: + """Return the concrete ``EngineConfig`` type argument.""" config_type, _ = _declared_engine_types(cls) return config_type @@ -207,21 +228,6 @@ def get_resources_type(cls) -> object: _, resources_type = _declared_engine_types(cls) return resources_type - @classmethod - def _validate_class_contract(cls) -> None: - supported_strategies = getattr(cls, "supported_strategies", None) - if ( - not isinstance(supported_strategies, frozenset) - or not supported_strategies - or any( - not isinstance(strategy, EntityProcessingStrategy) - for strategy in supported_strategies - ) - ): - raise EngineConfigurationError("engine supported strategies are invalid") - cls.get_config_type() - cls.get_resources_type() - @property def config(self) -> _ConfigT: """Return the immutable, concrete engine configuration.""" @@ -232,9 +238,6 @@ def resources(self) -> _ResourcesT: """Return the validated, injected runtime resources.""" return self.__resources - def _initialize(self) -> None: - """Optionally initialize reusable state from config and resources.""" - def run( self, text: str, @@ -262,6 +265,42 @@ def run( timeout.raise_if_expired() return _validate_result(validated_text, result, strategy=strategy) + @classmethod + def _validate_config( + cls, + config: _ConfigT, + resources: _ResourcesT, + ) -> None: + """Optionally validate resource-backed config without side effects.""" + + @classmethod + def _validate_run_config( + cls, + config: _ConfigT, + resources: _ResourcesT, + *, + strategy: EntityProcessingStrategy, + ) -> None: + """Optionally validate requirements specific to one run strategy.""" + + @classmethod + def _validate_class_contract(cls) -> None: + supported_strategies = getattr(cls, "supported_strategies", None) + if ( + not isinstance(supported_strategies, frozenset) + or not supported_strategies + or any( + not isinstance(strategy, EntityProcessingStrategy) + for strategy in supported_strategies + ) + ): + raise EngineConfigurationError("engine supported strategies are invalid") + cls.get_config_type() + cls.get_resources_type() + + def _initialize(self) -> None: + """Optionally initialize reusable state from config and resources.""" + @abstractmethod def _run( self, @@ -276,7 +315,7 @@ def _run( def _declared_engine_types( engine_type: type[object], -) -> tuple[type[EngineConfig[StrictDomainModel]], object]: +) -> tuple[type[EngineConfig], object]: for candidate in engine_type.__mro__: for base in getattr(candidate, "__orig_bases__", ()): if get_origin(base) is not EntityProcessingEngine: @@ -332,6 +371,7 @@ def _validate_result( "EngineContractError", "EngineExecutionError", "EngineLimitExceeded", + "EngineResources", "EntityDetection", "EntityName", "EntityProcessingEngine", diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index 31a1668..b8ecb9a 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -147,11 +147,12 @@ def _template_is_safe_and_bounded(cls, value: str) -> str: return value -class RegexEngineConfig(EngineConfig[RegexReplacement]): +class RegexEngineConfig(EngineConfig): """Exact policy configuration owned by ``RegexEngine``.""" engine: Literal["regex"] = "regex" pattern_catalog: RegexPatternCatalog = Field(repr=False) + replacement: RegexReplacement | None = None @model_validator(mode="after") def _patterns_are_valid(self) -> Self: @@ -170,7 +171,7 @@ def _patterns_are_valid(self) -> Self: return self -class RegexEngine(EntityProcessingEngine[RegexEngineConfig, None]): +class RegexEngine(EntityProcessingEngine[RegexEngineConfig]): """Detect overlapping regex matches and optionally replace deterministic winners.""" supported_strategies = frozenset( @@ -180,6 +181,20 @@ class RegexEngine(EntityProcessingEngine[RegexEngineConfig, None]): } ) + @classmethod + def _validate_run_config( + cls, + config: RegexEngineConfig, + resources: None, + *, + strategy: EntityProcessingStrategy, + ) -> None: + del cls, resources + if strategy is EntityProcessingStrategy.REPLACE and config.replacement is None: + raise EngineConfigurationError( + "regex replacement configuration is required" + ) + def _initialize(self) -> None: try: self._rules = tuple( diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 0975608..09110f6 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -3,8 +3,10 @@ from __future__ import annotations import asyncio +import importlib import json import logging +from dataclasses import dataclass from typing import Annotated import grpc @@ -25,12 +27,11 @@ ) -def create_default_registry() -> EngineRegistry: - """Build the operator registry shipped by the base package.""" +def create_builtin_registry() -> EngineRegistry: + """Build the finalized registry shipped by the base package.""" registry = EngineRegistry() registry.register(RegexEngine) - registry.finalize() - return registry + return registry.finalize() class MiddlewareServer: @@ -38,12 +39,12 @@ class MiddlewareServer: def __init__( self, + registry: EngineRegistry, *, - registry: EngineRegistry | None = None, log_request_content: bool = False, ) -> None: self._servicer = PrivacyGuardMiddleware( - registry or create_default_registry(), + registry, log_request_content=log_request_content, ) @@ -85,6 +86,15 @@ async def serve( @app.callback() def main( context: typer.Context, + registry_factory: Annotated[ + str | None, + typer.Option( + help=( + "Python module and callable that return a finalized engine registry, " + "formatted as module:factory." + ), + ), + ] = None, debug: Annotated[ bool, typer.Option(help="Enable content-safe processing diagnostics."), @@ -104,7 +114,10 @@ def main( logging.getLogger("privacy_guard").setLevel( logging.DEBUG if debug or debug_log_content else logging.INFO ) - context.obj = debug_log_content + context.obj = _CommandOptions( + registry=_load_registry(registry_factory), + log_request_content=debug_log_content, + ) if debug_log_content: _LOGGER.warning( "privacy_guard_request_content_logging_enabled " @@ -121,16 +134,20 @@ def run_server( ] = "127.0.0.1:50051", ) -> None: """Run the service; entity behavior comes from prepared policy config.""" + options = _command_options(context) _LOGGER.info("privacy_guard_server_starting listen=%s", listen) - MiddlewareServer(log_request_content=context.obj is True).serve(listen) + MiddlewareServer( + options.registry, + log_request_content=options.log_request_content, + ).serve(listen) @app.command("schema") -def show_schema() -> None: +def show_schema(context: typer.Context) -> None: """Print the exact finalized policy JSON Schema.""" typer.echo( json.dumps( - create_default_registry().configuration_json_schema(), + _command_options(context).registry.configuration_json_schema(), indent=2, ensure_ascii=False, sort_keys=True, @@ -139,9 +156,9 @@ def show_schema() -> None: @app.command("engines") -def show_engines() -> None: +def show_engines(context: typer.Context) -> None: """List installed engines and every supported processing strategy.""" - for description in create_default_registry().describe_engines(): + for description in _command_options(context).registry.describe_engines(): strategies = ",".join( strategy.value for strategy in EntityProcessingStrategy @@ -153,6 +170,61 @@ def show_engines() -> None: _LOGGER = logging.getLogger(__name__) +@dataclass(frozen=True) +class _CommandOptions: + registry: EngineRegistry + log_request_content: bool + + +def _load_registry(factory_reference: str | None) -> EngineRegistry: + if factory_reference is None: + return create_builtin_registry() + module_name, separator, factory_name = factory_reference.partition(":") + if not separator or not module_name or not factory_name: + raise typer.BadParameter( + "registry factory must use module:factory", + param_hint="--registry-factory", + ) + try: + module = importlib.import_module(module_name) + factory = getattr(module, factory_name) + except Exception: + raise typer.BadParameter( + "registry factory could not be loaded", + param_hint="--registry-factory", + ) from None + if not callable(factory): + raise typer.BadParameter( + "registry factory is not callable", + param_hint="--registry-factory", + ) + try: + registry = factory() + except Exception: + raise typer.BadParameter( + "registry factory failed", + param_hint="--registry-factory", + ) from None + if not isinstance(registry, EngineRegistry): + raise typer.BadParameter( + "registry factory returned an invalid object", + param_hint="--registry-factory", + ) + if not registry.is_finalized: + raise typer.BadParameter( + "registry factory returned an unfinalized registry", + param_hint="--registry-factory", + ) + return registry + + +def _command_options(context: typer.Context) -> _CommandOptions: + options = context.obj + if not isinstance(options, _CommandOptions): + raise RuntimeError("Privacy Guard command context is unavailable") + return options + + if __name__ == "__main__": app() @@ -160,7 +232,7 @@ def show_engines() -> None: __all__ = [ "MiddlewareServer", "app", - "create_default_registry", + "create_builtin_registry", "create_server", "serve", ] diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index 21858b6..208428f 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -31,7 +31,7 @@ SERVICE_NAME, SERVICE_VERSION, ) -from privacy_guard.engine_registry import EngineRegistry +from privacy_guard.engine_registry import EngineRegistry, EngineRegistryError from privacy_guard.engines import ConfidenceLevel from privacy_guard.errors import ErrorCode, ErrorKind, PrivacyGuardError from privacy_guard.request_processor import ( @@ -52,7 +52,7 @@ def __init__( log_request_content: bool = False, ) -> None: if not registry.is_finalized: - registry.finalize() + raise EngineRegistryError("middleware requires a finalized engine registry") self._registry = registry self._processors = _RequestProcessorCache( registry, diff --git a/projects/privacy-guard/tests/engines/test_base.py b/projects/privacy-guard/tests/engines/test_base.py index 13bd749..bf09912 100644 --- a/projects/privacy-guard/tests/engines/test_base.py +++ b/projects/privacy-guard/tests/engines/test_base.py @@ -12,6 +12,7 @@ ConfidenceLevel, EngineConfig, EngineContractError, + EngineResources, EntityDetection, EntityProcessingEngine, EntityProcessingStrategy, @@ -24,12 +25,13 @@ class _Replacement(StrictDomainModel): strategy: Literal["token"] = "token" -class _Config(EngineConfig[_Replacement]): +class _Config(EngineConfig): engine: Literal["test"] = "test" + replacement: _Replacement | None = None @dataclass(frozen=True) -class _Resources: +class _Resources(EngineResources): prefix: str @@ -120,7 +122,7 @@ def test_detection_confidence_and_metadata_are_strict_bounded_values() -> None: ) -class _DetectOnlyEngine(EntityProcessingEngine[_Config, None]): +class _DetectOnlyEngine(EntityProcessingEngine[_Config]): supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) def _run( @@ -136,6 +138,8 @@ def _run( def test_detect_only_engine_rejects_replacement_before_running() -> None: engine = _DetectOnlyEngine(_Config(), None) + assert _DetectOnlyEngine.get_resources_type() is None + assert engine.resources is None with pytest.raises(EngineContractError): engine.run( "text", @@ -144,7 +148,7 @@ def test_detect_only_engine_rejects_replacement_before_running() -> None: ) -class _ReplaceOnlyEngine(EntityProcessingEngine[_Config, None]): +class _ReplaceOnlyEngine(EntityProcessingEngine[_Config]): supported_strategies = frozenset({EntityProcessingStrategy.REPLACE}) def _run( @@ -169,7 +173,7 @@ def test_replace_only_engine_rejects_detection_before_running() -> None: ) -class _MutatingDetectEngine(EntityProcessingEngine[_Config, None]): +class _MutatingDetectEngine(EntityProcessingEngine[_Config]): supported_strategies = frozenset( { EntityProcessingStrategy.DETECT, @@ -201,7 +205,7 @@ def test_detection_strategy_rejects_mutated_text() -> None: ) -class _InvalidSpanEngine(EntityProcessingEngine[_Config, None]): +class _InvalidSpanEngine(EntityProcessingEngine[_Config]): supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) def _run( diff --git a/projects/privacy-guard/tests/examples/test_custom_engine.py b/projects/privacy-guard/tests/examples/test_custom_engine.py new file mode 100644 index 0000000..d7caf56 --- /dev/null +++ b/projects/privacy-guard/tests/examples/test_custom_engine.py @@ -0,0 +1,128 @@ +"""End-to-end checks for the custom engine application example.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +EXAMPLE_DIRECTORY = Path(__file__).parents[2] / "examples" / "custom-engine" + + +def test_custom_engine_runs_through_the_middleware_boundary() -> None: + probe = r""" +import asyncio + +from google.protobuf import json_format + +from custom_engine import create_registry +from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 +from privacy_guard.service.servicer import PrivacyGuardMiddleware + +values = { + "entity_processing": { + "stages": [ + { + "name": "project-names", + "config": { + "engine": "keyword-tool", + "entity": "confidential-project", + "keyword": "Project Cobalt", + "replacement": { + "strategy": "token", + "token": "[confidential-project]", + }, + }, + } + ] + }, + "on_detection": {"action": "replace"}, +} +config = pb2.HttpRequestEvaluation().config +json_format.ParseDict(values, config) + + +async def evaluate() -> None: + middleware = PrivacyGuardMiddleware(create_registry()) + try: + result = await middleware._evaluate_http_request( + pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + config=config, + body=b"Discuss Project Cobalt safely.", + ) + ) + finally: + await middleware.close() + + assert result.decision == pb2.DECISION_ALLOW + assert result.has_body is True + assert result.body == b"Discuss [confidential-project] safely." + assert len(result.findings) == 1 + assert result.findings[0].label == ( + "confidential-project (project-names)" + ) + + +asyncio.run(evaluate()) +""" + + subprocess.run( + [sys.executable, "-c", probe], + cwd=EXAMPLE_DIRECTORY, + check=True, + ) + + +def test_custom_registry_drives_cli_discovery_and_schema() -> None: + command = [ + sys.executable, + "-m", + "privacy_guard.service.server", + "--registry-factory", + "custom_engine:create_registry", + ] + + engines = subprocess.run( + [*command, "engines"], + cwd=EXAMPLE_DIRECTORY, + check=True, + capture_output=True, + text=True, + ) + schema = subprocess.run( + [*command, "schema"], + cwd=EXAMPLE_DIRECTORY, + check=True, + capture_output=True, + text=True, + ) + + assert engines.stdout.startswith("keyword-tool\tdetect,replace\t") + assert "regex" not in engines.stdout + serialized_schema = json.loads(schema.stdout) + assert "KeywordEngineConfig" in serialized_schema["$defs"] + keyword_properties = serialized_schema["$defs"]["KeywordEngineConfig"]["properties"] + assert set(keyword_properties) == { + "replacement", + "engine", + "entity", + "keyword", + } + + +def test_openshell_walkthrough_uses_the_custom_registry_and_current_policy() -> None: + policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() + config = (EXAMPLE_DIRECTORY / "privacy-guard-config.yaml").read_text() + gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() + readme = (EXAMPLE_DIRECTORY / "README.md").read_text() + + assert "middleware: privacy-guard-custom-engine" in policy + assert 'name = "privacy-guard-custom-engine"' in gateway + assert "engine: keyword-tool" in policy + assert "engine: keyword-tool" in config + assert "action: replace" in policy + assert "--registry-factory custom_engine:create_registry" in readme + assert "cd projects/privacy-guard/examples/custom-engine" in readme + assert "transformed:true" in readme diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 6166edf..0678671 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -6,19 +6,21 @@ import json import logging import re +from types import SimpleNamespace import grpc import pytest from typer.testing import CliRunner, Result from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES +from privacy_guard.engine_registry import EngineRegistry, EngineRegistryError from privacy_guard.engines import EntityProcessingStrategy from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.service import server as server_module from privacy_guard.service.server import ( MiddlewareServer, app, - create_default_registry, + create_builtin_registry, create_server, serve, ) @@ -70,11 +72,11 @@ def _plain_output(result: Result) -> str: def _middleware() -> PrivacyGuardMiddleware: - return PrivacyGuardMiddleware(create_default_registry()) + return PrivacyGuardMiddleware(create_builtin_registry()) -def test_default_registry_contains_the_builtin_regex_engine() -> None: - registry = create_default_registry() +def test_builtin_registry_contains_the_builtin_regex_engine() -> None: + registry = create_builtin_registry() assert registry.is_finalized is True assert registry.engine_names == ("regex",) @@ -91,7 +93,7 @@ def test_default_registry_contains_the_builtin_regex_engine() -> None: def test_middleware_server_uses_an_injected_registry_and_default_address( monkeypatch: pytest.MonkeyPatch, ) -> None: - registry = create_default_registry() + registry = create_builtin_registry() served: list[tuple[PrivacyGuardMiddleware, str]] = [] async def record_serve(servicer: PrivacyGuardMiddleware, listen: str) -> None: @@ -108,6 +110,11 @@ async def record_serve(servicer: PrivacyGuardMiddleware, listen: str) -> None: assert served[0][0]._processors._log_request_content is True +def test_middleware_server_requires_an_explicit_finalized_registry() -> None: + with pytest.raises(EngineRegistryError, match="finalized"): + MiddlewareServer(EngineRegistry()) + + def test_create_server_sets_transport_limits_and_registers_servicer( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -161,6 +168,7 @@ def test_cli_help_exposes_only_pipeline_server_and_discovery_commands() -> None: assert "engines" in output assert "--debug" in output assert "--debug-log-content" in output + assert "--registry-factory" in output assert "--config" not in output assert "--profile" not in output assert "--scanner-name" not in output @@ -185,6 +193,78 @@ def test_cli_schema_prints_the_finalized_discriminated_policy_schema() -> None: assert '"on_detection"' in serialized +def test_cli_loads_one_finalized_operator_registry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = create_builtin_registry() + factory_calls = 0 + + def create_registry() -> EngineRegistry: + nonlocal factory_calls + factory_calls += 1 + return registry + + monkeypatch.setattr( + server_module.importlib, + "import_module", + lambda module_name: ( + SimpleNamespace(create_registry=create_registry) + if module_name == "operator_engines" + else None + ), + ) + + result = CliRunner().invoke( + app, + ["--registry-factory", "operator_engines:create_registry", "engines"], + ) + + assert result.exit_code == 0 + assert factory_calls == 1 + assert result.output.startswith("regex\tdetect,replace\t") + + +@pytest.mark.parametrize( + ("factory_reference", "reason"), + [ + ("missing-separator", "module:factory"), + ("operator_engines:missing", "could not be loaded"), + ("operator_engines:not_callable", "not callable"), + ("operator_engines:failed", "factory failed"), + ("operator_engines:wrong_type", "invalid"), + ("operator_engines:unfinished", "unfinalized"), + ], +) +def test_cli_rejects_invalid_registry_factories( + monkeypatch: pytest.MonkeyPatch, + factory_reference: str, + reason: str, +) -> None: + def fail() -> EngineRegistry: + raise RuntimeError("sensitive factory failure") + + module = SimpleNamespace( + not_callable=object(), + failed=fail, + wrong_type=lambda: object(), + unfinished=lambda: EngineRegistry(), + ) + monkeypatch.setattr( + server_module.importlib, + "import_module", + lambda _: module, + ) + + result = CliRunner().invoke( + app, + ["--registry-factory", factory_reference, "engines"], + ) + + assert result.exit_code == 2 + assert reason in _plain_output(result) + assert "sensitive factory failure" not in _plain_output(result) + + def test_cli_serve_forwards_operational_options_only( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index 3af4052..7ccb298 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -10,7 +10,7 @@ from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.service.server import create_default_registry +from privacy_guard.service.server import create_builtin_registry from privacy_guard.service.servicer import PrivacyGuardMiddleware @@ -70,7 +70,7 @@ def test_copied_proto_remains_the_current_openshell_contract() -> None: def test_validate_config_is_pure_and_reports_invalid_config() -> None: - middleware = PrivacyGuardMiddleware(create_default_registry()) + middleware = PrivacyGuardMiddleware(create_builtin_registry()) valid = middleware._validate_config( pb2.ValidateConfigRequest(config=_proto_config(_values())) @@ -86,7 +86,7 @@ def test_validate_config_is_pure_and_reports_invalid_config() -> None: def test_evaluation_decodes_one_utf8_text_and_encodes_replacement() -> None: async def evaluate() -> pb2.HttpRequestResult: - middleware = PrivacyGuardMiddleware(create_default_registry()) + middleware = PrivacyGuardMiddleware(create_builtin_registry()) try: return await middleware._evaluate_http_request(_request(b"email a@b.com")) finally: @@ -104,7 +104,7 @@ async def evaluate() -> pb2.HttpRequestResult: def test_invalid_utf8_fails_before_invoking_an_engine() -> None: async def evaluate() -> None: - middleware = PrivacyGuardMiddleware(create_default_registry()) + middleware = PrivacyGuardMiddleware(create_builtin_registry()) try: with pytest.raises(PrivacyGuardError) as captured: await middleware._evaluate_http_request(_request(b"\xff")) @@ -117,7 +117,7 @@ async def evaluate() -> None: def test_detect_returns_no_body_mutation() -> None: async def evaluate() -> pb2.HttpRequestResult: - middleware = PrivacyGuardMiddleware(create_default_registry()) + middleware = PrivacyGuardMiddleware(create_builtin_registry()) try: return await middleware._evaluate_http_request( _request(b"a@b.com", action="detect") diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py index 3cfd74b..334607e 100644 --- a/projects/privacy-guard/tests/test_config.py +++ b/projects/privacy-guard/tests/test_config.py @@ -165,8 +165,10 @@ def test_dormant_replacement_recipe_is_valid_for_detection_only_actions( replacement={"strategy": "template", "template": "[redacted]"}, ) ) + engine_config = config.entity_processing.stages[0].config - assert config.entity_processing.stages[0].config.replacement is not None + assert isinstance(engine_config, RegexEngineConfig) + assert engine_config.replacement is not None @pytest.mark.parametrize( diff --git a/projects/privacy-guard/tests/test_engine_registry.py b/projects/privacy-guard/tests/test_engine_registry.py index 816bf53..7e2bb68 100644 --- a/projects/privacy-guard/tests/test_engine_registry.py +++ b/projects/privacy-guard/tests/test_engine_registry.py @@ -13,6 +13,8 @@ from privacy_guard.engine_registry import EngineRegistry, EngineRegistryError from privacy_guard.engines import ( EngineConfig, + EngineConfigurationError, + EngineResources, EntityProcessingEngine, EntityProcessingStrategy, RegexEngine, @@ -26,9 +28,10 @@ class AcmeReplacement(StrictDomainModel): strategy: Literal["token"] = "token" -class AcmeConfig(EngineConfig[AcmeReplacement]): +class AcmeConfig(EngineConfig): engine: Literal["acme-pii"] = "acme-pii" entities: tuple[str, ...] + replacement: AcmeReplacement | None = None @field_validator("entities", mode="before") @classmethod @@ -39,7 +42,7 @@ def _entities_are_a_tuple(cls, value: object) -> object: @dataclass(frozen=True) -class AcmeResources: +class AcmeResources(EngineResources): prefix: str @@ -51,6 +54,18 @@ class AcmeEngine(EntityProcessingEngine[AcmeConfig, AcmeResources]): } ) + @classmethod + def _validate_run_config( + cls, + config: AcmeConfig, + resources: AcmeResources, + *, + strategy: EntityProcessingStrategy, + ) -> None: + del cls, resources + if strategy is EntityProcessingStrategy.REPLACE and config.replacement is None: + raise EngineConfigurationError("acme replacement configuration is required") + def _run( self, text: str, @@ -62,11 +77,11 @@ def _run( return TextProcessingResult(text=text, detections=()) -class DetectConfig(EngineConfig[AcmeReplacement]): +class DetectConfig(EngineConfig): engine: Literal["detect-only"] = "detect-only" -class DetectEngine(EntityProcessingEngine[DetectConfig, None]): +class DetectEngine(EntityProcessingEngine[DetectConfig]): supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) def _run( @@ -127,11 +142,33 @@ def test_detection_only_engine_is_rejected_for_replace_action() -> None: registry.validate_config(values) -class ReplaceOnlyConfig(EngineConfig[AcmeReplacement]): +def test_engine_owns_strategy_specific_configuration_requirements() -> None: + registry = EngineRegistry() + registry.register(AcmeEngine, resources=AcmeResources(prefix="token")) + registry.finalize() + values = { + "entity_processing": { + "stages": [ + { + "config": { + "engine": "acme-pii", + "entities": ["account"], + } + } + ] + }, + "on_detection": {"action": "replace"}, + } + + with pytest.raises(PrivacyGuardError): + registry.validate_config(values) + + +class ReplaceOnlyConfig(EngineConfig): engine: Literal["replace-only"] = "replace-only" -class ReplaceOnlyEngine(EntityProcessingEngine[ReplaceOnlyConfig, None]): +class ReplaceOnlyEngine(EntityProcessingEngine[ReplaceOnlyConfig]): supported_strategies = frozenset({EntityProcessingStrategy.REPLACE}) def _run( @@ -155,7 +192,6 @@ def test_replacement_only_engine_is_rejected_for_detect_action() -> None: { "config": { "engine": "replace-only", - "replacement": {"strategy": "token"}, } } ] @@ -167,15 +203,18 @@ def test_replacement_only_engine_is_rejected_for_detect_action() -> None: registry.validate_config(values) values["on_detection"] = {"action": "replace"} - registry.validate_config(values) + config = registry.validate_config(values) + + config_type = type(config.entity_processing.stages[0].config) + assert "replacement" not in config_type.model_fields def test_registry_is_frozen_after_finalize_and_finalize_is_idempotent() -> None: registry = EngineRegistry() registry.register(RegexEngine) - first_type = registry.finalize() - assert registry.finalize() is first_type + assert registry.finalize() is registry + assert registry.finalize() is registry with pytest.raises(EngineRegistryError): registry.register(DetectEngine) @@ -188,12 +227,14 @@ def test_registry_rejects_duplicate_discriminators_and_resource_mismatch() -> No registry.register(AcmeEngine, resources=AcmeResources(prefix="other")) with pytest.raises(EngineRegistryError): EngineRegistry().register(AcmeEngine) + with pytest.raises(EngineRegistryError, match="must extend EngineResources"): + EngineRegistry().register(AcmeEngine, resources=object()) with pytest.raises(EngineRegistryError): EngineRegistry().register(DetectEngine, resources=object()) def test_describe_does_not_construct_an_engine() -> None: - class CountingEngine(EntityProcessingEngine[DetectConfig, None]): + class CountingEngine(EntityProcessingEngine[DetectConfig]): supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) initialized = 0 From 5123ba9385fdca20f603a7d887e336028edd83b3 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 24 Jul 2026 21:17:23 +0000 Subject: [PATCH 32/82] Centralize Privacy Guard errors --- .../src/privacy_guard/engine_registry.py | 13 +++++---- .../src/privacy_guard/engines/__init__.py | 12 +++++---- .../src/privacy_guard/engines/base.py | 27 +++++-------------- .../src/privacy_guard/engines/regex.py | 8 +++--- .../privacy-guard/src/privacy_guard/errors.py | 24 +++++++++++++++++ .../src/privacy_guard/request_processor.py | 9 ++++--- .../src/privacy_guard/service/servicer.py | 9 +++++-- .../tests/service/test_server.py | 4 +-- .../tests/test_engine_registry.py | 4 +-- 9 files changed, 66 insertions(+), 44 deletions(-) diff --git a/projects/privacy-guard/src/privacy_guard/engine_registry.py b/projects/privacy-guard/src/privacy_guard/engine_registry.py index 3fe510d..e7ae9b0 100644 --- a/projects/privacy-guard/src/privacy_guard/engine_registry.py +++ b/projects/privacy-guard/src/privacy_guard/engine_registry.py @@ -20,16 +20,16 @@ ) from privacy_guard.engines import ( EngineConfig, - EngineConfigurationError, EngineResources, EntityProcessingEngine, EntityProcessingStrategy, ) -from privacy_guard.errors import ErrorCode, PrivacyGuardError - - -class EngineRegistryError(Exception): - """A content-safe engine registration or registry lifecycle failure.""" +from privacy_guard.errors import ( + EngineConfigurationError, + EngineRegistryError, + ErrorCode, + PrivacyGuardError, +) @dataclass(frozen=True) @@ -277,5 +277,4 @@ def _engine_description( __all__ = [ "EngineDescription", "EngineRegistry", - "EngineRegistryError", ] diff --git a/projects/privacy-guard/src/privacy_guard/engines/__init__.py b/projects/privacy-guard/src/privacy_guard/engines/__init__.py index 4edc6d8..06e33e7 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/__init__.py +++ b/projects/privacy-guard/src/privacy_guard/engines/__init__.py @@ -7,15 +7,10 @@ ConfidenceLevel, DetectionConfidence, EngineConfig, - EngineConfigurationError, - EngineContractError, - EngineExecutionError, - EngineLimitExceeded, EngineResources, EntityDetection, EntityName, EntityProcessingEngine, - EntityProcessingError, EntityProcessingStrategy, TextProcessingResult, UnitInterval, @@ -28,6 +23,13 @@ RegexPatternCatalog, RegexReplacement, ) +from privacy_guard.errors import ( + EngineConfigurationError, + EngineContractError, + EngineExecutionError, + EngineLimitExceeded, + EntityProcessingError, +) __all__ = [ "BoundedMetadata", diff --git a/projects/privacy-guard/src/privacy_guard/engines/base.py b/projects/privacy-guard/src/privacy_guard/engines/base.py index c881d07..4847f71 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/base.py +++ b/projects/privacy-guard/src/privacy_guard/engines/base.py @@ -30,6 +30,13 @@ MAX_DETECTIONS_PER_STAGE, MAX_FINDING_METADATA_ENTRIES, ) +from privacy_guard.errors import ( + EngineConfigurationError, + EngineContractError, + EngineExecutionError, + EngineLimitExceeded, + EntityProcessingError, +) from privacy_guard.string_validators import ( ScalarString, validate_bounded_metadata_string, @@ -122,26 +129,6 @@ class EngineConfig(StrictDomainModel): """Nominal base for an engine's exact policy configuration.""" -class EntityProcessingError(Exception): - """Base for content-safe entity-processing failures.""" - - -class EngineConfigurationError(EntityProcessingError): - """An engine class or configured instance is invalid.""" - - -class EngineContractError(EntityProcessingError): - """An engine invocation or returned result violated the public contract.""" - - -class EngineExecutionError(EntityProcessingError): - """An engine's configured runtime failed to complete one text input.""" - - -class EngineLimitExceeded(EntityProcessingError): - """An engine exceeded a bounded configuration or output limit.""" - - class EngineResources: """Optional operator-owned runtime dependencies shared by engine instances. diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index b8ecb9a..f89c28b 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -25,14 +25,16 @@ from privacy_guard.engines.base import ( ConfidenceLevel, EngineConfig, - EngineConfigurationError, - EngineContractError, - EngineLimitExceeded, EntityDetection, EntityProcessingEngine, EntityProcessingStrategy, TextProcessingResult, ) +from privacy_guard.errors import ( + EngineConfigurationError, + EngineContractError, + EngineLimitExceeded, +) from privacy_guard.string_validators import ScalarString, validate_scalar_string from privacy_guard.timeout import Timeout, TimeoutExpired diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py index b062339..bcb1cbb 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -83,6 +83,30 @@ def __str__(self) -> str: ) +class EntityProcessingError(Exception): + """Base for content-safe entity-processing failures.""" + + +class EngineConfigurationError(EntityProcessingError): + """An engine class or configured instance is invalid.""" + + +class EngineContractError(EntityProcessingError): + """An engine invocation or returned result violated the public contract.""" + + +class EngineExecutionError(EntityProcessingError): + """An engine's configured runtime failed to complete one text input.""" + + +class EngineLimitExceeded(EntityProcessingError): + """An engine exceeded a bounded configuration or output limit.""" + + +class EngineRegistryError(Exception): + """A content-safe engine registration or registry lifecycle failure.""" + + _ERROR_SPECS: dict[ErrorCode, ErrorSpec] = { ErrorCode.CONFIG_INVALID: ErrorSpec( ErrorKind.INVALID_INPUT, diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index af6121d..b5458f8 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -24,13 +24,16 @@ ) from privacy_guard.engines import ( DetectionConfidence, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.errors import ( EngineContractError, EngineLimitExceeded, EntityProcessingError, - EntityProcessingStrategy, - TextProcessingResult, + ErrorCode, + PrivacyGuardError, ) -from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.string_validators import validate_scalar_string from privacy_guard.timeout import Timeout, TimeoutExpired diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index 208428f..e7b2605 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -31,9 +31,14 @@ SERVICE_NAME, SERVICE_VERSION, ) -from privacy_guard.engine_registry import EngineRegistry, EngineRegistryError +from privacy_guard.engine_registry import EngineRegistry from privacy_guard.engines import ConfidenceLevel -from privacy_guard.errors import ErrorCode, ErrorKind, PrivacyGuardError +from privacy_guard.errors import ( + EngineRegistryError, + ErrorCode, + ErrorKind, + PrivacyGuardError, +) from privacy_guard.request_processor import ( EntityDetectionSummary, RequestDecision, diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 0678671..658f9db 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -13,9 +13,9 @@ from typer.testing import CliRunner, Result from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES -from privacy_guard.engine_registry import EngineRegistry, EngineRegistryError +from privacy_guard.engine_registry import EngineRegistry from privacy_guard.engines import EntityProcessingStrategy -from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.errors import EngineRegistryError, ErrorCode, PrivacyGuardError from privacy_guard.service import server as server_module from privacy_guard.service.server import ( MiddlewareServer, diff --git a/projects/privacy-guard/tests/test_engine_registry.py b/projects/privacy-guard/tests/test_engine_registry.py index 7e2bb68..470974d 100644 --- a/projects/privacy-guard/tests/test_engine_registry.py +++ b/projects/privacy-guard/tests/test_engine_registry.py @@ -10,7 +10,7 @@ from pydantic import field_validator from privacy_guard.base import StrictDomainModel -from privacy_guard.engine_registry import EngineRegistry, EngineRegistryError +from privacy_guard.engine_registry import EngineRegistry from privacy_guard.engines import ( EngineConfig, EngineConfigurationError, @@ -20,7 +20,7 @@ RegexEngine, TextProcessingResult, ) -from privacy_guard.errors import PrivacyGuardError +from privacy_guard.errors import EngineRegistryError, PrivacyGuardError from privacy_guard.timeout import Timeout From 3cada6759d916f422e1e31dbe39c85913620f732 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 24 Jul 2026 21:17:55 +0000 Subject: [PATCH 33/82] Complete the custom engine deployment example --- projects/privacy-guard/README.md | 6 +- .../examples/custom-engine/README.md | 214 ++++++++++++++---- .../examples/custom-engine/custom_engine.py | 11 - .../custom-engine/privacy_guard_app.py | 15 ++ .../tests/examples/test_custom_engine.py | 79 ++++--- 5 files changed, 238 insertions(+), 87 deletions(-) create mode 100644 projects/privacy-guard/examples/custom-engine/privacy_guard_app.py diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 27f56dc..90f30db 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -119,7 +119,8 @@ def create_registry() -> EngineRegistry: ``` Pass that factory to every CLI operation so discovery, schema generation, and -the running server use the same engine inventory: +the running server use the same engine inventory. The module must be installed +or otherwise present on Python's import path: ```bash uv run privacy-guard --registry-factory my_engines:create_registry engines @@ -130,7 +131,8 @@ uv run privacy-guard --registry-factory my_engines:create_registry serve The [custom engine end-to-end example](examples/custom-engine/README.md) contains a complete tool adapter, typed policy and replacement configuration, runtime resource registration, registry factory, OpenShell policy, and -walkthrough. +walkthrough. It also shows the explicit `PYTHONPATH` setup needed when the +factory is a standalone local module rather than an installed package. The registry is application-scoped, not a process-global singleton. A `MiddlewareServer` requires an explicit finalized registry. The finalized diff --git a/projects/privacy-guard/examples/custom-engine/README.md b/projects/privacy-guard/examples/custom-engine/README.md index 9f4b847..9e01449 100644 --- a/projects/privacy-guard/examples/custom-engine/README.md +++ b/projects/privacy-guard/examples/custom-engine/README.md @@ -1,8 +1,10 @@ # Custom engine end-to-end example This example implements `KeywordEngine`, registers an operator-owned -`KeywordAnalysisTool`, runs that custom registry through the standard Privacy -Guard CLI, and supplies an OpenShell policy for an end-to-end replacement. +`KeywordAnalysisTool`, and runs the custom engine through Privacy Guard and +OpenShell. The final check sends a Claude Code request containing +`Project Cobalt` and verifies that OpenShell forwards `[confidential-project]` +instead. The analyzer is intentionally small. Its purpose is to show the complete integration boundary that a production adapter for a library or service such as @@ -13,82 +15,178 @@ NeMo Anonymizer would use: `KeywordEngineResources` bundle and never appears in policy. - `KeywordEngine` translates tool matches into `EntityDetection` objects and implements detection and replacement. -- `create_registry()` registers the implementation and resource, finalizes the - exact Pydantic policy union, and returns the application-scoped registry. +- `privacy_guard_app.py` is deployment-owned wiring. Its `create_registry()` + selects the installed engine, creates its runtime resource, and returns the + finalized application-scoped registry. -Run commands from this directory: +An engine author implements only the types in `custom_engine.py`; registration +is not part of the engine contract. The deployment owner performs the small +amount of explicit application assembly in `privacy_guard_app.py`. Privacy Guard +needs that complete inventory at startup to build the exact Pydantic policy +union and inject operator-owned resources. + +## Prerequisites + +This walkthrough targets the protocol and policy schema in OpenShell `v0.0.90`, +the version recorded in Privacy Guard's `.openshell-middleware-manifest.json`. +Other OpenShell releases may have different middleware configuration or CLI +syntax. + +Before starting, have: + +- Python 3.11 or newer and `uv` 0.11 or newer +- OpenShell `v0.0.90`, installed with its package-managed local gateway +- a running Docker or Podman backend supported by OpenShell +- Claude Code subscription access if you want to perform the final provider call + +The gateway lifecycle commands below cover macOS Homebrew and Linux Debian/RPM +installations. Snap, Kubernetes, remote, and custom gateway deployments need +equivalent service-management, TLS, and middleware-routing configuration. + +Confirm the important versions: + +```bash +uv --version +openshell --version +openshell-gateway --version +``` + +Run every command below from this example directory. In each new terminal, +repeat the `cd` command: ```bash cd projects/privacy-guard/examples/custom-engine +uv sync --locked ``` ## Inspect the custom installation -The registry-factory option uses the same custom registry for discovery, -schema generation, and serving: +The console script does not automatically add its current directory to Python's +module path. Export it explicitly so the local example modules are importable: + +```bash +export PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" +``` + +Use the same custom registry for discovery, schema generation, and serving: ```bash uv run privacy-guard \ - --registry-factory custom_engine:create_registry \ + --registry-factory privacy_guard_app:create_registry \ engines uv run privacy-guard \ - --registry-factory custom_engine:create_registry \ + --registry-factory privacy_guard_app:create_registry \ schema ``` -The engine listing should contain `keyword-tool`, and the schema should contain -its exact `KeywordEngineConfig` fields. Registry factories execute operator -Python code in the Privacy Guard process; use only trusted modules. +The first command should print one `keyword-tool` row with `detect,replace`. +The schema should contain `KeywordEngineConfig`, including its exact `entity`, +`keyword`, and `replacement` fields. Registry factories execute operator Python +code in the Privacy Guard process; use only trusted modules. -## Run Privacy Guard +`privacy-guard-config.yaml` shows the standalone engine configuration. OpenShell +does not load that file separately; `policy.yaml` contains the same configuration +inline under `network_middlewares`. -In terminal 1: +## Start Privacy Guard + +In terminal 1, enter this example directory, export `PYTHONPATH` again, and run: ```bash +export PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" + uv run privacy-guard \ - --registry-factory custom_engine:create_registry \ + --registry-factory privacy_guard_app:create_registry \ serve \ --listen 0.0.0.0:50051 ``` -The development server uses unauthenticated plaintext gRPC and receives -potentially sensitive request bodies. Restrict port 50051 to the host and -OpenShell sandbox network. +Leave this terminal running. The development server is unauthenticated +plaintext gRPC and receives potentially sensitive request bodies. Binding to +`0.0.0.0` is necessary for the sandbox supervisor to reach it, but port 50051 +must remain restricted to the host and trusted sandbox network. -## Connect OpenShell +## Create the gateway configuration -Enter the host's physical Ethernet or Wi-Fi IPv4 address, then generate the -example-specific gateway configuration: +The gateway and sandbox supervisor must both be able to reach Privacy Guard. +Find a non-loopback IPv4 address for the physical Ethernet or Wi-Fi interface: ```bash -YOUR_HOST_IP= -sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml -grep grpc_endpoint gateway.local.toml +# macOS examples; use the interface that is actually connected. +ipconfig getifaddr en0 +ipconfig getifaddr en1 + +# Linux: inspect the addresses and choose the LAN address. +hostname -I ``` -Do not use `127.0.0.1`, a VPN address, or `host.openshell.internal`; the sandbox -supervisor must be able to reach the middleware. +In terminal 2, from this example directory, assign the selected address and +generate the local gateway configuration: + +```bash +YOUR_HOST_IP=YOUR_HOST_IPV4 +if [ "$YOUR_HOST_IP" = "YOUR_HOST_IPV4" ] || [ "$YOUR_HOST_IP" = "127.0.0.1" ]; then + echo "Set YOUR_HOST_IP to a non-loopback IPv4 address" +else + sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml + grep grpc_endpoint gateway.local.toml +fi +``` + +Replace `YOUR_HOST_IPV4` with the address you selected. Do not use +`127.0.0.1`, a VPN address, or `host.openshell.internal`: the foreground gateway +process and the sandbox supervisor must both be able to resolve and reach the +configured endpoint. + +## Restart the local gateway with middleware enabled -In terminal 2, stop the background gateway and run the installed gateway with -the example configuration. These paths target the recommended macOS Homebrew -installation: +The installed gateway does not dynamically reload middleware registrations. +Stop its package-managed service, then run the same gateway binary in the +foreground with `gateway.local.toml`. + +Run the command for your host: ```bash +# macOS/Homebrew brew services stop openshell -OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" \ +# Linux Debian/RPM package +systemctl --user stop openshell-gateway +``` + +Still in terminal 2, select the package-managed TLS directory for your host and +start the gateway: + +```bash +# macOS/Homebrew +export OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" + +# Linux Debian/RPM package +export OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/tls" + openshell-gateway --config "$PWD/gateway.local.toml" ``` -In terminal 3, register that local gateway and create the sandbox: +Run only one `export` line. Leave the foreground gateway running. + +## Verify OpenShell and create the sandbox + +The package installer normally creates an `openshell` gateway registration. +Reuse it; attempting to add another gateway with that name fails because it +already exists. + +In terminal 3, from this example directory: ```bash -openshell gateway add \ - https://127.0.0.1:17670 \ - --local \ - --name openshell +openshell gateway select openshell +openshell status +``` +Do not continue until status reports the foreground gateway as connected. +Then create the sandbox: + +```bash openshell sandbox create \ --name privacy-guard-custom-engine \ --from base \ @@ -97,6 +195,10 @@ openshell sandbox create \ -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude ``` +Sandbox creation validates the external middleware registration and the exact +`KeywordEngineConfig` embedded in the policy. A successful creation is therefore +also the end-to-end configuration check. + After authenticating Claude Code, enter: ```text @@ -104,15 +206,20 @@ Tell me something that rhymes with the confidential name Project Cobalt ``` Privacy Guard should send `[confidential-project]` instead of `Project Cobalt` -to the provider. Verify the middleware result from finite recent logs, rather -than relying on the model's reply: +to the provider. + +## Verify the middleware result + +Do not infer success from the model's wording. From another host terminal, +inspect a finite recent log window: ```bash -openshell logs privacy-guard-custom-engine -n 100 +openshell logs privacy-guard-custom-engine -n 100 --source sandbox ``` Look for the `api.anthropic.com/v1/messages` request with `transformed:true` and -a `confidential-project (project-names)` finding. +a `confidential-project (project-names)` finding. The raw confidential value +must not appear in middleware findings. ## Cleanup @@ -122,9 +229,34 @@ Exit Claude and delete the sandbox: openshell sandbox delete privacy-guard-custom-engine ``` -Stop the foreground gateway and Privacy Guard with `Ctrl-C`, then restore the -normal gateway: +Stop the foreground gateway and Privacy Guard with `Ctrl-C`. Restore the +package-managed gateway with the command for your host: ```bash +# macOS/Homebrew brew services start openshell + +# Linux Debian/RPM package +systemctl --user start openshell-gateway ``` + +Verify recovery and remove the generated configuration: + +```bash +openshell gateway select openshell +openshell status +rm gateway.local.toml +``` + +## Troubleshooting + +- `registry factory could not be loaded`: export `PYTHONPATH` in the terminal + running `privacy-guard`. +- Port 17670 is already in use: the package-managed gateway was not stopped. +- The foreground gateway cannot find certificates: use the TLS directory for + your platform exactly as shown above. +- Sandbox creation reports unavailable middleware: confirm terminal 1 is still + running, check the IP in `gateway.local.toml`, and allow trusted sandbox + traffic to host port 50051. +- Policy or middleware registration fields are rejected: confirm both + `openshell` and `openshell-gateway` are from `v0.0.90`. diff --git a/projects/privacy-guard/examples/custom-engine/custom_engine.py b/projects/privacy-guard/examples/custom-engine/custom_engine.py index 5858044..a16dcf9 100644 --- a/projects/privacy-guard/examples/custom-engine/custom_engine.py +++ b/projects/privacy-guard/examples/custom-engine/custom_engine.py @@ -8,7 +8,6 @@ from pydantic import Field from privacy_guard.base import StrictDomainModel -from privacy_guard.engine_registry import EngineRegistry from privacy_guard.engines import ( ConfidenceLevel, EngineConfig, @@ -128,16 +127,6 @@ def _run( return TextProcessingResult(text=output, detections=detections) -def create_registry() -> EngineRegistry: - """Create the finalized application registry consumed by Privacy Guard.""" - registry = EngineRegistry() - registry.register( - KeywordEngine, - resources=KeywordEngineResources(analysis_tool=KeywordAnalysisTool()), - ) - return registry.finalize() - - def _replace_matches( text: str, matches: tuple[KeywordMatch, ...], diff --git a/projects/privacy-guard/examples/custom-engine/privacy_guard_app.py b/projects/privacy-guard/examples/custom-engine/privacy_guard_app.py new file mode 100644 index 0000000..b17181d --- /dev/null +++ b/projects/privacy-guard/examples/custom-engine/privacy_guard_app.py @@ -0,0 +1,15 @@ +"""Deployment-owned assembly for the custom-engine example.""" + +from custom_engine import KeywordAnalysisTool, KeywordEngine, KeywordEngineResources + +from privacy_guard.engine_registry import EngineRegistry + + +def create_registry() -> EngineRegistry: + """Create the application-scoped engine registry used by Privacy Guard.""" + registry = EngineRegistry() + registry.register( + KeywordEngine, + resources=KeywordEngineResources(analysis_tool=KeywordAnalysisTool()), + ) + return registry.finalize() diff --git a/projects/privacy-guard/tests/examples/test_custom_engine.py b/projects/privacy-guard/tests/examples/test_custom_engine.py index d7caf56..32e9c93 100644 --- a/projects/privacy-guard/tests/examples/test_custom_engine.py +++ b/projects/privacy-guard/tests/examples/test_custom_engine.py @@ -3,42 +3,31 @@ from __future__ import annotations import json +import os import subprocess import sys +import tomllib from pathlib import Path +import yaml + EXAMPLE_DIRECTORY = Path(__file__).parents[2] / "examples" / "custom-engine" def test_custom_engine_runs_through_the_middleware_boundary() -> None: probe = r""" import asyncio +from pathlib import Path from google.protobuf import json_format +import yaml -from custom_engine import create_registry from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.service.servicer import PrivacyGuardMiddleware +from privacy_guard_app import create_registry -values = { - "entity_processing": { - "stages": [ - { - "name": "project-names", - "config": { - "engine": "keyword-tool", - "entity": "confidential-project", - "keyword": "Project Cobalt", - "replacement": { - "strategy": "token", - "token": "[confidential-project]", - }, - }, - } - ] - }, - "on_detection": {"action": "replace"}, -} +values = yaml.safe_load(Path("privacy-guard-config.yaml").read_text()) +assert isinstance(values, dict) config = pb2.HttpRequestEvaluation().config json_format.ParseDict(values, config) @@ -76,12 +65,16 @@ async def evaluate() -> None: def test_custom_registry_drives_cli_discovery_and_schema() -> None: + environment = os.environ.copy() + python_path = str(EXAMPLE_DIRECTORY) + existing_python_path = environment.get("PYTHONPATH") + if existing_python_path: + python_path = os.pathsep.join((python_path, existing_python_path)) + environment["PYTHONPATH"] = python_path command = [ - sys.executable, - "-m", - "privacy_guard.service.server", + str(Path(sys.executable).with_name("privacy-guard")), "--registry-factory", - "custom_engine:create_registry", + "privacy_guard_app:create_registry", ] engines = subprocess.run( @@ -90,6 +83,7 @@ def test_custom_registry_drives_cli_discovery_and_schema() -> None: check=True, capture_output=True, text=True, + env=environment, ) schema = subprocess.run( [*command, "schema"], @@ -97,6 +91,7 @@ def test_custom_registry_drives_cli_discovery_and_schema() -> None: check=True, capture_output=True, text=True, + env=environment, ) assert engines.stdout.startswith("keyword-tool\tdetect,replace\t") @@ -113,16 +108,34 @@ def test_custom_registry_drives_cli_discovery_and_schema() -> None: def test_openshell_walkthrough_uses_the_custom_registry_and_current_policy() -> None: - policy = (EXAMPLE_DIRECTORY / "policy.yaml").read_text() - config = (EXAMPLE_DIRECTORY / "privacy-guard-config.yaml").read_text() - gateway = (EXAMPLE_DIRECTORY / "gateway.toml").read_text() + policy = yaml.safe_load((EXAMPLE_DIRECTORY / "policy.yaml").read_text()) + config = yaml.safe_load( + (EXAMPLE_DIRECTORY / "privacy-guard-config.yaml").read_text() + ) + gateway = tomllib.loads((EXAMPLE_DIRECTORY / "gateway.toml").read_text()) readme = (EXAMPLE_DIRECTORY / "README.md").read_text() - assert "middleware: privacy-guard-custom-engine" in policy - assert 'name = "privacy-guard-custom-engine"' in gateway - assert "engine: keyword-tool" in policy - assert "engine: keyword-tool" in config - assert "action: replace" in policy - assert "--registry-factory custom_engine:create_registry" in readme + assert isinstance(policy, dict) + assert isinstance(config, dict) + middleware_config = policy["network_middlewares"]["privacy_guard_replace"] + assert middleware_config["middleware"] == "privacy-guard-custom-engine" + assert middleware_config["config"] == config + middleware = gateway["openshell"]["supervisor"]["middleware"] + assert middleware == [ + { + "name": "privacy-guard-custom-engine", + "grpc_endpoint": "http://REPLACE_WITH_HOST_IP:50051", + "max_body_bytes": 4_194_304, + "timeout": "5s", + } + ] + stage_config = config["entity_processing"]["stages"][0]["config"] + assert stage_config["engine"] == "keyword-tool" + assert config["on_detection"]["action"] == "replace" + assert "--registry-factory privacy_guard_app:create_registry" in readme assert "cd projects/privacy-guard/examples/custom-engine" in readme + assert 'export PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}"' in readme + assert "openshell gateway select openshell" in readme + assert "openshell gateway add" not in readme + assert "OpenShell `v0.0.90`" in readme assert "transformed:true" in readme From ebf7163b9a7faef2a14dbfa68618c5e972edaf60 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 24 Jul 2026 21:18:11 +0000 Subject: [PATCH 34/82] Support file-backed regex pattern catalogs --- projects/privacy-guard/README.md | 13 +- .../examples/regex-engine/.gitignore | 1 + .../examples/regex-engine/README.md | 228 +++++++++++++++++- .../examples/regex-engine/gateway.toml | 11 + .../examples/regex-engine/policy.yaml | 55 +++++ .../regex-engine/privacy-guard-config.yaml | 13 +- projects/privacy-guard/pyproject.toml | 1 + .../src/privacy_guard/constants.py | 2 + .../src/privacy_guard/engines/regex.py | 154 +++++++++++- .../tests/examples/test_regex_engine.py | 116 +++++++++ projects/privacy-guard/tests/test_config.py | 178 +++++++++++++- projects/privacy-guard/uv.lock | 57 +++++ 12 files changed, 798 insertions(+), 31 deletions(-) create mode 100644 projects/privacy-guard/examples/regex-engine/.gitignore create mode 100644 projects/privacy-guard/examples/regex-engine/gateway.toml create mode 100644 projects/privacy-guard/examples/regex-engine/policy.yaml create mode 100644 projects/privacy-guard/tests/examples/test_regex_engine.py diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 90f30db..480a943 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -38,11 +38,14 @@ preceding stage's processed text. Detect and block run the same engines with the detection-only strategy, so replacement recipes may remain configured but dormant. -Privacy Guard accepts a structured catalog, not a filesystem path. The -[regex engine example](examples/regex-engine/README.md) includes a reference -catalog to copy and adapt. Transparent catalog-file expansion belongs in -OpenShell's policy installation flow and is not yet supported by the current -protocol. +`pattern_catalog` accepts either the structured catalog or a relative path to a +complete `.yaml` or `.yml` catalog. File-backed catalogs are resolved beneath +Privacy Guard's working directory and normalized to the same +`RegexPatternCatalog` model as inline input. Absolute paths, traversal, symlinks, +unsafe YAML tags, aliases, duplicate keys, invalid UTF-8, and oversized files +are rejected. The +[RegexEngine end-to-end example](examples/regex-engine/README.md) passes its +catalog as `patterns.yaml`. ## Architecture diff --git a/projects/privacy-guard/examples/regex-engine/.gitignore b/projects/privacy-guard/examples/regex-engine/.gitignore new file mode 100644 index 0000000..b223234 --- /dev/null +++ b/projects/privacy-guard/examples/regex-engine/.gitignore @@ -0,0 +1 @@ +gateway.local.toml diff --git a/projects/privacy-guard/examples/regex-engine/README.md b/projects/privacy-guard/examples/regex-engine/README.md index d40851d..cec2b74 100644 --- a/projects/privacy-guard/examples/regex-engine/README.md +++ b/projects/privacy-guard/examples/regex-engine/README.md @@ -1,15 +1,221 @@ -# RegexEngine policy example +# RegexEngine end-to-end example -This example keeps entity behavior in the OpenShell policy configuration. -`privacy-guard-config.yaml` contains the complete structured -`RegexPatternCatalog` accepted by Privacy Guard. +This example runs Privacy Guard's built-in `RegexEngine` through OpenShell. The +final check sends a Claude Code request containing an email address and customer +ID, then verifies that OpenShell forwards `[email]` and `[customer-id]`. -Privacy Guard does not ship authoritative regex presets. Copy and adapt this -example and the larger `patterns.yaml` reference catalog for the data you -actually need to identify, and test it against representative worst-case inputs +Privacy Guard does not ship authoritative regex presets. Copy and adapt +`patterns.yaml` for the data you actually need to identify, and test every +pattern against representative matches, non-matches, and worst-case inputs before deployment. -The current OpenShell policy flow does not expand catalog file paths. A catalog -must therefore be inline before it is passed to Privacy Guard. Transparent file -expansion and larger prepared catalogs require coordinated upstream OpenShell -support; this project does not fork the copied `.proto`. +## Prerequisites + +This walkthrough targets OpenShell `v0.0.90`, the version recorded in Privacy +Guard's `.openshell-middleware-manifest.json`. Other releases may use different +middleware configuration or CLI syntax. + +Before starting, have: + +- Python 3.11 or newer and `uv` 0.11 or newer +- OpenShell `v0.0.90`, installed with its package-managed local gateway +- a running Docker or Podman backend supported by OpenShell +- Claude Code subscription access if you want to perform the final provider call + +The gateway lifecycle commands below cover macOS Homebrew and Linux Debian/RPM +installations. Snap, Kubernetes, remote, and custom gateway deployments need +equivalent service-management, TLS, and middleware-routing configuration. + +Confirm the versions: + +```bash +uv --version +openshell --version +openshell-gateway --version +``` + +Run every command below from this example directory. In each new terminal, +repeat the `cd` command: + +```bash +cd projects/privacy-guard/examples/regex-engine +uv sync --locked +``` + +## Inspect the built-in installation + +```bash +uv run privacy-guard engines +uv run privacy-guard schema +``` + +The first command should print one `regex` row with `detect,replace`. The schema +should contain `RegexEngineConfig`, `RegexPatternCatalog`, and +`RegexReplacement`. + +`privacy-guard-config.yaml` is the standalone Privacy Guard configuration. +`policy.yaml` contains the same configuration inline under +`network_middlewares`, which is the form OpenShell sends to the middleware. +Both configurations pass `patterns.yaml` directly as the complete +`pattern_catalog`. Privacy Guard resolves that relative path from its working +directory, safely loads the YAML file, and validates it with the same +`RegexPatternCatalog` model used for an inline catalog. + +Catalog paths must be relative `.yaml` or `.yml` paths beneath Privacy Guard's +working directory. Absolute paths, `..` traversal, and symlinks are rejected. +This example therefore starts Privacy Guard from this directory. Restart the +service from the same directory whenever it is stopped. + +## Start Privacy Guard + +In terminal 1, from this example directory: + +```bash +uv run privacy-guard serve --listen 0.0.0.0:50051 +``` + +Leave this terminal running. The development server is unauthenticated +plaintext gRPC and receives potentially sensitive request bodies. Binding to +`0.0.0.0` is necessary for the sandbox supervisor to reach it, but port 50051 +must remain restricted to the host and trusted sandbox network. + +## Create the gateway configuration + +Find a non-loopback IPv4 address for the physical Ethernet or Wi-Fi interface: + +```bash +# macOS examples; use the interface that is actually connected. +ipconfig getifaddr en0 +ipconfig getifaddr en1 + +# Linux: inspect the addresses and choose the LAN address. +hostname -I +``` + +In terminal 2, from this example directory, assign the selected address: + +```bash +YOUR_HOST_IP=YOUR_HOST_IPV4 +if [ "$YOUR_HOST_IP" = "YOUR_HOST_IPV4" ] || [ "$YOUR_HOST_IP" = "127.0.0.1" ]; then + echo "Set YOUR_HOST_IP to a non-loopback IPv4 address" +else + sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml + grep grpc_endpoint gateway.local.toml +fi +``` + +Replace `YOUR_HOST_IPV4` with the address you selected. Do not use +`127.0.0.1`, a VPN address, or `host.openshell.internal`: the foreground gateway +process and the sandbox supervisor must both be able to resolve and reach the +configured endpoint. + +## Restart the local gateway with middleware enabled + +The installed gateway does not dynamically reload middleware registrations. +Stop its package-managed service with the command for your host: + +```bash +# macOS/Homebrew +brew services stop openshell + +# Linux Debian/RPM package +systemctl --user stop openshell-gateway +``` + +Still in terminal 2, select the package-managed TLS directory for your host and +start the gateway: + +```bash +# macOS/Homebrew +export OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" + +# Linux Debian/RPM package +export OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/tls" + +openshell-gateway --config "$PWD/gateway.local.toml" +``` + +Run only one `export` line. Leave the foreground gateway running. + +## Verify OpenShell and create the sandbox + +In terminal 3, from this example directory: + +```bash +openshell gateway select openshell +openshell status +``` + +Do not continue until status reports the foreground gateway as connected. +Then create the sandbox: + +```bash +openshell sandbox create \ + --name privacy-guard-regex \ + --from base \ + --no-auto-providers \ + --policy "$PWD/policy.yaml" \ + -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude +``` + +Sandbox creation validates the external middleware registration and the exact +`RegexEngineConfig` embedded in the policy. + +After authenticating Claude Code, enter: + +```text +Draft a short greeting for user@example.com about customer CUST-12345678. +``` + +Privacy Guard should send `[email]` and `[customer-id]` instead of the original +identifiers to the provider. + +## Verify the middleware result + +From another host terminal: + +```bash +openshell logs privacy-guard-regex -n 100 --source sandbox +``` + +Look for the `api.anthropic.com/v1/messages` request with `transformed:true`, +plus `email (identifiers)` and `customer-id (identifiers)` findings. Findings +must not contain the matched email address or customer ID. + +## Cleanup + +Exit Claude and delete the sandbox: + +```bash +openshell sandbox delete privacy-guard-regex +``` + +Stop the foreground gateway and Privacy Guard with `Ctrl-C`. Restore the +package-managed gateway with the command for your host: + +```bash +# macOS/Homebrew +brew services start openshell + +# Linux Debian/RPM package +systemctl --user start openshell-gateway +``` + +Verify recovery and remove the generated gateway file: + +```bash +openshell gateway select openshell +openshell status +rm gateway.local.toml +``` + +## Troubleshooting + +- Port 17670 is already in use: the package-managed gateway was not stopped. +- The foreground gateway cannot find certificates: use the TLS directory for + your platform exactly as shown above. +- Sandbox creation reports unavailable middleware: confirm terminal 1 is still + running, check the IP in `gateway.local.toml`, and allow trusted sandbox + traffic to host port 50051. +- Policy or middleware registration fields are rejected: confirm both + `openshell` and `openshell-gateway` are from `v0.0.90`. diff --git a/projects/privacy-guard/examples/regex-engine/gateway.toml b/projects/privacy-guard/examples/regex-engine/gateway.toml new file mode 100644 index 0000000..2458578 --- /dev/null +++ b/projects/privacy-guard/examples/regex-engine/gateway.toml @@ -0,0 +1,11 @@ +# OpenShell gateway configuration for the RegexEngine example. +# Generate gateway.local.toml with the host address reachable by supervisors. + +[openshell] +version = 1 + +[[openshell.supervisor.middleware]] +name = "privacy-guard-regex" +grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051" +max_body_bytes = 4194304 +timeout = "5s" diff --git a/projects/privacy-guard/examples/regex-engine/policy.yaml b/projects/privacy-guard/examples/regex-engine/policy.yaml new file mode 100644 index 0000000..e69cd9d --- /dev/null +++ b/projects/privacy-guard/examples/regex-engine/policy.yaml @@ -0,0 +1,55 @@ +version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + claude_code: + name: Claude Code subscription access + endpoints: + - host: api.anthropic.com + port: 443 + protocol: rest + enforcement: enforce + access: full + - host: platform.claude.com + port: 443 + protocol: rest + enforcement: enforce + access: full + - host: claude.ai + port: 443 + - { host: statsig.anthropic.com, port: 443 } + - { host: sentry.io, port: 443 } + binaries: + - { path: /usr/local/bin/claude } + - { path: /usr/bin/node } + +network_middlewares: + privacy_guard_replace: + name: Replace email addresses and customer IDs + middleware: privacy-guard-regex + order: 0 + config: + entity_processing: + stages: + - name: identifiers + config: + engine: regex + pattern_catalog: patterns.yaml + replacement: + strategy: template + template: "[{entity}]" + on_detection: + action: replace + on_error: fail_closed + endpoints: + include: + - api.anthropic.com diff --git a/projects/privacy-guard/examples/regex-engine/privacy-guard-config.yaml b/projects/privacy-guard/examples/regex-engine/privacy-guard-config.yaml index 70c8434..219c5fb 100644 --- a/projects/privacy-guard/examples/regex-engine/privacy-guard-config.yaml +++ b/projects/privacy-guard/examples/regex-engine/privacy-guard-config.yaml @@ -3,18 +3,7 @@ entity_processing: - name: identifiers config: engine: regex - pattern_catalog: - entities: - - name: email - patterns: - - name: conventional-email - pattern: '(?=1.81.1,<2", "protobuf>=6.33.5,<7", "pydantic>=2.11,<3", + "pyyaml>=6,<7", "regex>=2026.7.19,<2027", "typer>=0.16,<1", "typing-extensions>=4.12,<5", diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index d89f3df..5c4fa90 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -33,6 +33,8 @@ MAX_REGEX_ENTITIES_PER_CATALOG = 2_000 MAX_REGEX_PATTERNS_PER_CATALOG = 10_000 MAX_REGEX_PATTERN_BYTES = 16 * 1024 +MAX_REGEX_CATALOG_FILE_BYTES = 16 * 1024 * 1024 +MAX_REGEX_CATALOG_PATH_BYTES = 1024 MAX_MATCHES_PER_PATTERN = 256 DEFAULT_TIMEOUT_SECONDS = 1.0 MAX_TIMEOUT_SECONDS = 30.0 diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index f89c28b..b9d42da 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -4,11 +4,19 @@ from collections.abc import Mapping from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from stat import S_ISREG from string import Formatter from typing import Literal, Protocol, Self import regex +import yaml from pydantic import Field, field_validator, model_validator +from yaml.constructor import ConstructorError +from yaml.events import AliasEvent +from yaml.nodes import MappingNode +from yaml.resolver import BaseResolver from privacy_guard.base import StrictDomainModel from privacy_guard.constants import ( @@ -17,6 +25,8 @@ MAX_DETECTIONS_PER_STAGE, MAX_DIAGNOSTIC_TEXT_BYTES, MAX_MATCHES_PER_PATTERN, + MAX_REGEX_CATALOG_FILE_BYTES, + MAX_REGEX_CATALOG_PATH_BYTES, MAX_REGEX_ENTITIES_PER_CATALOG, MAX_REGEX_NAME_BYTES, MAX_REGEX_PATTERN_BYTES, @@ -153,9 +163,29 @@ class RegexEngineConfig(EngineConfig): """Exact policy configuration owned by ``RegexEngine``.""" engine: Literal["regex"] = "regex" - pattern_catalog: RegexPatternCatalog = Field(repr=False) + pattern_catalog: RegexPatternCatalog = Field( + repr=False, + description=( + "Complete structured catalog or relative path to a complete YAML catalog." + ), + ) replacement: RegexReplacement | None = None + @field_validator( + "pattern_catalog", + mode="before", + json_schema_input_type=RegexPatternCatalog | str, + ) + @classmethod + def _load_pattern_catalog( + cls, + value: object, + ) -> object: + del cls + if isinstance(value, str): + return _load_pattern_catalog_file(value) + return value + @model_validator(mode="after") def _patterns_are_valid(self) -> Self: try: @@ -317,6 +347,128 @@ def search( ... +class _StrictCatalogLoader(yaml.SafeLoader): + """Safe YAML loader that rejects aliases and duplicate mapping keys.""" + + def compose_node( + self, + parent: object, + index: object, + ) -> yaml.Node: + if self.check_event(AliasEvent): + raise ConstructorError( + None, + None, + "YAML aliases are not supported", + self.peek_event().start_mark, + ) + return super().compose_node(parent, index) + + +def _construct_unique_mapping( + loader: _StrictCatalogLoader, + node: MappingNode, + deep: bool = False, +) -> dict[object, object]: + loader.flatten_mapping(node) + mapping: dict[object, object] = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + try: + duplicate = key in mapping + except TypeError: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + "found an unhashable key", + key_node.start_mark, + ) from None + if duplicate: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + "found a duplicate key", + key_node.start_mark, + ) + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +_StrictCatalogLoader.add_constructor( + BaseResolver.DEFAULT_MAPPING_TAG, + _construct_unique_mapping, +) + + +def _load_pattern_catalog_file(value: str) -> RegexPatternCatalog: + try: + path_text = validate_scalar_string(value) + if ( + not path_text + or len(path_text.encode("utf-8")) > MAX_REGEX_CATALOG_PATH_BYTES + ): + raise ValueError + relative_path = Path(path_text) + if ( + relative_path.is_absolute() + or relative_path.suffix.lower() not in {".yaml", ".yml"} + or ".." in relative_path.parts + ): + raise ValueError + + catalog_root = Path.cwd().resolve(strict=True) + catalog_path = (catalog_root / relative_path).resolve(strict=True) + catalog_path.relative_to(catalog_root) + current_path = catalog_root + for part in relative_path.parts: + current_path /= part + if current_path.is_symlink(): + raise ValueError + + metadata = catalog_path.stat() + if ( + not S_ISREG(metadata.st_mode) + or metadata.st_size > MAX_REGEX_CATALOG_FILE_BYTES + ): + raise ValueError + return _read_pattern_catalog_file( + str(catalog_path), + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + except ( + OSError, + RecursionError, + UnicodeError, + ValueError, + yaml.YAMLError, + ): + raise ValueError("regex pattern catalog file is invalid") from None + + +@lru_cache(maxsize=64) +def _read_pattern_catalog_file( + path: str, + device: int, + inode: int, + size: int, + modified_at_ns: int, + changed_at_ns: int, +) -> RegexPatternCatalog: + del device, inode, modified_at_ns, changed_at_ns + contents = Path(path).read_bytes() + if len(contents) != size or len(contents) > MAX_REGEX_CATALOG_FILE_BYTES: + raise ValueError + values = yaml.load( + contents.decode("utf-8", errors="strict"), + Loader=_StrictCatalogLoader, + ) + return RegexPatternCatalog.model_validate(values) + + def _validate_name(value: str) -> str: if ( not isinstance(value, str) diff --git a/projects/privacy-guard/tests/examples/test_regex_engine.py b/projects/privacy-guard/tests/examples/test_regex_engine.py new file mode 100644 index 0000000..33e3ec8 --- /dev/null +++ b/projects/privacy-guard/tests/examples/test_regex_engine.py @@ -0,0 +1,116 @@ +"""End-to-end checks for the built-in RegexEngine example.""" + +from __future__ import annotations + +import asyncio +import json +import subprocess +import sys +import tomllib +from pathlib import Path + +import pytest +import yaml +from google.protobuf import json_format + +from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 +from privacy_guard.engines import RegexPatternCatalog +from privacy_guard.service.server import create_builtin_registry +from privacy_guard.service.servicer import PrivacyGuardMiddleware + +EXAMPLE_DIRECTORY = Path(__file__).parents[2] / "examples" / "regex-engine" + + +def test_regex_example_runs_through_the_middleware_boundary( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(EXAMPLE_DIRECTORY) + values = yaml.safe_load( + (EXAMPLE_DIRECTORY / "privacy-guard-config.yaml").read_text() + ) + assert isinstance(values, dict) + config = pb2.HttpRequestEvaluation().config + json_format.ParseDict(values, config) + + async def evaluate() -> None: + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + try: + result = await middleware._evaluate_http_request( + pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + config=config, + body=(b"Contact user@example.com about customer CUST-12345678."), + ) + ) + finally: + await middleware.close() + + assert result.decision == pb2.DECISION_ALLOW + assert result.has_body is True + assert result.body == (b"Contact [email] about customer [customer-id].") + assert {finding.label for finding in result.findings} == { + "email (identifiers)", + "customer-id (identifiers)", + } + + asyncio.run(evaluate()) + + +def test_builtin_registry_drives_documented_cli_discovery_and_schema() -> None: + command = str(Path(sys.executable).with_name("privacy-guard")) + engines = subprocess.run( + [command, "engines"], + cwd=EXAMPLE_DIRECTORY, + check=True, + capture_output=True, + text=True, + ) + schema = subprocess.run( + [command, "schema"], + cwd=EXAMPLE_DIRECTORY, + check=True, + capture_output=True, + text=True, + ) + + assert engines.stdout.startswith("regex\tdetect,replace\t") + serialized_schema = json.loads(schema.stdout) + assert "RegexEngineConfig" in serialized_schema["$defs"] + assert "RegexPatternCatalog" in serialized_schema["$defs"] + assert "RegexReplacement" in serialized_schema["$defs"] + + +def test_regex_walkthrough_uses_current_policy_and_gateway_schema() -> None: + policy = yaml.safe_load((EXAMPLE_DIRECTORY / "policy.yaml").read_text()) + config = yaml.safe_load( + (EXAMPLE_DIRECTORY / "privacy-guard-config.yaml").read_text() + ) + catalog = yaml.safe_load((EXAMPLE_DIRECTORY / "patterns.yaml").read_text()) + gateway = tomllib.loads((EXAMPLE_DIRECTORY / "gateway.toml").read_text()) + readme = (EXAMPLE_DIRECTORY / "README.md").read_text() + + assert isinstance(policy, dict) + assert isinstance(config, dict) + assert isinstance(catalog, dict) + middleware = gateway["openshell"]["supervisor"]["middleware"] + assert middleware == [ + { + "name": "privacy-guard-regex", + "grpc_endpoint": "http://REPLACE_WITH_HOST_IP:50051", + "max_body_bytes": 4_194_304, + "timeout": "5s", + } + ] + middleware_config = policy["network_middlewares"]["privacy_guard_replace"] + assert middleware_config["middleware"] == "privacy-guard-regex" + assert middleware_config["config"] == config + assert config["on_detection"]["action"] == "replace" + stage_config = config["entity_processing"]["stages"][0]["config"] + assert stage_config["engine"] == "regex" + assert stage_config["pattern_catalog"] == "patterns.yaml" + RegexPatternCatalog.model_validate(catalog) + assert "uv run privacy-guard serve --listen 0.0.0.0:50051" in readme + assert "openshell gateway select openshell" in readme + assert "openshell gateway add" not in readme + assert "OpenShell `v0.0.90`" in readme + assert "transformed:true" in readme diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py index 334607e..74430ab 100644 --- a/projects/privacy-guard/tests/test_config.py +++ b/projects/privacy-guard/tests/test_config.py @@ -2,10 +2,13 @@ from collections.abc import Callable from copy import deepcopy +from pathlib import Path import pytest +import yaml from pydantic import ValidationError +import privacy_guard.engines.regex as regex_module from privacy_guard.config import ( PolicyAction, configuration_fingerprint, @@ -136,11 +139,182 @@ def _required_dict(mapping: object, key: str): return value -def test_runtime_config_rejects_a_catalog_file_path() -> None: +def test_catalog_file_and_inline_catalog_produce_the_same_config( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = _registry() + inline_values = _config() + file_values = deepcopy(inline_values) + inline_catalog = file_values["entity_processing"]["stages"][0]["config"][ + "pattern_catalog" + ] + (tmp_path / "patterns.yaml").write_text( + yaml.safe_dump(inline_catalog), + encoding="utf-8", + ) + file_values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( + "patterns.yaml" + ) + monkeypatch.chdir(tmp_path) + + inline_config = registry.validate_config(inline_values) + file_config = registry.validate_config(file_values) + + assert file_config == inline_config + assert configuration_fingerprint(file_config) == configuration_fingerprint( + inline_config + ) + serialized_catalog = file_config.model_dump(mode="json")["entity_processing"][ + "stages" + ][0]["config"]["pattern_catalog"] + inline_serialized_catalog = inline_config.model_dump(mode="json")[ + "entity_processing" + ]["stages"][0]["config"]["pattern_catalog"] + assert serialized_catalog == inline_serialized_catalog + assert isinstance(serialized_catalog, dict) + + +def test_catalog_file_change_produces_a_new_fingerprint( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + catalog_path = tmp_path / "patterns.yaml" + values = _config() + catalog = values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] + catalog_path.write_text(yaml.safe_dump(catalog), encoding="utf-8") + values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( + "patterns.yaml" + ) + monkeypatch.chdir(tmp_path) + registry = _registry() + first = registry.validate_config(values) + + catalog["entities"][0]["patterns"][0]["confidence"] = "low" + catalog_path.write_text(yaml.safe_dump(catalog), encoding="utf-8") + second = registry.validate_config(values) + + assert configuration_fingerprint(first) != configuration_fingerprint(second) + + +@pytest.mark.parametrize( + "catalog_path", + [ + "missing.yaml", + "../patterns.yaml", + "patterns.json", + ], +) +def test_catalog_file_rejects_invalid_paths( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + catalog_path: str, +) -> None: + values = _config() + values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = catalog_path + monkeypatch.chdir(tmp_path) + + with pytest.raises(PrivacyGuardError) as exception_info: + _registry().validate_config(values) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + + +def test_catalog_file_rejects_absolute_paths( + tmp_path: Path, +) -> None: + values = _config() + values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = str( + tmp_path / "patterns.yaml" + ) + + with pytest.raises(PrivacyGuardError) as exception_info: + _registry().validate_config(values) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + + +def test_catalog_file_rejects_symlinks( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + target = tmp_path / "target.yaml" + target.write_text("entities: []\n", encoding="utf-8") + (tmp_path / "patterns.yaml").symlink_to(target) + values = _config() + values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( + "patterns.yaml" + ) + monkeypatch.chdir(tmp_path) + + with pytest.raises(PrivacyGuardError) as exception_info: + _registry().validate_config(values) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + + +@pytest.mark.parametrize( + "contents", + [ + "entities:\n - name: first\n name: duplicate\n patterns: []\n", + ( + "entities:\n" + " - &shared\n" + " name: first\n" + " patterns:\n" + " - pattern: x\n" + " confidence: high\n" + " - *shared\n" + ), + "entities: !!python/object/apply:builtins.list []\n", + ], +) +def test_catalog_file_rejects_unsafe_yaml( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + contents: str, +) -> None: + (tmp_path / "patterns.yaml").write_text(contents, encoding="utf-8") + values = _config() + values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( + "patterns.yaml" + ) + monkeypatch.chdir(tmp_path) + + with pytest.raises(PrivacyGuardError) as exception_info: + _registry().validate_config(values) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + + +def test_catalog_file_rejects_invalid_utf8( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "patterns.yaml").write_bytes(b"\xff") + values = _config() + values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( + "patterns.yaml" + ) + monkeypatch.chdir(tmp_path) + + with pytest.raises(PrivacyGuardError) as exception_info: + _registry().validate_config(values) + + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + + +def test_catalog_file_rejects_oversized_content( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(regex_module, "MAX_REGEX_CATALOG_FILE_BYTES", 1) + (tmp_path / "patterns.yaml").write_text("entities: []\n", encoding="utf-8") values = _config() values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( - "./patterns.yaml" + "patterns.yaml" ) + monkeypatch.chdir(tmp_path) with pytest.raises(PrivacyGuardError) as exception_info: _registry().validate_config(values) diff --git a/projects/privacy-guard/uv.lock b/projects/privacy-guard/uv.lock index fe2a9ae..2d17fe6 100644 --- a/projects/privacy-guard/uv.lock +++ b/projects/privacy-guard/uv.lock @@ -136,6 +136,7 @@ dependencies = [ { name = "grpcio" }, { name = "protobuf" }, { name = "pydantic" }, + { name = "pyyaml" }, { name = "regex" }, { name = "typer" }, { name = "typing-extensions" }, @@ -154,6 +155,7 @@ requires-dist = [ { name = "grpcio", specifier = ">=1.81.1,<2" }, { name = "protobuf", specifier = ">=6.33.5,<7" }, { name = "pydantic", specifier = ">=2.11,<3" }, + { name = "pyyaml", specifier = ">=6,<7" }, { name = "regex", specifier = ">=2026.7.19,<2027" }, { name = "typer", specifier = ">=0.16,<1" }, { name = "typing-extensions", specifier = ">=4.12,<5" }, @@ -337,6 +339,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + [[package]] name = "regex" version = "2026.7.19" From dad5316843855975f594e2318e9fa3cdde5e9690 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 25 Jul 2026 15:38:43 +0000 Subject: [PATCH 35/82] Harden regex catalog preparation --- .../architecture/configuration.md | 50 +++---- .../architecture/entity-processing-engines.md | 2 + .../src/privacy_guard/engines/regex.py | 130 +++++++++++++----- .../privacy-guard/tests/engines/test_regex.py | 1 + projects/privacy-guard/tests/test_config.py | 106 ++++++++++++++ 5 files changed, 229 insertions(+), 60 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/configuration.md b/docs/documentation/privacy-guard/architecture/configuration.md index df4ae0a..3a1c5bd 100644 --- a/docs/documentation/privacy-guard/architecture/configuration.md +++ b/docs/documentation/privacy-guard/architecture/configuration.md @@ -7,8 +7,8 @@ agent_markdown: true # Configuration and text boundary Privacy Guard processes the complete request body as one text string. Its -runtime configuration is self-contained structured data supplied by OpenShell -for each evaluation. +runtime configuration is supplied by OpenShell for each evaluation and +normalized into strict structured models before processing. ## Bytes and text @@ -72,44 +72,44 @@ credentials, and data-egress constraints. ## Regex catalogs -`RegexEngineConfig.pattern_catalog` is always a structured -`RegexPatternCatalog`. Privacy Guard maintains its schema and safety limits but -does not ship an authoritative pattern set. - -The repository may publish reference catalog YAML for users to copy and adapt. -Those files are examples, not presets, runtime defaults, or a second -configuration source. - -The current OpenShell policy flow does not expand: +`RegexEngineConfig.pattern_catalog` accepts either a structured +`RegexPatternCatalog` or a relative `.yaml` or `.yml` path: ```yaml -pattern_catalog: ./patterns.yaml +pattern_catalog: patterns.yaml ``` -Privacy Guard cannot resolve that path because the middleware process does not -own the policy bundle. Accepting middleware-local paths would also make policy -behavior depend on deployment filesystem state. +Paths resolve beneath Privacy Guard's working directory. Absolute paths, +traversal, and symlinks are rejected. Catalog files must be bounded UTF-8 YAML +without aliases, duplicate keys, or unsafe tags. + +File-backed and inline inputs normalize to the same `RegexPatternCatalog`. +Serialization and canonical fingerprints contain the validated structured +catalog rather than its source path. File metadata keys a bounded parser cache; +the normalized immutable catalog keys a bounded compiled-rule cache. A content +change produces a newly validated configuration, compiled rule set, and +processor fingerprint. -Transparent catalog-file support belongs in OpenShell's policy installation -flow. OpenShell would resolve the path relative to the policy bundle, validate -the referenced value, retain the expanded configuration, and send only the -self-contained mapping to Privacy Guard. Until that upstream feature exists, -catalogs must be inline in the configuration sent to the middleware. +Privacy Guard maintains the catalog schema and safety limits but does not ship +an authoritative pattern set. Repository catalogs are examples to copy and +adapt, not presets or runtime defaults. ## Current transport constraint The copied OpenShell protocol carries policy configuration in a -per-evaluation `google.protobuf.Struct` limited to 64 KiB. This bounds the -catalog size that can reach Privacy Guard today. +per-evaluation `google.protobuf.Struct` limited to 64 KiB. An inline catalog is +bounded by that transport. A file-backed configuration carries only its bounded +relative path through the protocol and loads the deployment-mounted catalog in +the middleware process. The service validates the complete configuration, computes its canonical fingerprint, and uses a bounded internal `RequestProcessor` cache. Caching avoids repeated engine initialization but does not increase the transport limit. -Supporting larger catalogs requires an upstream OpenShell contract for -preparing expanded configuration and referring to it during evaluation. -Privacy Guard must not create a private protocol fork. +A future self-contained transport for larger expanded catalogs requires an +upstream OpenShell contract for preparing configuration and referring to it +during evaluation. Privacy Guard must not create a private protocol fork. ## Configuration identity diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md index 437b07b..62b7644 100644 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -293,6 +293,8 @@ Important properties: - each backend search receives the shared remaining timeout - pattern names are optional; unnamed patterns receive deterministic diagnostic identities without changing serialized configuration +- catalogs may be supplied inline or through a bounded relative YAML path and + normalize to the same structured configuration - replacement resolves overlaps by categorical confidence, span length, offsets, entity, and pattern identity - templates allow literal text and `{entity}` only diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index b9d42da..5916199 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -2,12 +2,15 @@ from __future__ import annotations +import os +from collections import OrderedDict from collections.abc import Mapping from dataclasses import dataclass from functools import lru_cache from pathlib import Path from stat import S_ISREG from string import Formatter +from threading import RLock from typing import Literal, Protocol, Self import regex @@ -189,15 +192,7 @@ def _load_pattern_catalog( @model_validator(mode="after") def _patterns_are_valid(self) -> Self: try: - for global_index, (entity, pattern_index, pattern) in enumerate( - _iter_catalog_patterns(self.pattern_catalog) - ): - _compile_rule( - entity, - pattern, - catalog_index=global_index, - entity_pattern_index=pattern_index, - ) + _compile_pattern_catalog(self.pattern_catalog) except (RecursionError, ValueError, regex.error): raise ValueError("regex pattern catalog is invalid") from None return self @@ -229,17 +224,7 @@ def _validate_run_config( def _initialize(self) -> None: try: - self._rules = tuple( - _compile_rule( - entity, - pattern, - catalog_index=global_index, - entity_pattern_index=pattern_index, - ) - for global_index, (entity, pattern_index, pattern) in enumerate( - _iter_catalog_patterns(self.config.pattern_catalog) - ) - ) + self._rules = _compile_pattern_catalog(self.config.pattern_catalog) except (RecursionError, ValueError, regex.error): raise EngineConfigurationError( "regex engine configuration is invalid" @@ -401,6 +386,7 @@ def _construct_unique_mapping( def _load_pattern_catalog_file(value: str) -> RegexPatternCatalog: + descriptor: int | None = None try: path_text = validate_scalar_string(value) if ( @@ -416,23 +402,16 @@ def _load_pattern_catalog_file(value: str) -> RegexPatternCatalog: ): raise ValueError - catalog_root = Path.cwd().resolve(strict=True) - catalog_path = (catalog_root / relative_path).resolve(strict=True) - catalog_path.relative_to(catalog_root) - current_path = catalog_root - for part in relative_path.parts: - current_path /= part - if current_path.is_symlink(): - raise ValueError - - metadata = catalog_path.stat() + descriptor = _open_pattern_catalog_file(relative_path) + metadata = os.fstat(descriptor) if ( not S_ISREG(metadata.st_mode) or metadata.st_size > MAX_REGEX_CATALOG_FILE_BYTES ): raise ValueError return _read_pattern_catalog_file( - str(catalog_path), + descriptor, + path_text, metadata.st_dev, metadata.st_ino, metadata.st_size, @@ -447,10 +426,27 @@ def _load_pattern_catalog_file(value: str) -> RegexPatternCatalog: yaml.YAMLError, ): raise ValueError("regex pattern catalog file is invalid") from None + finally: + if descriptor is not None: + os.close(descriptor) + + +def _open_pattern_catalog_file(relative_path: Path) -> int: + directory_flags = os.O_RDONLY | os.O_CLOEXEC | os.O_DIRECTORY | os.O_NOFOLLOW + file_flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK | os.O_NOFOLLOW + directory = os.open(".", directory_flags) + try: + for part in relative_path.parts[:-1]: + child = os.open(part, directory_flags, dir_fd=directory) + os.close(directory) + directory = child + return os.open(relative_path.parts[-1], file_flags, dir_fd=directory) + finally: + os.close(directory) -@lru_cache(maxsize=64) def _read_pattern_catalog_file( + descriptor: int, path: str, device: int, inode: int, @@ -458,15 +454,56 @@ def _read_pattern_catalog_file( modified_at_ns: int, changed_at_ns: int, ) -> RegexPatternCatalog: - del device, inode, modified_at_ns, changed_at_ns - contents = Path(path).read_bytes() + cache_key = ( + path, + device, + inode, + size, + modified_at_ns, + changed_at_ns, + ) + with _PATTERN_CATALOG_CACHE_LOCK: + cached = _PATTERN_CATALOG_CACHE.get(cache_key) + if cached is not None: + _PATTERN_CATALOG_CACHE.move_to_end(cache_key) + return cached + + contents = _read_bounded_file(descriptor) if len(contents) != size or len(contents) > MAX_REGEX_CATALOG_FILE_BYTES: raise ValueError + final_metadata = os.fstat(descriptor) + if ( + final_metadata.st_dev, + final_metadata.st_ino, + final_metadata.st_size, + final_metadata.st_mtime_ns, + final_metadata.st_ctime_ns, + ) != (device, inode, size, modified_at_ns, changed_at_ns): + raise ValueError values = yaml.load( contents.decode("utf-8", errors="strict"), Loader=_StrictCatalogLoader, ) - return RegexPatternCatalog.model_validate(values) + catalog = RegexPatternCatalog.model_validate(values) + with _PATTERN_CATALOG_CACHE_LOCK: + _PATTERN_CATALOG_CACHE[cache_key] = catalog + _PATTERN_CATALOG_CACHE.move_to_end(cache_key) + while len(_PATTERN_CATALOG_CACHE) > _MAX_CACHED_PATTERN_CATALOGS: + _PATTERN_CATALOG_CACHE.popitem(last=False) + return catalog + + +def _read_bounded_file(descriptor: int) -> bytes: + contents = bytearray() + while len(contents) <= MAX_REGEX_CATALOG_FILE_BYTES: + chunk = os.read( + descriptor, + min(64 * 1024, MAX_REGEX_CATALOG_FILE_BYTES + 1 - len(contents)), + ) + if not chunk: + break + contents.extend(chunk) + return bytes(contents) def _validate_name(value: str) -> str: @@ -479,6 +516,23 @@ def _validate_name(value: str) -> str: return value +@lru_cache(maxsize=128) +def _compile_pattern_catalog( + catalog: RegexPatternCatalog, +) -> tuple[_CompiledRule, ...]: + return tuple( + _compile_rule( + entity, + pattern, + catalog_index=global_index, + entity_pattern_index=pattern_index, + ) + for global_index, (entity, pattern_index, pattern) in enumerate( + _iter_catalog_patterns(catalog) + ) + ) + + def _compile_rule( entity: RegexEntity, pattern: RegexPattern, @@ -622,6 +676,12 @@ def _rendered_template_size(template: str, entity: str) -> int: _NAME_PATTERN = regex.compile(r"[A-Za-z_][A-Za-z0-9_-]*\Z") _INLINE_FLAG_PATTERN = regex.compile(r"[A-Za-z0-9-]+(?=[:)])") _PATTERN_METADATA_KEY = "pattern" +_MAX_CACHED_PATTERN_CATALOGS = 64 +_PATTERN_CATALOG_CACHE: OrderedDict[ + tuple[str, int, int, int, int, int], + RegexPatternCatalog, +] = OrderedDict() +_PATTERN_CATALOG_CACHE_LOCK = RLock() __all__ = [ diff --git a/projects/privacy-guard/tests/engines/test_regex.py b/projects/privacy-guard/tests/engines/test_regex.py index d6866bc..1c8a1bb 100644 --- a/projects/privacy-guard/tests/engines/test_regex.py +++ b/projects/privacy-guard/tests/engines/test_regex.py @@ -242,6 +242,7 @@ def test_pattern_search_has_an_enforceable_timeout() -> None: def test_patterns_compile_during_validation_and_preparation_not_run( monkeypatch: pytest.MonkeyPatch, ) -> None: + regex_module._compile_pattern_catalog.cache_clear() compile_count = 0 original_compile = regex_module.regex.compile diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py index 74430ab..b16cc99 100644 --- a/projects/privacy-guard/tests/test_config.py +++ b/projects/privacy-guard/tests/test_config.py @@ -1,5 +1,8 @@ from __future__ import annotations +import os +import subprocess +import sys from collections.abc import Callable from copy import deepcopy from pathlib import Path @@ -197,6 +200,45 @@ def test_catalog_file_change_produces_a_new_fingerprint( assert configuration_fingerprint(first) != configuration_fingerprint(second) +def test_equivalent_catalogs_reuse_compiled_regex_rules( + monkeypatch: pytest.MonkeyPatch, +) -> None: + regex_module._compile_pattern_catalog.cache_clear() + original_compile_rule = regex_module._compile_rule + compile_calls = 0 + + def record_compile_rule( + entity: regex_module.RegexEntity, + pattern: regex_module.RegexPattern, + catalog_index: int, + entity_pattern_index: int, + ) -> regex_module._CompiledRule: + nonlocal compile_calls + compile_calls += 1 + return original_compile_rule( + entity, + pattern, + catalog_index, + entity_pattern_index, + ) + + monkeypatch.setattr(regex_module, "_compile_rule", record_compile_rule) + registry = _registry() + first = registry.validate_config(_config()) + registry.create_engine(first.entity_processing.stages[0].config) + second = registry.validate_config(deepcopy(_config())) + registry.create_engine(second.entity_processing.stages[0].config) + changed_values = deepcopy(_config()) + changed_values["entity_processing"]["stages"][0]["config"]["pattern_catalog"][ + "entities" + ][0]["patterns"][0]["pattern"] = "changed" + changed = registry.validate_config(changed_values) + registry.create_engine(changed.entity_processing.stages[0].config) + + assert compile_calls == 2 + regex_module._compile_pattern_catalog.cache_clear() + + @pytest.mark.parametrize( "catalog_path", [ @@ -253,6 +295,70 @@ def test_catalog_file_rejects_symlinks( assert exception_info.value.code is ErrorCode.CONFIG_INVALID +def test_catalog_file_rejects_a_symlink_swap_during_open( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + catalog_root = tmp_path / "catalog-root" + catalog_root.mkdir() + catalog_path = catalog_root / "patterns.yaml" + values = _config() + catalog = values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] + catalog_path.write_text(yaml.safe_dump(catalog), encoding="utf-8") + outside_path = tmp_path / "outside.yaml" + outside_path.write_text(yaml.safe_dump(catalog), encoding="utf-8") + values["entity_processing"]["stages"][0]["config"]["pattern_catalog"] = ( + "patterns.yaml" + ) + monkeypatch.chdir(catalog_root) + + original_open = os.open + swapped = False + + def swap_before_final_open( + path: str | bytes | Path, + flags: int, + mode: int = 0o777, + *, + dir_fd: int | None = None, + ) -> int: + nonlocal swapped + if path == "patterns.yaml" and dir_fd is not None and not swapped: + swapped = True + catalog_path.unlink() + catalog_path.symlink_to(outside_path) + return original_open(path, flags, mode, dir_fd=dir_fd) + + monkeypatch.setattr(regex_module.os, "open", swap_before_final_open) + + with pytest.raises(PrivacyGuardError) as exception_info: + _registry().validate_config(values) + + assert swapped is True + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + + +def test_catalog_file_rejects_a_fifo_without_blocking(tmp_path: Path) -> None: + os.mkfifo(tmp_path / "patterns.yaml") + probe = """ +from privacy_guard.engines.regex import _load_pattern_catalog_file + +try: + _load_pattern_catalog_file("patterns.yaml") +except ValueError: + pass +else: + raise AssertionError("FIFO catalog was accepted") +""" + + subprocess.run( + [sys.executable, "-c", probe], + cwd=tmp_path, + check=True, + timeout=5, + ) + + @pytest.mark.parametrize( "contents", [ From 1dcc465dc6a93d6d6dd95d5855765a5e19d785a7 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 25 Jul 2026 15:38:54 +0000 Subject: [PATCH 36/82] Bound custom engine example results --- .../examples/custom-engine/custom_engine.py | 26 ++++++++----- .../tests/examples/test_custom_engine.py | 39 +++++++++++++++++++ 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/projects/privacy-guard/examples/custom-engine/custom_engine.py b/projects/privacy-guard/examples/custom-engine/custom_engine.py index a16dcf9..6153ae5 100644 --- a/projects/privacy-guard/examples/custom-engine/custom_engine.py +++ b/projects/privacy-guard/examples/custom-engine/custom_engine.py @@ -2,12 +2,14 @@ from __future__ import annotations +from collections.abc import Iterator from dataclasses import dataclass from typing import Literal from pydantic import Field from privacy_guard.base import StrictDomainModel +from privacy_guard.constants import MAX_DETECTIONS_PER_STAGE from privacy_guard.engines import ( ConfidenceLevel, EngineConfig, @@ -19,6 +21,7 @@ EntityProcessingStrategy, TextProcessingResult, ) +from privacy_guard.errors import EngineLimitExceeded from privacy_guard.timeout import Timeout @@ -49,15 +52,14 @@ class KeywordMatch: class KeywordAnalysisTool: """Small stand-in for an operator-provided entity-analysis library or client.""" - def find_matches(self, text: str, keyword: str) -> tuple[KeywordMatch, ...]: - matches: list[KeywordMatch] = [] + def iter_matches(self, text: str, keyword: str) -> Iterator[KeywordMatch]: start = 0 while True: start = text.find(keyword, start) if start < 0: - return tuple(matches) + return end = start + len(keyword) - matches.append(KeywordMatch(start=start, end=end)) + yield KeywordMatch(start=start, end=end) start = end @@ -102,10 +104,16 @@ def _run( timeout: Timeout, ) -> TextProcessingResult: timeout.raise_if_expired() - matches = self.resources.analysis_tool.find_matches( + matches: list[KeywordMatch] = [] + for match in self.resources.analysis_tool.iter_matches( text, self.config.keyword, - ) + ): + timeout.raise_if_expired() + if len(matches) >= MAX_DETECTIONS_PER_STAGE: + raise EngineLimitExceeded("keyword detection count exceeds the limit") + matches.append(match) + bounded_matches = tuple(matches) detections = tuple( EntityDetection( entity=self.config.entity, @@ -113,9 +121,9 @@ def _run( end=match.end, confidence=ConfidenceLevel.HIGH, ) - for match in matches + for match in bounded_matches ) - if strategy is EntityProcessingStrategy.DETECT or not matches: + if strategy is EntityProcessingStrategy.DETECT or not bounded_matches: return TextProcessingResult(text=text, detections=detections) replacement = self.config.replacement @@ -123,7 +131,7 @@ def _run( raise EngineConfigurationError( "keyword replacement configuration is required" ) - output = _replace_matches(text, matches, replacement.token) + output = _replace_matches(text, bounded_matches, replacement.token) return TextProcessingResult(text=output, detections=detections) diff --git a/projects/privacy-guard/tests/examples/test_custom_engine.py b/projects/privacy-guard/tests/examples/test_custom_engine.py index 32e9c93..88e6a70 100644 --- a/projects/privacy-guard/tests/examples/test_custom_engine.py +++ b/projects/privacy-guard/tests/examples/test_custom_engine.py @@ -64,6 +64,45 @@ async def evaluate() -> None: ) +def test_custom_engine_bounds_matches_before_building_detections() -> None: + probe = r""" +from custom_engine import ( + KeywordAnalysisTool, + KeywordEngine, + KeywordEngineConfig, + KeywordEngineResources, +) +from privacy_guard.engines import EntityProcessingStrategy +from privacy_guard.errors import EngineLimitExceeded +from privacy_guard.timeout import Timeout + +engine = KeywordEngine( + KeywordEngineConfig( + entity="keyword", + keyword="x", + ), + KeywordEngineResources(analysis_tool=KeywordAnalysisTool()), +) + +try: + engine.run( + "x" * 257, + strategy=EntityProcessingStrategy.DETECT, + timeout=Timeout.from_seconds(1), + ) +except EngineLimitExceeded: + pass +else: + raise AssertionError("custom engine accepted too many detections") +""" + + subprocess.run( + [sys.executable, "-c", probe], + cwd=EXAMPLE_DIRECTORY, + check=True, + ) + + def test_custom_registry_drives_cli_discovery_and_schema() -> None: environment = os.environ.copy() python_path = str(EXAMPLE_DIRECTORY) From 2221fa6c9e28ab10f803c55a38ea142839fdad2b Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 25 Jul 2026 15:39:07 +0000 Subject: [PATCH 37/82] Separate Privacy Guard application boundaries --- .../privacy-guard/architecture/index.md | 9 +- .../architecture/service-boundary.md | 96 +++-- projects/privacy-guard/AGENTS.md | 1 + projects/privacy-guard/README.md | 34 +- projects/privacy-guard/pyproject.toml | 2 +- .../privacy-guard/src/privacy_guard/cli.py | 170 +++++++++ .../src/privacy_guard/engine_registry.py | 9 + .../src/privacy_guard/service/__init__.py | 4 +- .../src/privacy_guard/service/server.py | 243 +++--------- .../src/privacy_guard/service/servicer.py | 46 ++- .../tests/examples/test_regex_engine.py | 2 +- .../tests/service/test_server.py | 350 ++++++++---------- .../tests/service/test_servicer.py | 38 +- projects/privacy-guard/tests/test_cli.py | 163 ++++++++ .../tests/test_engine_registry.py | 17 +- 15 files changed, 728 insertions(+), 456 deletions(-) create mode 100644 projects/privacy-guard/src/privacy_guard/cli.py create mode 100644 projects/privacy-guard/tests/test_cli.py diff --git a/docs/documentation/privacy-guard/architecture/index.md b/docs/documentation/privacy-guard/architecture/index.md index a1dc274..ce0c4af 100644 --- a/docs/documentation/privacy-guard/architecture/index.md +++ b/docs/documentation/privacy-guard/architecture/index.md @@ -54,6 +54,9 @@ Source paths on these pages are relative to bounded worker scheduling, processor caching, and finding serialization. Outside generated `bindings/`, no other package imports gRPC or generated bindings. +- `cli.py` owns command parsing, registry-factory loading, engine discovery, + schema output, logging options, and the adapter that starts the programmatic + server. - `request_processor.py` runs configured stages over one text value, shares one timeout across them, aggregates detections, and applies the user-facing policy action. It does not import gRPC or implement an engine's algorithms. @@ -82,7 +85,7 @@ HttpRequestEvaluation protobuf v PrivacyGuardMiddleware validates phase and body size - validates expanded policy configuration + validates and normalizes policy configuration resolves or builds a cached RequestProcessor decodes a non-empty body as strict UTF-8 | @@ -150,8 +153,8 @@ representations. - There is no generic replacement field or replacement-strategy enum. Each engine owns any replacement settings appropriate to its underlying algorithm. -- Runtime policy models do not accept catalog paths. Transparent file expansion - requires an upstream OpenShell policy-authoring feature. +- Regex catalogs may be supplied inline or as bounded relative YAML paths + beneath Privacy Guard's working directory. ## Read next diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index 1e79268..1cd1b75 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -22,7 +22,7 @@ Privacy Guard implements the three methods currently defined by | RPC | Behavior | | --- | --- | | `Describe` | Advertises service identity, one pre-credentials HTTP binding, and the 4 MiB body limit | -| `ValidateConfig` | Purely validates expanded policy configuration and registered resources | +| `ValidateConfig` | Validates supplied policy configuration and registered resources | | `EvaluateHttpRequest` | Validates transport input, resolves a processor, processes text, and returns a decision | `Describe` advertises only @@ -47,13 +47,18 @@ validation and every request evaluation. 5. returns `valid=true`, or a content-safe reason It does not construct engines, populate the processor cache, contact model -providers, download resources, or write artifacts. - -During evaluation, the service validates the same expanded config, computes its -canonical SHA-256 fingerprint, and resolves a configured processor from a -bounded 128-entry LRU cache. A cache miss constructs engines directly from the -exact stage configs and operator-injected resources, then constructs the -`RequestProcessor`. +providers, download resources, or write artifacts. Validation runs in the +bounded worker pool so catalog parsing and engine-specific checks do not block +the async gRPC event loop. + +During evaluation, the service validates and normalizes the same config, +computes its canonical SHA-256 fingerprint, and resolves a configured processor +from a bounded 128-entry LRU cache. A cache miss constructs engines directly +from the exact stage configs and operator-injected resources, then constructs +the `RequestProcessor`. Validation, cache resolution, engine construction, +strict UTF-8 decoding, and processing all run in the bounded worker pool. +Validation still occurs for every evaluation, while equivalent normalized regex +catalogs reuse their bounded compiled-rule entry instead of recompiling. The cache is protected for concurrent access and is not correctness-relevant. Eviction or process restart simply causes reconstruction from a later @@ -65,10 +70,11 @@ For each evaluation, the service: 1. validates the pre-credentials phase 2. validates the transport body byte limit -3. validates and resolves the expanded policy config -4. allows an empty body without invoking an engine -5. decodes a non-empty body as strict UTF-8 -6. schedules `RequestProcessor.process(text)` in the bounded worker pool +3. acquires one bounded worker slot +4. validates, normalizes, and resolves the supplied policy config in that worker +5. allows an empty body without invoking an engine +6. decodes a non-empty body as strict UTF-8 +7. calls `RequestProcessor.process(text)` in the same worker Request context, target, headers, middleware name, and protobuf values remain at the service boundary. The request ID is used only for content-safe operational @@ -121,14 +127,18 @@ gRPC event loop 4-thread executor | v +configuration validation +and processor resolution + | + v RequestProcessor.process | v ordered engine pipeline ``` -This keeps synchronous engine work off the async event loop and bounds the -number of active processor runs. +This keeps synchronous configuration and engine work off the async event loop +and bounds the number of active operations. Cached processors, engine instances, and injected resources may be used by multiple worker threads. They must retain no mutable per-request state and must @@ -140,7 +150,8 @@ Cancelling an async RPC cannot stop Python code already running in its worker thread. The service shields the worker bridge and releases the semaphore slot only after that worker actually finishes. -Cancellation therefore cannot create more than four active processor runs. +Cancellation therefore cannot create more than four active synchronous +operations. An engine should pass the remaining shared timeout to any delegated API that supports bounded execution. A non-preemptible call continues to occupy its slot until it exits. @@ -167,9 +178,9 @@ A gRPC failure is distinct from a successful policy deny. OpenShell applies the middleware registration's failure behavior when an RPC fails. A policy deny is a successful RPC result that explicitly stops the request. -## Server lifecycle and discovery +## Programmatic server lifecycle -`MiddlewareServer` is the high-level API. It owns: +`PrivacyGuardServer` is the high-level programmatic API. It owns: ```text EngineRegistry @@ -179,27 +190,27 @@ EngineRegistry The built-in registry includes `RegexEngine`. Operators register custom engines and resource-backed tool integrations before registry finalization, then pass -that registry to `MiddlewareServer`. +that registry to `PrivacyGuardServer`. The registry is an explicit application-scoped dependency, not a global -singleton. `MiddlewareServer` and `PrivacyGuardMiddleware` reject unfinalized +singleton. `PrivacyGuardServer` and `PrivacyGuardMiddleware` reject unfinalized registries. A deployment creates and finalizes one registry during startup; cached processors then construct configured stage engines from that registry. Different middleware applications in the same process may intentionally use different engine inventories or runtime resources. -The CLI accepts an operator registry factory in `module:factory` form. The -factory is invoked once, must return a finalized `EngineRegistry`, and supplies -the same engine inventory to discovery, schema generation, or serving: +Synchronous applications use: -```bash -privacy-guard --registry-factory my_engines:create_registry engines -privacy-guard --registry-factory my_engines:create_registry schema -privacy-guard --registry-factory my_engines:create_registry serve +```python +from privacy_guard.engine_registry import create_builtin_registry +from privacy_guard.service import PrivacyGuardServer + +server = PrivacyGuardServer(create_builtin_registry()) +server.run("127.0.0.1:50051") ``` -The factory is trusted operator code imported into the Privacy Guard process. -It is not a policy-controlled plugin hook. +Async applications call `await server.serve(address)` instead. Both entry +points use the same server instance and lifecycle. The server: @@ -211,6 +222,35 @@ The server: A bind failure becomes the stable `server_bind_failed` error. +The server module has no command framework, module-import-string, discovery, +schema-rendering, or command-logging responsibilities. + +## Command-line application + +`privacy_guard.cli` owns the Typer application. The `--registry-factory` option +identifies a Python function in `module:function` form. At startup, the CLI +imports the module, calls the function once, and uses the returned finalized +`EngineRegistry` for the selected command: + +```bash +privacy-guard --registry-factory my_engines:create_registry engines +privacy-guard --registry-factory my_engines:create_registry schema +privacy-guard --registry-factory my_engines:create_registry serve +``` + +This is a deployment setting, not part of an OpenShell policy. The person +starting Privacy Guard chooses the function, and importing its module executes +Python code, so it must refer to code that the deployer controls. A policy +cannot select a factory or cause Privacy Guard to import a module. + +Applications that start `PrivacyGuardServer` from Python do not need this CLI +option. They create the registry normally and pass it to the server: + +```python +registry = create_registry() +PrivacyGuardServer(registry).run() +``` + The CLI exposes: ```bash diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md index c11d802..8ae31e9 100644 --- a/projects/privacy-guard/AGENTS.md +++ b/projects/privacy-guard/AGENTS.md @@ -34,6 +34,7 @@ Run focused tests while working and `make check` before handoff. - `src/privacy_guard/config.py`: policy action and ordered stage configuration - `src/privacy_guard/engine_registry.py`: registration and finalized config union - `src/privacy_guard/request_processor.py`: stage execution and policy disposition +- `src/privacy_guard/cli.py`: command parsing, discovery, schema output, and server adapter - `src/privacy_guard/base.py`: package-wide strict immutable domain-model base - `src/privacy_guard/string_validators.py`: shared string validators and field types - `src/privacy_guard/service/`: gRPC lifecycle and protobuf adapter diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 480a943..06b741d 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -83,7 +83,9 @@ change in the canonical OpenShell protocol rather than a private proto fork. detection and deterministic template replacement. It preserves numeric backreferences by wrapping each configured pattern in a non-capturing group followed by a private named marker. Pattern names are optional diagnostic -identities; `pattern` is the only field containing the regex string. +identities; `pattern` is the only field containing the regex string. Equivalent +normalized catalogs reuse a bounded compiled-rule cache across validation and +engine construction. The third-party `regex` backend provides enforceable per-search timeouts. Explicit `ignore_case`, `multiline`, `dot_all`, and `ascii` flags are supported; @@ -138,7 +140,7 @@ walkthrough. It also shows the explicit `PYTHONPATH` setup needed when the factory is a standalone local module rather than an installed package. The registry is application-scoped, not a process-global singleton. A -`MiddlewareServer` requires an explicit finalized registry. The finalized +`PrivacyGuardServer` requires an explicit finalized registry. The finalized registry builds a Pydantic discriminated union containing the exact config type of every registered engine, so `stage.config` round-trips without dropping engine-specific or replacement-variant fields. @@ -147,11 +149,35 @@ The base installation has an explicit built-in registry containing `RegexEngine`: ```python -from privacy_guard.service.server import create_builtin_registry +from privacy_guard.engine_registry import create_builtin_registry registry = create_builtin_registry() ``` +## Programmatic server + +The server is a library API independent of the command-line application: + +```python +from privacy_guard.engine_registry import create_builtin_registry +from privacy_guard.service import PrivacyGuardServer + +server = PrivacyGuardServer(create_builtin_registry()) +server.run("127.0.0.1:50051") +``` + +`run()` is the blocking synchronous entry point. Async applications can use the +same server directly: + +```python +await server.serve("127.0.0.1:50051") +``` + +Custom applications pass their own finalized registry to +`PrivacyGuardServer`. The server owns the middleware adapter, bounded gRPC +transport, worker executor, and shutdown lifecycle; it does not load Python +modules, select engines, generate schemas, or parse command-line options. + ## CLI ```bash @@ -165,6 +191,8 @@ flags. Deployment startup owns only installed engine implementations and operator resources such as model profiles, endpoints, clients, and credentials. Use `--registry-factory module:factory` for a custom engine installation. Registry factories execute operator Python code; load only trusted modules. +The `privacy_guard.cli` module owns the executable application and adapts its +`serve` command to the programmatic server API. ## Safety and limits diff --git a/projects/privacy-guard/pyproject.toml b/projects/privacy-guard/pyproject.toml index 68d277e..36050c8 100644 --- a/projects/privacy-guard/pyproject.toml +++ b/projects/privacy-guard/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ ] [project.scripts] -privacy-guard = "privacy_guard.service.server:app" +privacy-guard = "privacy_guard.cli:app" [project.urls] Repository = "https://github.com/NVIDIA/OpenShell-Research" diff --git a/projects/privacy-guard/src/privacy_guard/cli.py b/projects/privacy-guard/src/privacy_guard/cli.py new file mode 100644 index 0000000..8219c91 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/cli.py @@ -0,0 +1,170 @@ +"""Privacy Guard command-line application.""" + +from __future__ import annotations + +import importlib +import json +import logging +from dataclasses import dataclass +from typing import Annotated + +import typer + +from privacy_guard.engine_registry import EngineRegistry, create_builtin_registry +from privacy_guard.engines import EntityProcessingStrategy +from privacy_guard.service.server import DEFAULT_LISTEN_ADDRESS, PrivacyGuardServer + +app = typer.Typer( + name="privacy-guard", + help="Run Privacy Guard and inspect installed entity-processing engines.", + no_args_is_help=True, + add_completion=False, +) + + +@app.callback() +def configure_cli( + context: typer.Context, + registry_factory: Annotated[ + str | None, + typer.Option( + help=( + "Python module and callable that return a finalized engine registry, " + "formatted as module:factory." + ), + ), + ] = None, + debug: Annotated[ + bool, + typer.Option(help="Enable content-safe processing diagnostics."), + ] = False, + debug_log_content: Annotated[ + bool, + typer.Option( + help="DANGEROUS: log complete input and processed text.", + ), + ] = False, +) -> None: + """Configure the command application and its engine inventory.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + logging.getLogger("privacy_guard").setLevel( + logging.DEBUG if debug or debug_log_content else logging.INFO + ) + context.obj = _CommandOptions( + registry=_load_registry(registry_factory), + log_request_content=debug_log_content, + ) + if debug_log_content: + _LOGGER.warning( + "privacy_guard_request_content_logging_enabled " + "complete_request_text_may_contain_secrets" + ) + + +@app.command("serve") +def serve( + context: typer.Context, + listen: Annotated[ + str, + typer.Option(help="Address on which the middleware server listens."), + ] = DEFAULT_LISTEN_ADDRESS, +) -> None: + """Run the middleware server with the selected engine inventory.""" + options = _command_options(context) + PrivacyGuardServer( + options.registry, + log_request_content=options.log_request_content, + ).run(listen) + + +@app.command("schema") +def schema(context: typer.Context) -> None: + """Print the exact finalized policy JSON Schema.""" + typer.echo( + json.dumps( + _command_options(context).registry.configuration_json_schema(), + indent=2, + ensure_ascii=False, + sort_keys=True, + ) + ) + + +@app.command("engines") +def engines(context: typer.Context) -> None: + """List installed engines and every supported processing strategy.""" + for description in _command_options(context).registry.describe_engines(): + strategies = ",".join( + strategy.value + for strategy in EntityProcessingStrategy + if strategy in description.supported_strategies + ) + typer.echo(f"{description.engine}\t{strategies}\t{description.description}") + + +_LOGGER = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _CommandOptions: + registry: EngineRegistry + log_request_content: bool + + +def _load_registry(factory_reference: str | None) -> EngineRegistry: + if factory_reference is None: + return create_builtin_registry() + module_name, separator, factory_name = factory_reference.partition(":") + if not separator or not module_name or not factory_name: + raise typer.BadParameter( + "registry factory must use module:factory", + param_hint="--registry-factory", + ) + try: + module = importlib.import_module(module_name) + factory = getattr(module, factory_name) + except Exception: + raise typer.BadParameter( + "registry factory could not be loaded", + param_hint="--registry-factory", + ) from None + if not callable(factory): + raise typer.BadParameter( + "registry factory is not callable", + param_hint="--registry-factory", + ) + try: + registry = factory() + except Exception: + raise typer.BadParameter( + "registry factory failed", + param_hint="--registry-factory", + ) from None + if not isinstance(registry, EngineRegistry): + raise typer.BadParameter( + "registry factory returned an invalid object", + param_hint="--registry-factory", + ) + if not registry.is_finalized: + raise typer.BadParameter( + "registry factory returned an unfinalized registry", + param_hint="--registry-factory", + ) + return registry + + +def _command_options(context: typer.Context) -> _CommandOptions: + options = context.obj + if not isinstance(options, _CommandOptions): + raise RuntimeError("Privacy Guard command context is unavailable") + return options + + +if __name__ == "__main__": + app() + + +__all__ = ["app"] diff --git a/projects/privacy-guard/src/privacy_guard/engine_registry.py b/projects/privacy-guard/src/privacy_guard/engine_registry.py index e7ae9b0..84d964e 100644 --- a/projects/privacy-guard/src/privacy_guard/engine_registry.py +++ b/projects/privacy-guard/src/privacy_guard/engine_registry.py @@ -23,6 +23,7 @@ EngineResources, EntityProcessingEngine, EntityProcessingStrategy, + RegexEngine, ) from privacy_guard.errors import ( EngineConfigurationError, @@ -233,6 +234,13 @@ def _resolve_registration( return registration +def create_builtin_registry() -> EngineRegistry: + """Build the finalized registry shipped by the base package.""" + registry = EngineRegistry() + registry.register(RegexEngine) + return registry.finalize() + + @dataclass(frozen=True) class _Registration: engine_type: type[object] @@ -277,4 +285,5 @@ def _engine_description( __all__ = [ "EngineDescription", "EngineRegistry", + "create_builtin_registry", ] diff --git a/projects/privacy-guard/src/privacy_guard/service/__init__.py b/projects/privacy-guard/src/privacy_guard/service/__init__.py index 802d49e..d492388 100644 --- a/projects/privacy-guard/src/privacy_guard/service/__init__.py +++ b/projects/privacy-guard/src/privacy_guard/service/__init__.py @@ -1,6 +1,6 @@ """gRPC transport and servicer for the Privacy Guard middleware.""" -from privacy_guard.service.server import MiddlewareServer +from privacy_guard.service.server import PrivacyGuardServer from privacy_guard.service.servicer import PrivacyGuardMiddleware -__all__ = ["MiddlewareServer", "PrivacyGuardMiddleware"] +__all__ = ["PrivacyGuardMiddleware", "PrivacyGuardServer"] diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 09110f6..530a14c 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -1,41 +1,23 @@ -"""Loopback gRPC server and configuration-discovery CLI.""" +"""Programmatic Privacy Guard gRPC server lifecycle.""" from __future__ import annotations import asyncio -import importlib -import json import logging -from dataclasses import dataclass -from typing import Annotated import grpc -import typer from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES from privacy_guard.engine_registry import EngineRegistry -from privacy_guard.engines import EntityProcessingStrategy, RegexEngine from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.service.servicer import PrivacyGuardMiddleware -app = typer.Typer( - name="privacy-guard", - help="Run Privacy Guard or inspect its registered entity-processing engines.", - no_args_is_help=True, - add_completion=False, -) +DEFAULT_LISTEN_ADDRESS = "127.0.0.1:50051" -def create_builtin_registry() -> EngineRegistry: - """Build the finalized registry shipped by the base package.""" - registry = EngineRegistry() - registry.register(RegexEngine) - return registry.finalize() - - -class MiddlewareServer: - """High-level server owning registry, middleware, gRPC, and shutdown.""" +class PrivacyGuardServer: + """One-shot programmatic server for a finalized engine registry.""" def __init__( self, @@ -43,196 +25,63 @@ def __init__( *, log_request_content: bool = False, ) -> None: - self._servicer = PrivacyGuardMiddleware( + self._middleware = PrivacyGuardMiddleware( registry, log_request_content=log_request_content, ) - def serve(self, listen: str = "127.0.0.1:50051") -> None: - """Serve until termination through a managed synchronous entry point.""" - asyncio.run(serve(self._servicer, listen)) + def run(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: + """Run synchronously until termination.""" + try: + asyncio.run(self.serve(listen)) + except KeyboardInterrupt: + return + + async def serve(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: + """Serve asynchronously until termination, then close owned resources.""" + server = _create_grpc_server(self._middleware) + _LOGGER.info("privacy_guard_server_starting listen=%s", listen) + try: + try: + bound_port = server.add_insecure_port(listen) + except RuntimeError: + raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) from None + if bound_port == 0: + raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) + await server.start() + await server.wait_for_termination() + finally: + try: + await _stop_grpc_server(server) + finally: + await self._middleware.close() -def create_server(servicer: PrivacyGuardMiddleware) -> grpc.aio.Server: - """Build an unstarted bounded gRPC server.""" +_LOGGER = logging.getLogger(__name__) + + +def _create_grpc_server( + middleware: PrivacyGuardMiddleware, +) -> grpc.aio.Server: server = grpc.aio.server( maximum_concurrent_rpcs=MAX_CONCURRENT_RPCS, options=(("grpc.max_receive_message_length", MAX_RECEIVE_MESSAGE_BYTES),), ) - pb2_grpc.add_SupervisorMiddlewareServicer_to_server(servicer, server) + pb2_grpc.add_SupervisorMiddlewareServicer_to_server(middleware, server) return server -async def serve( - servicer: PrivacyGuardMiddleware, - listen: str = "127.0.0.1:50051", -) -> None: - """Bind, start, and serve until termination.""" - server = create_server(servicer) - try: - try: - bound_port = server.add_insecure_port(listen) - except RuntimeError: - raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) from None - if bound_port == 0: - raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) - await server.start() - await server.wait_for_termination() - finally: - await server.stop(grace=0) - await servicer.close() - - -@app.callback() -def main( - context: typer.Context, - registry_factory: Annotated[ - str | None, - typer.Option( - help=( - "Python module and callable that return a finalized engine registry, " - "formatted as module:factory." - ), - ), - ] = None, - debug: Annotated[ - bool, - typer.Option(help="Enable content-safe processing diagnostics."), - ] = False, - debug_log_content: Annotated[ - bool, - typer.Option( - help="DANGEROUS: log complete input and processed text.", - ), - ] = False, -) -> None: - """Configure Privacy Guard command logging.""" - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(name)s: %(message)s", - ) - logging.getLogger("privacy_guard").setLevel( - logging.DEBUG if debug or debug_log_content else logging.INFO - ) - context.obj = _CommandOptions( - registry=_load_registry(registry_factory), - log_request_content=debug_log_content, - ) - if debug_log_content: - _LOGGER.warning( - "privacy_guard_request_content_logging_enabled " - "complete_request_text_may_contain_secrets" - ) - - -@app.command("serve") -def run_server( - context: typer.Context, - listen: Annotated[ - str, - typer.Option(help="Address on which the middleware server listens."), - ] = "127.0.0.1:50051", -) -> None: - """Run the service; entity behavior comes from prepared policy config.""" - options = _command_options(context) - _LOGGER.info("privacy_guard_server_starting listen=%s", listen) - MiddlewareServer( - options.registry, - log_request_content=options.log_request_content, - ).serve(listen) - - -@app.command("schema") -def show_schema(context: typer.Context) -> None: - """Print the exact finalized policy JSON Schema.""" - typer.echo( - json.dumps( - _command_options(context).registry.configuration_json_schema(), - indent=2, - ensure_ascii=False, - sort_keys=True, - ) - ) - - -@app.command("engines") -def show_engines(context: typer.Context) -> None: - """List installed engines and every supported processing strategy.""" - for description in _command_options(context).registry.describe_engines(): - strategies = ",".join( - strategy.value - for strategy in EntityProcessingStrategy - if strategy in description.supported_strategies - ) - typer.echo(f"{description.engine}\t{strategies}\t{description.description}") - - -_LOGGER = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class _CommandOptions: - registry: EngineRegistry - log_request_content: bool - - -def _load_registry(factory_reference: str | None) -> EngineRegistry: - if factory_reference is None: - return create_builtin_registry() - module_name, separator, factory_name = factory_reference.partition(":") - if not separator or not module_name or not factory_name: - raise typer.BadParameter( - "registry factory must use module:factory", - param_hint="--registry-factory", - ) - try: - module = importlib.import_module(module_name) - factory = getattr(module, factory_name) - except Exception: - raise typer.BadParameter( - "registry factory could not be loaded", - param_hint="--registry-factory", - ) from None - if not callable(factory): - raise typer.BadParameter( - "registry factory is not callable", - param_hint="--registry-factory", - ) +async def _stop_grpc_server(server: grpc.aio.Server) -> None: + shutdown = asyncio.create_task(server.stop(grace=0)) try: - registry = factory() - except Exception: - raise typer.BadParameter( - "registry factory failed", - param_hint="--registry-factory", - ) from None - if not isinstance(registry, EngineRegistry): - raise typer.BadParameter( - "registry factory returned an invalid object", - param_hint="--registry-factory", - ) - if not registry.is_finalized: - raise typer.BadParameter( - "registry factory returned an unfinalized registry", - param_hint="--registry-factory", - ) - return registry - - -def _command_options(context: typer.Context) -> _CommandOptions: - options = context.obj - if not isinstance(options, _CommandOptions): - raise RuntimeError("Privacy Guard command context is unavailable") - return options - - -if __name__ == "__main__": - app() + await asyncio.shield(shutdown) + except asyncio.CancelledError: + if not shutdown.done(): + await shutdown + raise __all__ = [ - "MiddlewareServer", - "app", - "create_builtin_registry", - "create_server", - "serve", + "DEFAULT_LISTEN_ADDRESS", + "PrivacyGuardServer", ] diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index e7b2605..b9d4dc9 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -6,9 +6,10 @@ import logging import time from collections import OrderedDict +from collections.abc import Callable from concurrent.futures import Future, ThreadPoolExecutor from threading import RLock -from typing import Never, Protocol, TypedDict +from typing import Never, Protocol, TypedDict, TypeVar import grpc from google.protobuf import json_format @@ -92,7 +93,7 @@ async def ValidateConfig( ], ) -> pb2.ValidateConfigResponse: """Validate expanded configuration without preparing runtime state.""" - return self._validate_config(request) + return await self._run_in_worker(lambda: self._validate_config(request)) @override async def EvaluateHttpRequest( @@ -184,27 +185,43 @@ async def _evaluate_http_request( raise PrivacyGuardError(ErrorCode.REQUEST_PHASE_INVALID) if len(request.body) > MAX_BODY_BYTES: raise PrivacyGuardError(ErrorCode.REQUEST_BODY_TOO_LARGE) - processor = self._processors.resolve(_mapping_from_proto(request.config)) - if not request.body: + values = _mapping_from_proto(request.config) + result = await self._run_in_worker( + lambda: self._prepare_and_process(values, request.body) + ) + if result is None: return pb2.HttpRequestResult(decision=pb2.DECISION_ALLOW) + + return _result_to_proto(result) + + def _prepare_and_process( + self, + values: object, + body: bytes, + ) -> RequestProcessingResult | None: + processor = self._processors.resolve(values) + if not body: + return None try: - text = request.body.decode("utf-8", errors="strict") + text = body.decode("utf-8", errors="strict") except UnicodeDecodeError: raise PrivacyGuardError(ErrorCode.BODY_ENCODING_INVALID) from None + return processor.process(text) + async def _run_in_worker( + self, + operation: Callable[[], _WorkerResultT], + ) -> _WorkerResultT: + """Run one bounded synchronous operation without blocking the event loop.""" await self._processing_slots.acquire() try: - worker = self._processing_executor.submit( - processor.process, - text, - ) + worker = self._processing_executor.submit(operation) future = asyncio.create_task(_await_worker(worker)) except BaseException: self._processing_slots.release() raise future.add_done_callback(lambda _: self._processing_slots.release()) - result = await asyncio.shield(future) - return _result_to_proto(result) + return await asyncio.shield(future) class _RequestProcessorCache: @@ -263,9 +280,10 @@ def _build_processor( ) -async def _await_worker( - worker: Future[RequestProcessingResult], -) -> RequestProcessingResult: +_WorkerResultT = TypeVar("_WorkerResultT") + + +async def _await_worker(worker: Future[_WorkerResultT]) -> _WorkerResultT: """Bridge a worker without relying on broken cross-thread loop wakeups.""" while not worker.done(): await asyncio.sleep(0.001) diff --git a/projects/privacy-guard/tests/examples/test_regex_engine.py b/projects/privacy-guard/tests/examples/test_regex_engine.py index 33e3ec8..4369338 100644 --- a/projects/privacy-guard/tests/examples/test_regex_engine.py +++ b/projects/privacy-guard/tests/examples/test_regex_engine.py @@ -14,8 +14,8 @@ from google.protobuf import json_format from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 +from privacy_guard.engine_registry import create_builtin_registry from privacy_guard.engines import RegexPatternCatalog -from privacy_guard.service.server import create_builtin_registry from privacy_guard.service.servicer import PrivacyGuardMiddleware EXAMPLE_DIRECTORY = Path(__file__).parents[2] / "examples" / "regex-engine" diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 658f9db..9b4a047 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -1,35 +1,23 @@ -"""Registry-based server lifecycle and discovery CLI tests.""" +"""Programmatic Privacy Guard server lifecycle tests.""" from __future__ import annotations import asyncio -import json -import logging -import re -from types import SimpleNamespace +import subprocess +import sys import grpc import pytest -from typer.testing import CliRunner, Result from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES -from privacy_guard.engine_registry import EngineRegistry -from privacy_guard.engines import EntityProcessingStrategy +from privacy_guard.engine_registry import EngineRegistry, create_builtin_registry from privacy_guard.errors import EngineRegistryError, ErrorCode, PrivacyGuardError from privacy_guard.service import server as server_module -from privacy_guard.service.server import ( - MiddlewareServer, - app, - create_builtin_registry, - create_server, - serve, -) +from privacy_guard.service.server import PrivacyGuardServer from privacy_guard.service.servicer import PrivacyGuardMiddleware -_ANSI_STYLE_PATTERN = re.compile(r"\x1b\[[0-9;]*m") - -class LifecycleServerFake: +class _LifecycleServerFake: """Minimal async-server fake for lifecycle-only tests.""" def __init__( @@ -38,14 +26,21 @@ def __init__( bound_port: int = 50051, bind_error: RuntimeError | None = None, start_error: RuntimeError | None = None, + wait_error: BaseException | None = None, + block_stop: bool = False, ) -> None: self.bound_port = bound_port self.bind_error = bind_error self.start_error = start_error + self.wait_error = wait_error self.addresses: list[str] = [] self.started = False self.waited = False self.stop_graces: list[float | None] = [] + self.stop_started = asyncio.Event() + self.stop_release = asyncio.Event() + if not block_stop: + self.stop_release.set() def add_insecure_port(self, address: str) -> int: self.addresses.append(address) @@ -60,62 +55,81 @@ async def start(self) -> None: async def wait_for_termination(self, timeout: float | None = None) -> bool: del timeout + if self.wait_error is not None: + raise self.wait_error self.waited = True return True async def stop(self, grace: float | None) -> None: self.stop_graces.append(grace) + self.stop_started.set() + await self.stop_release.wait() -def _plain_output(result: Result) -> str: - return _ANSI_STYLE_PATTERN.sub("", result.output) +def test_programmatic_server_runs_with_injected_registry_and_default_address( + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = create_builtin_registry() + served: list[tuple[PrivacyGuardServer, str]] = [] + async def record_serve(self: PrivacyGuardServer, listen: str) -> None: + served.append((self, listen)) + await self._middleware.close() -def _middleware() -> PrivacyGuardMiddleware: - return PrivacyGuardMiddleware(create_builtin_registry()) + monkeypatch.setattr(PrivacyGuardServer, "serve", record_serve) + server = PrivacyGuardServer(registry=registry, log_request_content=True) + server.run() -def test_builtin_registry_contains_the_builtin_regex_engine() -> None: - registry = create_builtin_registry() + assert served == [(server, "127.0.0.1:50051")] + assert server._middleware._registry is registry + assert server._middleware._processors._log_request_content is True - assert registry.is_finalized is True - assert registry.engine_names == ("regex",) - description = registry.describe_engines()[0] - assert description.engine == "regex" - assert description.supported_strategies == frozenset( - { - EntityProcessingStrategy.DETECT, - EntityProcessingStrategy.REPLACE, - } - ) + +def test_programmatic_server_requires_an_explicit_finalized_registry() -> None: + with pytest.raises(EngineRegistryError, match="finalized"): + PrivacyGuardServer(EngineRegistry()) -def test_middleware_server_uses_an_injected_registry_and_default_address( +def test_synchronous_server_exits_cleanly_after_keyboard_interrupt( monkeypatch: pytest.MonkeyPatch, ) -> None: - registry = create_builtin_registry() - served: list[tuple[PrivacyGuardMiddleware, str]] = [] + server = PrivacyGuardServer(create_builtin_registry()) - async def record_serve(servicer: PrivacyGuardMiddleware, listen: str) -> None: - served.append((servicer, listen)) - await servicer.close() + async def interrupt(self: PrivacyGuardServer, listen: str) -> None: + del self, listen + raise KeyboardInterrupt - monkeypatch.setattr(server_module, "serve", record_serve) + monkeypatch.setattr(PrivacyGuardServer, "serve", interrupt) - MiddlewareServer(registry=registry, log_request_content=True).serve() + server.run() + asyncio.run(server._middleware.close()) - assert len(served) == 1 - assert served[0][1] == "127.0.0.1:50051" - assert served[0][0]._registry is registry - assert served[0][0]._processors._log_request_content is True +def test_programmatic_server_import_does_not_load_the_cli_framework() -> None: + probe = ( + "import sys; " + "from privacy_guard.service import PrivacyGuardServer; " + "assert PrivacyGuardServer.__name__ == 'PrivacyGuardServer'; " + "assert 'privacy_guard.cli' not in sys.modules; " + "assert 'typer' not in sys.modules" + ) -def test_middleware_server_requires_an_explicit_finalized_registry() -> None: - with pytest.raises(EngineRegistryError, match="finalized"): - MiddlewareServer(EngineRegistry()) + subprocess.run([sys.executable, "-c", probe], check=True) + + +def test_engine_import_does_not_load_the_server_transport() -> None: + probe = ( + "import sys; " + "import privacy_guard.engines; " + "assert 'privacy_guard.service' not in sys.modules; " + "assert 'grpc' not in sys.modules" + ) + + subprocess.run([sys.executable, "-c", probe], check=True) -def test_create_server_sets_transport_limits_and_registers_servicer( +def test_server_sets_transport_limits_and_registers_middleware( monkeypatch: pytest.MonkeyPatch, ) -> None: fake_server = object() @@ -131,10 +145,10 @@ def fake_server_factory( return fake_server def record_registration( - servicer: PrivacyGuardMiddleware, + middleware: PrivacyGuardMiddleware, server: object, ) -> None: - registrations.append((servicer, server)) + registrations.append((middleware, server)) middleware = _middleware() monkeypatch.setattr(grpc.aio, "server", fake_server_factory) @@ -144,7 +158,7 @@ def record_registration( record_registration, ) try: - result = create_server(middleware) + result = server_module._create_grpc_server(middleware) finally: asyncio.run(middleware.close()) @@ -158,142 +172,13 @@ def record_registration( assert registrations == [(middleware, fake_server)] -def test_cli_help_exposes_only_pipeline_server_and_discovery_commands() -> None: - result = CliRunner().invoke(app, ["--help"]) - - assert result.exit_code == 0 - output = _plain_output(result) - assert "serve" in output - assert "schema" in output - assert "engines" in output - assert "--debug" in output - assert "--debug-log-content" in output - assert "--registry-factory" in output - assert "--config" not in output - assert "--profile" not in output - assert "--scanner-name" not in output - - -def test_cli_engines_describes_the_installed_engine() -> None: - result = CliRunner().invoke(app, ["engines"]) - - assert result.exit_code == 0 - assert result.output.startswith("regex\tdetect,replace\t") - assert "Detect overlapping regex matches" in result.output - - -def test_cli_schema_prints_the_finalized_discriminated_policy_schema() -> None: - result = CliRunner().invoke(app, ["schema"]) - - assert result.exit_code == 0 - schema = json.loads(result.output) - serialized = json.dumps(schema, sort_keys=True) - assert '"propertyName": "engine"' in serialized - assert '"regex"' in serialized - assert '"on_detection"' in serialized - - -def test_cli_loads_one_finalized_operator_registry( - monkeypatch: pytest.MonkeyPatch, -) -> None: - registry = create_builtin_registry() - factory_calls = 0 - - def create_registry() -> EngineRegistry: - nonlocal factory_calls - factory_calls += 1 - return registry - - monkeypatch.setattr( - server_module.importlib, - "import_module", - lambda module_name: ( - SimpleNamespace(create_registry=create_registry) - if module_name == "operator_engines" - else None - ), - ) - - result = CliRunner().invoke( - app, - ["--registry-factory", "operator_engines:create_registry", "engines"], - ) - - assert result.exit_code == 0 - assert factory_calls == 1 - assert result.output.startswith("regex\tdetect,replace\t") - - -@pytest.mark.parametrize( - ("factory_reference", "reason"), - [ - ("missing-separator", "module:factory"), - ("operator_engines:missing", "could not be loaded"), - ("operator_engines:not_callable", "not callable"), - ("operator_engines:failed", "factory failed"), - ("operator_engines:wrong_type", "invalid"), - ("operator_engines:unfinished", "unfinalized"), - ], -) -def test_cli_rejects_invalid_registry_factories( - monkeypatch: pytest.MonkeyPatch, - factory_reference: str, - reason: str, -) -> None: - def fail() -> EngineRegistry: - raise RuntimeError("sensitive factory failure") - - module = SimpleNamespace( - not_callable=object(), - failed=fail, - wrong_type=lambda: object(), - unfinished=lambda: EngineRegistry(), - ) - monkeypatch.setattr( - server_module.importlib, - "import_module", - lambda _: module, - ) - - result = CliRunner().invoke( - app, - ["--registry-factory", factory_reference, "engines"], - ) - - assert result.exit_code == 2 - assert reason in _plain_output(result) - assert "sensitive factory failure" not in _plain_output(result) - - -def test_cli_serve_forwards_operational_options_only( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - calls: list[tuple[str, bool]] = [] - - def record_serve(self: MiddlewareServer, listen: str) -> None: - calls.append((listen, self._servicer._processors._log_request_content)) - - monkeypatch.setattr(MiddlewareServer, "serve", record_serve) - - with caplog.at_level(logging.WARNING, logger="privacy_guard.service.server"): - result = CliRunner().invoke( - app, - ["--debug-log-content", "serve", "--listen", "127.0.0.1:50052"], - ) - - assert result.exit_code == 0 - assert calls == [("127.0.0.1:50052", True)] - assert "privacy_guard_request_content_logging_enabled" in caplog.text - - @pytest.mark.asyncio @pytest.mark.parametrize( ("fake_server", "sensitive_address"), [ - (LifecycleServerFake(bound_port=0), "invalid-sensitive-listen-8472"), + (_LifecycleServerFake(bound_port=0), "invalid-sensitive-listen-8472"), ( - LifecycleServerFake( + _LifecycleServerFake( bind_error=RuntimeError("invalid-sensitive-listen-9472") ), "invalid-sensitive-listen-9472", @@ -302,20 +187,20 @@ def record_serve(self: MiddlewareServer, listen: str) -> None: ) async def test_serve_sanitizes_bind_failures_and_closes_resources( monkeypatch: pytest.MonkeyPatch, - fake_server: LifecycleServerFake, + fake_server: _LifecycleServerFake, sensitive_address: str, ) -> None: closed: list[PrivacyGuardMiddleware] = [] - async def record_close(servicer: PrivacyGuardMiddleware) -> None: - closed.append(servicer) + async def record_close(middleware: PrivacyGuardMiddleware) -> None: + closed.append(middleware) - middleware = _middleware() - monkeypatch.setattr(server_module, "create_server", lambda _: fake_server) + server = PrivacyGuardServer(create_builtin_registry()) + monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) with pytest.raises(PrivacyGuardError) as captured: - await serve(middleware, sensitive_address) + await server.serve(sensitive_address) assert captured.value.code is ErrorCode.SERVER_BIND_FAILED assert captured.value.__cause__ is None @@ -323,49 +208,104 @@ async def record_close(servicer: PrivacyGuardMiddleware) -> None: assert fake_server.started is False assert fake_server.waited is False assert fake_server.stop_graces == [0] - assert closed == [middleware] + assert closed == [server._middleware] @pytest.mark.asyncio async def test_serve_starts_waits_and_closes_on_normal_termination( monkeypatch: pytest.MonkeyPatch, ) -> None: - fake_server = LifecycleServerFake() + fake_server = _LifecycleServerFake() closed: list[PrivacyGuardMiddleware] = [] - async def record_close(servicer: PrivacyGuardMiddleware) -> None: - closed.append(servicer) + async def record_close(middleware: PrivacyGuardMiddleware) -> None: + closed.append(middleware) - middleware = _middleware() - monkeypatch.setattr(server_module, "create_server", lambda _: fake_server) + server = PrivacyGuardServer(create_builtin_registry()) + monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - await serve(middleware, "127.0.0.1:50053") + await server.serve("127.0.0.1:50053") assert fake_server.addresses == ["127.0.0.1:50053"] assert fake_server.started is True assert fake_server.waited is True assert fake_server.stop_graces == [0] - assert closed == [middleware] + assert closed == [server._middleware] + + +@pytest.mark.asyncio +async def test_serve_propagates_cancellation_after_closing_resources( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_server = _LifecycleServerFake(wait_error=asyncio.CancelledError()) + closed: list[PrivacyGuardMiddleware] = [] + + async def record_close(middleware: PrivacyGuardMiddleware) -> None: + closed.append(middleware) + + server = PrivacyGuardServer(create_builtin_registry()) + monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) + monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) + + with pytest.raises(asyncio.CancelledError): + await server.serve("127.0.0.1:50054") + + assert fake_server.started is True + assert fake_server.stop_graces == [0] + assert closed == [server._middleware] + + +@pytest.mark.asyncio +async def test_serve_preserves_cancellation_during_server_shutdown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_server = _LifecycleServerFake(block_stop=True) + closed: list[PrivacyGuardMiddleware] = [] + + async def record_close(middleware: PrivacyGuardMiddleware) -> None: + closed.append(middleware) + + server = PrivacyGuardServer(create_builtin_registry()) + monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) + monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) + + serving = asyncio.create_task(server.serve("127.0.0.1:50055")) + await fake_server.stop_started.wait() + serving.cancel() + await asyncio.sleep(0) + + assert serving.done() is False + + fake_server.stop_release.set() + with pytest.raises(asyncio.CancelledError): + await serving + + assert fake_server.stop_graces == [0] + assert closed == [server._middleware] @pytest.mark.asyncio async def test_serve_closes_resources_when_startup_fails( monkeypatch: pytest.MonkeyPatch, ) -> None: - fake_server = LifecycleServerFake(start_error=RuntimeError("startup failed")) + fake_server = _LifecycleServerFake(start_error=RuntimeError("startup failed")) closed: list[PrivacyGuardMiddleware] = [] - async def record_close(servicer: PrivacyGuardMiddleware) -> None: - closed.append(servicer) + async def record_close(middleware: PrivacyGuardMiddleware) -> None: + closed.append(middleware) - middleware = _middleware() - monkeypatch.setattr(server_module, "create_server", lambda _: fake_server) + server = PrivacyGuardServer(create_builtin_registry()) + monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) with pytest.raises(RuntimeError, match="startup failed"): - await serve(middleware, "127.0.0.1:50054") + await server.serve("127.0.0.1:50056") assert fake_server.waited is False assert fake_server.stop_graces == [0] - assert closed == [middleware] + assert closed == [server._middleware] + + +def _middleware() -> PrivacyGuardMiddleware: + return PrivacyGuardMiddleware(create_builtin_registry()) diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index 7ccb298..4a86ea8 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -3,14 +3,17 @@ from __future__ import annotations import asyncio +from threading import get_ident import pytest from google.protobuf import json_format from google.protobuf.message import Message from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 +from privacy_guard.engine_registry import create_builtin_registry from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.service.server import create_builtin_registry +from privacy_guard.request_processor import RequestProcessor +from privacy_guard.service import servicer as servicer_module from privacy_guard.service.servicer import PrivacyGuardMiddleware @@ -102,6 +105,39 @@ async def evaluate() -> pb2.HttpRequestResult: assert result.findings[0].label == "email (regex[1])" +def test_evaluation_prepares_configuration_off_the_event_loop( + monkeypatch: pytest.MonkeyPatch, +) -> None: + event_loop_thread = get_ident() + preparation_threads: list[int] = [] + original_resolve = servicer_module._RequestProcessorCache.resolve + + def record_resolve( + cache: servicer_module._RequestProcessorCache, + values: object, + ) -> RequestProcessor: + preparation_threads.append(get_ident()) + return original_resolve(cache, values) + + monkeypatch.setattr( + servicer_module._RequestProcessorCache, + "resolve", + record_resolve, + ) + + async def evaluate() -> None: + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + try: + await middleware._evaluate_http_request(_request(b"email a@b.com")) + finally: + await middleware.close() + + asyncio.run(evaluate()) + + assert len(preparation_threads) == 1 + assert preparation_threads[0] != event_loop_thread + + def test_invalid_utf8_fails_before_invoking_an_engine() -> None: async def evaluate() -> None: middleware = PrivacyGuardMiddleware(create_builtin_registry()) diff --git a/projects/privacy-guard/tests/test_cli.py b/projects/privacy-guard/tests/test_cli.py new file mode 100644 index 0000000..892ba1d --- /dev/null +++ b/projects/privacy-guard/tests/test_cli.py @@ -0,0 +1,163 @@ +"""Privacy Guard command-line application tests.""" + +from __future__ import annotations + +import json +import logging +import re +from importlib.metadata import entry_points +from types import SimpleNamespace + +import pytest +from typer.testing import CliRunner, Result + +from privacy_guard import cli as cli_module +from privacy_guard.cli import app +from privacy_guard.engine_registry import EngineRegistry, create_builtin_registry +from privacy_guard.service.server import PrivacyGuardServer + + +def test_cli_help_exposes_server_and_discovery_commands() -> None: + result = CliRunner().invoke(app, ["--help"]) + + assert result.exit_code == 0 + output = _plain_output(result) + assert "serve" in output + assert "schema" in output + assert "engines" in output + assert "--debug" in output + assert "--debug-log-content" in output + assert "--registry-factory" in output + assert "--config" not in output + assert "--profile" not in output + assert "--scanner-name" not in output + + +def test_console_script_targets_the_cli_module() -> None: + console_script = next( + entry_point + for entry_point in entry_points(group="console_scripts") + if entry_point.name == "privacy-guard" + ) + + assert console_script.value == "privacy_guard.cli:app" + + +def test_cli_engines_describes_the_installed_engine() -> None: + result = CliRunner().invoke(app, ["engines"]) + + assert result.exit_code == 0 + assert result.output.startswith("regex\tdetect,replace\t") + assert "Detect overlapping regex matches" in result.output + + +def test_cli_schema_prints_the_finalized_discriminated_policy_schema() -> None: + result = CliRunner().invoke(app, ["schema"]) + + assert result.exit_code == 0 + schema = json.loads(result.output) + serialized = json.dumps(schema, sort_keys=True) + assert '"propertyName": "engine"' in serialized + assert '"regex"' in serialized + assert '"on_detection"' in serialized + + +def test_cli_loads_one_finalized_operator_registry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = create_builtin_registry() + factory_calls = 0 + + def create_registry() -> EngineRegistry: + nonlocal factory_calls + factory_calls += 1 + return registry + + monkeypatch.setattr( + cli_module.importlib, + "import_module", + lambda module_name: ( + SimpleNamespace(create_registry=create_registry) + if module_name == "operator_engines" + else None + ), + ) + + result = CliRunner().invoke( + app, + ["--registry-factory", "operator_engines:create_registry", "engines"], + ) + + assert result.exit_code == 0 + assert factory_calls == 1 + assert result.output.startswith("regex\tdetect,replace\t") + + +@pytest.mark.parametrize( + ("factory_reference", "reason"), + [ + ("missing-separator", "module:factory"), + ("operator_engines:missing", "could not be loaded"), + ("operator_engines:not_callable", "not callable"), + ("operator_engines:failed", "factory failed"), + ("operator_engines:wrong_type", "invalid"), + ("operator_engines:unfinished", "unfinalized"), + ], +) +def test_cli_rejects_invalid_registry_factories( + monkeypatch: pytest.MonkeyPatch, + factory_reference: str, + reason: str, +) -> None: + def fail() -> EngineRegistry: + raise RuntimeError("sensitive factory failure") + + module = SimpleNamespace( + not_callable=object(), + failed=fail, + wrong_type=lambda: object(), + unfinished=lambda: EngineRegistry(), + ) + monkeypatch.setattr( + cli_module.importlib, + "import_module", + lambda _: module, + ) + + result = CliRunner().invoke( + app, + ["--registry-factory", factory_reference, "engines"], + ) + + assert result.exit_code == 2 + assert reason in _plain_output(result) + assert "sensitive factory failure" not in _plain_output(result) + + +def test_cli_serve_adapts_operational_options_to_the_programmatic_server( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + calls: list[tuple[str, bool]] = [] + + def record_run(self: PrivacyGuardServer, listen: str) -> None: + calls.append((listen, self._middleware._processors._log_request_content)) + + monkeypatch.setattr(PrivacyGuardServer, "run", record_run) + + with caplog.at_level(logging.WARNING, logger="privacy_guard.cli"): + result = CliRunner().invoke( + app, + ["--debug-log-content", "serve", "--listen", "127.0.0.1:50052"], + ) + + assert result.exit_code == 0 + assert calls == [("127.0.0.1:50052", True)] + assert "privacy_guard_request_content_logging_enabled" in caplog.text + + +def _plain_output(result: Result) -> str: + return _ANSI_STYLE_PATTERN.sub("", result.output) + + +_ANSI_STYLE_PATTERN = re.compile(r"\x1b\[[0-9;]*m") diff --git a/projects/privacy-guard/tests/test_engine_registry.py b/projects/privacy-guard/tests/test_engine_registry.py index 470974d..7116673 100644 --- a/projects/privacy-guard/tests/test_engine_registry.py +++ b/projects/privacy-guard/tests/test_engine_registry.py @@ -10,7 +10,7 @@ from pydantic import field_validator from privacy_guard.base import StrictDomainModel -from privacy_guard.engine_registry import EngineRegistry +from privacy_guard.engine_registry import EngineRegistry, create_builtin_registry from privacy_guard.engines import ( EngineConfig, EngineConfigurationError, @@ -112,6 +112,21 @@ def _acme_values(*, action: str = "detect") -> dict[str, object]: } +def test_builtin_registry_contains_the_builtin_regex_engine() -> None: + registry = create_builtin_registry() + + assert registry.is_finalized is True + assert registry.engine_names == ("regex",) + description = registry.describe_engines()[0] + assert description.engine == "regex" + assert description.supported_strategies == frozenset( + { + EntityProcessingStrategy.DETECT, + EntityProcessingStrategy.REPLACE, + } + ) + + def test_custom_engine_config_joins_the_exact_discriminated_union() -> None: resources = AcmeResources(prefix="token") registry = EngineRegistry() From 633020d284f41957ad1968a7833663454ed9795c Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 25 Jul 2026 16:00:47 +0000 Subject: [PATCH 38/82] Simplify custom engine authoring --- .../architecture/entity-processing-engines.md | 88 ++++-------- .../architecture/safety-and-limits.md | 7 +- projects/privacy-guard/AGENTS.md | 12 +- projects/privacy-guard/README.md | 11 +- .../examples/custom-engine/README.md | 62 ++++---- .../examples/custom-engine/custom_engine.py | 132 ++++-------------- .../examples/custom-engine/policy.yaml | 9 +- .../custom-engine/privacy-guard-config.yaml | 5 +- .../custom-engine/privacy_guard_app.py | 15 -- .../src/privacy_guard/engines/base.py | 18 ++- .../privacy-guard/tests/engines/test_base.py | 20 +++ .../tests/examples/test_custom_engine.py | 61 ++------ .../tests/service/test_servicer.py | 34 +++++ 13 files changed, 183 insertions(+), 291 deletions(-) delete mode 100644 projects/privacy-guard/examples/custom-engine/privacy_guard_app.py diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md index 62b7644..6859bec 100644 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -132,7 +132,7 @@ the strategy, never the user-facing `PolicyAction`. | Field | Meaning | | --- | --- | | `text` | Authoritative text returned by this stage | -| `detections` | Tuple of all `EntityDetection` occurrences | +| `detections` | Bounded tuple of all `EntityDetection` occurrences | Each `EntityDetection` contains: @@ -145,7 +145,9 @@ Each `EntityDetection` contains: | `metadata` | Optional bounded engine-specific attribution retained inside the processing boundary | A detection span is non-empty and must fall within the stage input. The public -wrapper also enforces stage detection and output-size limits. +wrapper also enforces stage detection and output-size limits. Engines that +produce detections lazily should use `TextProcessingResult.from_detections()`; +it stops consuming the iterable as soon as the stage limit is exceeded. For `DETECT`, output text must exactly equal input text. For `REPLACE`, a successful return is the engine's authoritative completed result. Text may not @@ -159,56 +161,28 @@ across engines. ## Custom engine example ```python +import re from typing import Literal from pydantic import Field from privacy_guard.engines import ( EngineConfig, - EngineConfigurationError, EntityDetection, EntityProcessingEngine, EntityProcessingStrategy, TextProcessingResult, ) from privacy_guard.timeout import Timeout -from privacy_guard.base import StrictDomainModel - - -class KeywordReplacement(StrictDomainModel): - strategy: Literal["token"] = "token" - token: str = "[keyword]" class KeywordEngineConfig(EngineConfig): engine: Literal["keyword"] = "keyword" keyword: str = Field(min_length=1) - replacement: KeywordReplacement | None = None class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig]): - supported_strategies = frozenset( - { - EntityProcessingStrategy.DETECT, - EntityProcessingStrategy.REPLACE, - } - ) - - @classmethod - def _validate_run_config( - cls, - config: KeywordEngineConfig, - resources: None, - *, - strategy: EntityProcessingStrategy, - ) -> None: - if ( - strategy is EntityProcessingStrategy.REPLACE - and config.replacement is None - ): - raise EngineConfigurationError( - "keyword replacement configuration is required" - ) + supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) def _run( self, @@ -217,27 +191,18 @@ class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig]): strategy: EntityProcessingStrategy, timeout: Timeout, ) -> TextProcessingResult: - timeout.raise_if_expired() - start = text.find(self.config.keyword) - if start < 0: - return TextProcessingResult(text=text, detections=()) - - end = start + len(self.config.keyword) - detection = EntityDetection( - entity="keyword", - start=start, - end=end, - ) - if strategy is EntityProcessingStrategy.DETECT: - output = text - else: - replacement = self.config.replacement - if replacement is None: - raise EngineConfigurationError( - "keyword replacement configuration is required" + matches = re.finditer(re.escape(self.config.keyword), text) + return TextProcessingResult.from_detections( + text=text, + detections=( + EntityDetection( + entity="keyword", + start=match.start(), + end=match.end(), ) - output = text[:start] + replacement.token + text[end:] - return TextProcessingResult(text=output, detections=(detection,)) + for match in matches + ), + ) ``` Register the implementation and, when required, its resources before finalizing @@ -253,23 +218,26 @@ Finalization freezes registration and constructs the exact policy config type, JSON Schema, and engine discovery metadata. The complete example at -`projects/privacy-guard/examples/custom-engine/README.md` adapts an -operator-provided analysis tool, injects it as a typed resource, and runs its -finalized registry through discovery, schema generation, serving, and an -OpenShell policy. +`projects/privacy-guard/examples/custom-engine/README.md` keeps the engine, +configuration, and registry factory in one Python file and runs that registry +through discovery, schema generation, serving, and an OpenShell policy. ## Timeout and concurrency One `Timeout` is created for the processor run and passed through every stage. -An engine must not create a fresh per-stage duration. It should call: +The public engine wrapper checks it immediately before and after `_run()`, so +ordinary in-memory implementations do not repeat those checks. An engine must +not create a fresh per-stage duration. + +When a delegated API accepts a timeout, pass the remaining duration: ```python -timeout.raise_if_expired() remaining = timeout.remaining_seconds() ``` -When a delegated API accepts a timeout, pass the remaining duration. Operations -that cannot be interrupted must be documented and bounded independently. +Unique long-running loops may also call `timeout.raise_if_expired()`. +Operations that cannot be interrupted must be documented and bounded +independently. Engine configuration, derived state, and injected resources are shared across concurrent requests. Keep request text, detections, and counters local to diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 50716a4..d556a42 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -191,10 +191,11 @@ Custom engines should: - return the exact `TextProcessingResult` contract - keep request data local to `_run()` - keep initialized state immutable and make resources concurrency-safe -- pass the shared remaining timeout to delegated APIs +- pass the shared remaining timeout to delegated APIs when they support it - fail the stage on native partial failure -- check applicable bounds before allocating output proportional to input or - detections +- use `TextProcessingResult.from_detections()` to bound lazy detection streams +- add manual bounds only when a low-level collaborator allocates proportional + output before Privacy Guard can consume it - translate expected collaborator failures to the content-safe engine exception hierarchy - avoid logging input or caught exception text diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md index 8ae31e9..fa3d559 100644 --- a/projects/privacy-guard/AGENTS.md +++ b/projects/privacy-guard/AGENTS.md @@ -107,15 +107,15 @@ class KeywordEngine(EntityProcessingEngine[KeywordConfig]): strategy: EntityProcessingStrategy, timeout: Timeout, ) -> TextProcessingResult: - timeout.raise_if_expired() return TextProcessingResult(text=text, detections=()) ``` -The public `run` method validates extension output. Register every engine before -finalizing the registry so policy serialization retains its exact config type. -Custom deployments expose a `module:factory` callable that returns the -application-scoped finalized registry and pass it to the CLI with -`--registry-factory`. +The public `run` method validates input, strategy, timeout, and extension +output. Custom code checks the timeout itself only for delegated calls or +unique long-running loops. Register every engine before finalizing the registry +so policy serialization retains its exact config type. Custom deployments +expose a `module:factory` callable that returns the application-scoped finalized +registry and pass it to the CLI with `--registry-factory`. ## Change limits diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 06b741d..486c4b2 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -99,6 +99,9 @@ Privacy Guard owns the catalog schema but maintains no authoritative patterns. Custom engines are a first-class extension point. Authors declare one typed config, optional typed resources, `supported_strategies`, and `_run`. They do not write `__init__`; `_initialize` is optional, and `@override` is not required. +The public wrapper validates strategy support, input, timeout, spans, output +size, mutation behavior, and result cardinality. Lazy detection streams use +`TextProcessingResult.from_detections()` for bounded materialization. Resource-backed engines define an `EngineResources` subclass containing their operator-owned runtime dependencies. Resource bundles may contain initialized @@ -134,10 +137,10 @@ uv run privacy-guard --registry-factory my_engines:create_registry serve ``` The [custom engine end-to-end example](examples/custom-engine/README.md) -contains a complete tool adapter, typed policy and replacement configuration, -runtime resource registration, registry factory, OpenShell policy, and -walkthrough. It also shows the explicit `PYTHONPATH` setup needed when the -factory is a standalone local module rather than an installed package. +contains one Python file with a typed detection config, engine implementation, +and registry factory, plus its OpenShell policy and walkthrough. It also shows +the explicit `PYTHONPATH` setup needed when the factory is a standalone local +module rather than an installed package. The registry is application-scoped, not a process-global singleton. A `PrivacyGuardServer` requires an explicit finalized registry. The finalized diff --git a/projects/privacy-guard/examples/custom-engine/README.md b/projects/privacy-guard/examples/custom-engine/README.md index 9e01449..8bbe751 100644 --- a/projects/privacy-guard/examples/custom-engine/README.md +++ b/projects/privacy-guard/examples/custom-engine/README.md @@ -1,29 +1,21 @@ # Custom engine end-to-end example -This example implements `KeywordEngine`, registers an operator-owned -`KeywordAnalysisTool`, and runs the custom engine through Privacy Guard and -OpenShell. The final check sends a Claude Code request containing -`Project Cobalt` and verifies that OpenShell forwards `[confidential-project]` -instead. - -The analyzer is intentionally small. Its purpose is to show the complete -integration boundary that a production adapter for a library or service such as -NeMo Anonymizer would use: - -- `KeywordEngineConfig` and `TokenReplacement` contain policy-owned behavior. -- `KeywordAnalysisTool` is an operator-created dependency held by the - `KeywordEngineResources` bundle and never appears in policy. -- `KeywordEngine` translates tool matches into `EntityDetection` objects and - implements detection and replacement. -- `privacy_guard_app.py` is deployment-owned wiring. Its `create_registry()` - selects the installed engine, creates its runtime resource, and returns the - finalized application-scoped registry. - -An engine author implements only the types in `custom_engine.py`; registration -is not part of the engine contract. The deployment owner performs the small -amount of explicit application assembly in `privacy_guard_app.py`. Privacy Guard -needs that complete inventory at startup to build the exact Pydantic policy -union and inject operator-owned resources. +This example implements and registers `KeywordEngine` in one Python file, then +runs it through Privacy Guard and OpenShell. The final check sends a Claude Code +request containing `Project Cobalt` and verifies that Privacy Guard reports the +configured confidential-project finding. + +The implementation is intentionally compact but complete: + +- `KeywordEngineConfig` defines the policy-owned entity and keyword. +- `KeywordEngine._run()` finds every literal occurrence and returns detections. +- `TextProcessingResult.from_detections()` bounds the lazy detection stream. +- `create_registry()` registers the implementation for every CLI command. + +The base engine wrapper validates strategy support, input, timeout, spans, +output size, mutation behavior, and result cardinality. Custom engines add +their own checks only when an underlying library or service has a unique +low-level requirement. ## Prerequisites @@ -72,18 +64,18 @@ Use the same custom registry for discovery, schema generation, and serving: ```bash uv run privacy-guard \ - --registry-factory privacy_guard_app:create_registry \ + --registry-factory custom_engine:create_registry \ engines uv run privacy-guard \ - --registry-factory privacy_guard_app:create_registry \ + --registry-factory custom_engine:create_registry \ schema ``` -The first command should print one `keyword-tool` row with `detect,replace`. +The first command should print one `keyword-tool` row with `detect`. The schema should contain `KeywordEngineConfig`, including its exact `entity`, -`keyword`, and `replacement` fields. Registry factories execute operator Python -code in the Privacy Guard process; use only trusted modules. +and `keyword` fields. Registry factories execute operator Python code in the +Privacy Guard process; use only trusted modules. `privacy-guard-config.yaml` shows the standalone engine configuration. OpenShell does not load that file separately; `policy.yaml` contains the same configuration @@ -97,7 +89,7 @@ In terminal 1, enter this example directory, export `PYTHONPATH` again, and run: export PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}" uv run privacy-guard \ - --registry-factory privacy_guard_app:create_registry \ + --registry-factory custom_engine:create_registry \ serve \ --listen 0.0.0.0:50051 ``` @@ -205,8 +197,8 @@ After authenticating Claude Code, enter: Tell me something that rhymes with the confidential name Project Cobalt ``` -Privacy Guard should send `[confidential-project]` instead of `Project Cobalt` -to the provider. +This detection-only example leaves the request body unchanged and records the +finding before the provider request continues. ## Verify the middleware result @@ -217,9 +209,9 @@ inspect a finite recent log window: openshell logs privacy-guard-custom-engine -n 100 --source sandbox ``` -Look for the `api.anthropic.com/v1/messages` request with `transformed:true` and -a `confidential-project (project-names)` finding. The raw confidential value -must not appear in middleware findings. +Look for the `api.anthropic.com/v1/messages` request with `transformed:false` +and a `confidential-project (project-names)` finding. The raw confidential +value must not appear in middleware findings. ## Cleanup diff --git a/projects/privacy-guard/examples/custom-engine/custom_engine.py b/projects/privacy-guard/examples/custom-engine/custom_engine.py index 6153ae5..9e8449e 100644 --- a/projects/privacy-guard/examples/custom-engine/custom_engine.py +++ b/projects/privacy-guard/examples/custom-engine/custom_engine.py @@ -1,100 +1,37 @@ -"""Example custom entity-processing engine and application registry.""" +"""Complete custom entity-processing engine example.""" from __future__ import annotations -from collections.abc import Iterator -from dataclasses import dataclass +import re from typing import Literal from pydantic import Field -from privacy_guard.base import StrictDomainModel -from privacy_guard.constants import MAX_DETECTIONS_PER_STAGE +from privacy_guard.engine_registry import EngineRegistry from privacy_guard.engines import ( ConfidenceLevel, EngineConfig, - EngineConfigurationError, - EngineResources, EntityDetection, EntityName, EntityProcessingEngine, EntityProcessingStrategy, TextProcessingResult, ) -from privacy_guard.errors import EngineLimitExceeded from privacy_guard.timeout import Timeout -class TokenReplacement(StrictDomainModel): - """Replace every detected keyword with one configured token.""" - - strategy: Literal["token"] = "token" - token: str = Field(min_length=1, max_length=256) - - class KeywordEngineConfig(EngineConfig): - """Policy-owned behavior for the example keyword-analysis tool.""" + """Policy-owned configuration for keyword detection.""" engine: Literal["keyword-tool"] = "keyword-tool" entity: EntityName keyword: str = Field(min_length=1, max_length=256, repr=False) - replacement: TokenReplacement | None = None - - -@dataclass(frozen=True) -class KeywordMatch: - """One match returned by the example third-party-style tool.""" - - start: int - end: int - - -class KeywordAnalysisTool: - """Small stand-in for an operator-provided entity-analysis library or client.""" - def iter_matches(self, text: str, keyword: str) -> Iterator[KeywordMatch]: - start = 0 - while True: - start = text.find(keyword, start) - if start < 0: - return - end = start + len(keyword) - yield KeywordMatch(start=start, end=end) - start = end +class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig]): + """Detect every occurrence of one configured keyword.""" -@dataclass(frozen=True) -class KeywordEngineResources(EngineResources): - """Operator-owned dependencies injected into every configured engine.""" - - analysis_tool: KeywordAnalysisTool - - -class KeywordEngine( - EntityProcessingEngine[KeywordEngineConfig, KeywordEngineResources], -): - """Adapt KeywordAnalysisTool results to the Privacy Guard engine contract.""" - - supported_strategies = frozenset( - { - EntityProcessingStrategy.DETECT, - EntityProcessingStrategy.REPLACE, - } - ) - - @classmethod - def _validate_run_config( - cls, - config: KeywordEngineConfig, - resources: KeywordEngineResources, - *, - strategy: EntityProcessingStrategy, - ) -> None: - del cls, resources - if strategy is EntityProcessingStrategy.REPLACE and config.replacement is None: - raise EngineConfigurationError( - "keyword replacement configuration is required" - ) + supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) def _run( self, @@ -103,44 +40,23 @@ def _run( strategy: EntityProcessingStrategy, timeout: Timeout, ) -> TextProcessingResult: - timeout.raise_if_expired() - matches: list[KeywordMatch] = [] - for match in self.resources.analysis_tool.iter_matches( - text, - self.config.keyword, - ): - timeout.raise_if_expired() - if len(matches) >= MAX_DETECTIONS_PER_STAGE: - raise EngineLimitExceeded("keyword detection count exceeds the limit") - matches.append(match) - bounded_matches = tuple(matches) - detections = tuple( - EntityDetection( - entity=self.config.entity, - start=match.start, - end=match.end, - confidence=ConfidenceLevel.HIGH, - ) - for match in bounded_matches + matches = re.finditer(re.escape(self.config.keyword), text) + return TextProcessingResult.from_detections( + text=text, + detections=( + EntityDetection( + entity=self.config.entity, + start=match.start(), + end=match.end(), + confidence=ConfidenceLevel.HIGH, + ) + for match in matches + ), ) - if strategy is EntityProcessingStrategy.DETECT or not bounded_matches: - return TextProcessingResult(text=text, detections=detections) - - replacement = self.config.replacement - if replacement is None: - raise EngineConfigurationError( - "keyword replacement configuration is required" - ) - output = _replace_matches(text, bounded_matches, replacement.token) - return TextProcessingResult(text=output, detections=detections) -def _replace_matches( - text: str, - matches: tuple[KeywordMatch, ...], - replacement: str, -) -> str: - output = text - for match in reversed(matches): - output = output[: match.start] + replacement + output[match.end :] - return output +def create_registry() -> EngineRegistry: + """Create the application-scoped registry used by every CLI command.""" + registry = EngineRegistry() + registry.register(KeywordEngine) + return registry.finalize() diff --git a/projects/privacy-guard/examples/custom-engine/policy.yaml b/projects/privacy-guard/examples/custom-engine/policy.yaml index f3ab891..61369d0 100644 --- a/projects/privacy-guard/examples/custom-engine/policy.yaml +++ b/projects/privacy-guard/examples/custom-engine/policy.yaml @@ -33,8 +33,8 @@ network_policies: - { path: /usr/bin/node } network_middlewares: - privacy_guard_replace: - name: Replace confidential project names + privacy_guard_detect: + name: Detect confidential project names middleware: privacy-guard-custom-engine order: 0 config: @@ -45,11 +45,8 @@ network_middlewares: engine: keyword-tool entity: confidential-project keyword: Project Cobalt - replacement: - strategy: token - token: "[confidential-project]" on_detection: - action: replace + action: detect on_error: fail_closed endpoints: include: diff --git a/projects/privacy-guard/examples/custom-engine/privacy-guard-config.yaml b/projects/privacy-guard/examples/custom-engine/privacy-guard-config.yaml index af4925b..1ee1d76 100644 --- a/projects/privacy-guard/examples/custom-engine/privacy-guard-config.yaml +++ b/projects/privacy-guard/examples/custom-engine/privacy-guard-config.yaml @@ -5,8 +5,5 @@ entity_processing: engine: keyword-tool entity: confidential-project keyword: Project Cobalt - replacement: - strategy: token - token: "[confidential-project]" on_detection: - action: replace + action: detect diff --git a/projects/privacy-guard/examples/custom-engine/privacy_guard_app.py b/projects/privacy-guard/examples/custom-engine/privacy_guard_app.py deleted file mode 100644 index b17181d..0000000 --- a/projects/privacy-guard/examples/custom-engine/privacy_guard_app.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Deployment-owned assembly for the custom-engine example.""" - -from custom_engine import KeywordAnalysisTool, KeywordEngine, KeywordEngineResources - -from privacy_guard.engine_registry import EngineRegistry - - -def create_registry() -> EngineRegistry: - """Create the application-scoped engine registry used by Privacy Guard.""" - registry = EngineRegistry() - registry.register( - KeywordEngine, - resources=KeywordEngineResources(analysis_tool=KeywordAnalysisTool()), - ) - return registry.finalize() diff --git a/projects/privacy-guard/src/privacy_guard/engines/base.py b/projects/privacy-guard/src/privacy_guard/engines/base.py index 4847f71..286c5e1 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/base.py +++ b/projects/privacy-guard/src/privacy_guard/engines/base.py @@ -3,13 +3,14 @@ from __future__ import annotations from abc import ABC, abstractmethod -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from enum import StrEnum from types import MappingProxyType from typing import ( Annotated, ClassVar, Generic, + Self, TypeAlias, get_args, get_origin, @@ -124,6 +125,21 @@ def _detections_are_a_tuple(cls, value: object) -> object: raise ValueError("detections must be a tuple") return value + @classmethod + def from_detections( + cls, + *, + text: str, + detections: Iterable[EntityDetection], + ) -> Self: + """Build a result while bounding a lazily produced detection stream.""" + bounded: list[EntityDetection] = [] + for detection in detections: + if len(bounded) >= MAX_DETECTIONS_PER_STAGE: + raise EngineLimitExceeded("engine returned too many detections") + bounded.append(detection) + return cls(text=text, detections=tuple(bounded)) + class EngineConfig(StrictDomainModel): """Nominal base for an engine's exact policy configuration.""" diff --git a/projects/privacy-guard/tests/engines/test_base.py b/projects/privacy-guard/tests/engines/test_base.py index bf09912..5673923 100644 --- a/projects/privacy-guard/tests/engines/test_base.py +++ b/projects/privacy-guard/tests/engines/test_base.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Iterator from dataclasses import dataclass from time import monotonic from typing import Literal @@ -12,6 +13,7 @@ ConfidenceLevel, EngineConfig, EngineContractError, + EngineLimitExceeded, EngineResources, EntityDetection, EntityProcessingEngine, @@ -122,6 +124,24 @@ def test_detection_confidence_and_metadata_are_strict_bounded_values() -> None: ) +def test_processing_result_bounds_a_lazy_detection_stream() -> None: + produced = 0 + + def detections() -> Iterator[EntityDetection]: + nonlocal produced + for index in range(1_000): + produced += 1 + yield EntityDetection(entity="token", start=index, end=index + 1) + + with pytest.raises(EngineLimitExceeded): + TextProcessingResult.from_detections( + text="x" * 1_000, + detections=detections(), + ) + + assert produced == 257 + + class _DetectOnlyEngine(EntityProcessingEngine[_Config]): supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) diff --git a/projects/privacy-guard/tests/examples/test_custom_engine.py b/projects/privacy-guard/tests/examples/test_custom_engine.py index 88e6a70..039b1b3 100644 --- a/projects/privacy-guard/tests/examples/test_custom_engine.py +++ b/projects/privacy-guard/tests/examples/test_custom_engine.py @@ -24,7 +24,7 @@ def test_custom_engine_runs_through_the_middleware_boundary() -> None: from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.service.servicer import PrivacyGuardMiddleware -from privacy_guard_app import create_registry +from custom_engine import create_registry values = yaml.safe_load(Path("privacy-guard-config.yaml").read_text()) assert isinstance(values, dict) @@ -46,8 +46,8 @@ async def evaluate() -> None: await middleware.close() assert result.decision == pb2.DECISION_ALLOW - assert result.has_body is True - assert result.body == b"Discuss [confidential-project] safely." + assert result.has_body is False + assert result.body == b"" assert len(result.findings) == 1 assert result.findings[0].label == ( "confidential-project (project-names)" @@ -64,45 +64,6 @@ async def evaluate() -> None: ) -def test_custom_engine_bounds_matches_before_building_detections() -> None: - probe = r""" -from custom_engine import ( - KeywordAnalysisTool, - KeywordEngine, - KeywordEngineConfig, - KeywordEngineResources, -) -from privacy_guard.engines import EntityProcessingStrategy -from privacy_guard.errors import EngineLimitExceeded -from privacy_guard.timeout import Timeout - -engine = KeywordEngine( - KeywordEngineConfig( - entity="keyword", - keyword="x", - ), - KeywordEngineResources(analysis_tool=KeywordAnalysisTool()), -) - -try: - engine.run( - "x" * 257, - strategy=EntityProcessingStrategy.DETECT, - timeout=Timeout.from_seconds(1), - ) -except EngineLimitExceeded: - pass -else: - raise AssertionError("custom engine accepted too many detections") -""" - - subprocess.run( - [sys.executable, "-c", probe], - cwd=EXAMPLE_DIRECTORY, - check=True, - ) - - def test_custom_registry_drives_cli_discovery_and_schema() -> None: environment = os.environ.copy() python_path = str(EXAMPLE_DIRECTORY) @@ -113,7 +74,7 @@ def test_custom_registry_drives_cli_discovery_and_schema() -> None: command = [ str(Path(sys.executable).with_name("privacy-guard")), "--registry-factory", - "privacy_guard_app:create_registry", + "custom_engine:create_registry", ] engines = subprocess.run( @@ -133,13 +94,12 @@ def test_custom_registry_drives_cli_discovery_and_schema() -> None: env=environment, ) - assert engines.stdout.startswith("keyword-tool\tdetect,replace\t") + assert engines.stdout.startswith("keyword-tool\tdetect\t") assert "regex" not in engines.stdout serialized_schema = json.loads(schema.stdout) assert "KeywordEngineConfig" in serialized_schema["$defs"] keyword_properties = serialized_schema["$defs"]["KeywordEngineConfig"]["properties"] assert set(keyword_properties) == { - "replacement", "engine", "entity", "keyword", @@ -153,10 +113,13 @@ def test_openshell_walkthrough_uses_the_custom_registry_and_current_policy() -> ) gateway = tomllib.loads((EXAMPLE_DIRECTORY / "gateway.toml").read_text()) readme = (EXAMPLE_DIRECTORY / "README.md").read_text() + implementation = (EXAMPLE_DIRECTORY / "custom_engine.py").read_text() assert isinstance(policy, dict) assert isinstance(config, dict) - middleware_config = policy["network_middlewares"]["privacy_guard_replace"] + assert not (EXAMPLE_DIRECTORY / "privacy_guard_app.py").exists() + assert "def create_registry() -> EngineRegistry:" in implementation + middleware_config = policy["network_middlewares"]["privacy_guard_detect"] assert middleware_config["middleware"] == "privacy-guard-custom-engine" assert middleware_config["config"] == config middleware = gateway["openshell"]["supervisor"]["middleware"] @@ -170,11 +133,11 @@ def test_openshell_walkthrough_uses_the_custom_registry_and_current_policy() -> ] stage_config = config["entity_processing"]["stages"][0]["config"] assert stage_config["engine"] == "keyword-tool" - assert config["on_detection"]["action"] == "replace" - assert "--registry-factory privacy_guard_app:create_registry" in readme + assert config["on_detection"]["action"] == "detect" + assert "--registry-factory custom_engine:create_registry" in readme assert "cd projects/privacy-guard/examples/custom-engine" in readme assert 'export PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}"' in readme assert "openshell gateway select openshell" in readme assert "openshell gateway add" not in readme assert "OpenShell `v0.0.90`" in readme - assert "transformed:true" in readme + assert "transformed:false" in readme diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index 4a86ea8..1e9e07e 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -138,6 +138,40 @@ async def evaluate() -> None: assert preparation_threads[0] != event_loop_thread +def test_evaluation_revalidates_configuration_before_reusing_cached_processor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + validation_count = 0 + original_validate = servicer_module.EngineRegistry.validate_config + + def record_validation( + registry: servicer_module.EngineRegistry, + values: object, + ) -> servicer_module.FinalizedPrivacyGuardConfig: + nonlocal validation_count + validation_count += 1 + return original_validate(registry, values) + + monkeypatch.setattr( + servicer_module.EngineRegistry, + "validate_config", + record_validation, + ) + + async def evaluate_twice() -> None: + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + try: + request = _request(b"email a@b.com") + await middleware._evaluate_http_request(request) + await middleware._evaluate_http_request(request) + finally: + await middleware.close() + + asyncio.run(evaluate_twice()) + + assert validation_count == 2 + + def test_invalid_utf8_fails_before_invoking_an_engine() -> None: async def evaluate() -> None: middleware = PrivacyGuardMiddleware(create_builtin_registry()) From a516e634d5971da725791c5665be9ba739abfd62 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 25 Jul 2026 16:04:36 +0000 Subject: [PATCH 39/82] Allow custom registries to include built-ins --- .../architecture/entity-processing-engines.md | 6 +++++- projects/privacy-guard/README.md | 6 +++++- .../privacy-guard/examples/custom-engine/README.md | 12 +++++++----- .../examples/custom-engine/custom_engine.py | 4 ++-- .../src/privacy_guard/engine_registry.py | 8 ++++---- .../tests/examples/test_custom_engine.py | 6 ++++-- projects/privacy-guard/tests/test_engine_registry.py | 11 +++++++++-- 7 files changed, 36 insertions(+), 17 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md index 6859bec..e17fc88 100644 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -209,11 +209,15 @@ Register the implementation and, when required, its resources before finalizing the registry: ```python -registry = EngineRegistry() +registry = EngineRegistry(include_builtin_engines=True) registry.register(KeywordEngine) registry = registry.finalize() ``` +The opt-in constructor argument registers the engines shipped by Privacy Guard +before application-specific engines. Omit it when the registry should contain +only explicitly registered implementations. + Finalization freezes registration and constructs the exact policy config type, JSON Schema, and engine discovery metadata. diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 486c4b2..d692b0b 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -121,11 +121,15 @@ from privacy_guard.engine_registry import EngineRegistry def create_registry() -> EngineRegistry: - registry = EngineRegistry() + registry = EngineRegistry(include_builtin_engines=True) registry.register(AcmeEngine, resources=AcmeResources(client=client)) return registry.finalize() ``` +Set `include_builtin_engines=True` when a custom deployment should extend the +standard engine inventory. Leave it at the default `False` for an intentionally +isolated registry. + Pass that factory to every CLI operation so discovery, schema generation, and the running server use the same engine inventory. The module must be installed or otherwise present on Python's import path: diff --git a/projects/privacy-guard/examples/custom-engine/README.md b/projects/privacy-guard/examples/custom-engine/README.md index 8bbe751..a8b2dd8 100644 --- a/projects/privacy-guard/examples/custom-engine/README.md +++ b/projects/privacy-guard/examples/custom-engine/README.md @@ -10,7 +10,8 @@ The implementation is intentionally compact but complete: - `KeywordEngineConfig` defines the policy-owned entity and keyword. - `KeywordEngine._run()` finds every literal occurrence and returns detections. - `TextProcessingResult.from_detections()` bounds the lazy detection stream. -- `create_registry()` registers the implementation for every CLI command. +- `create_registry()` includes the built-in engines and registers the custom + implementation for every CLI command. The base engine wrapper validates strategy support, input, timeout, spans, output size, mutation behavior, and result cardinality. Custom engines add @@ -72,10 +73,11 @@ uv run privacy-guard \ schema ``` -The first command should print one `keyword-tool` row with `detect`. -The schema should contain `KeywordEngineConfig`, including its exact `entity`, -and `keyword` fields. Registry factories execute operator Python code in the -Privacy Guard process; use only trusted modules. +The first command should print the built-in `regex` row with `detect,replace` +and the custom `keyword-tool` row with `detect`. The schema should contain both +`RegexEngineConfig` and `KeywordEngineConfig`; the latter includes its exact +`entity` and `keyword` fields. Registry factories execute operator Python code +in the Privacy Guard process; use only trusted modules. `privacy-guard-config.yaml` shows the standalone engine configuration. OpenShell does not load that file separately; `policy.yaml` contains the same configuration diff --git a/projects/privacy-guard/examples/custom-engine/custom_engine.py b/projects/privacy-guard/examples/custom-engine/custom_engine.py index 9e8449e..532de02 100644 --- a/projects/privacy-guard/examples/custom-engine/custom_engine.py +++ b/projects/privacy-guard/examples/custom-engine/custom_engine.py @@ -56,7 +56,7 @@ def _run( def create_registry() -> EngineRegistry: - """Create the application-scoped registry used by every CLI command.""" - registry = EngineRegistry() + """Create a registry containing the built-in and custom engines.""" + registry = EngineRegistry(include_builtin_engines=True) registry.register(KeywordEngine) return registry.finalize() diff --git a/projects/privacy-guard/src/privacy_guard/engine_registry.py b/projects/privacy-guard/src/privacy_guard/engine_registry.py index 84d964e..14e705f 100644 --- a/projects/privacy-guard/src/privacy_guard/engine_registry.py +++ b/projects/privacy-guard/src/privacy_guard/engine_registry.py @@ -46,10 +46,12 @@ class EngineDescription: class EngineRegistry: """Register engine implementations and finalize their exact policy union.""" - def __init__(self) -> None: + def __init__(self, *, include_builtin_engines: bool = False) -> None: self._registrations: dict[str, _Registration] = {} self._config_type: FinalizedPrivacyGuardConfigType | None = None self._config_adapter: TypeAdapter[FinalizedPrivacyGuardConfig] | None = None + if include_builtin_engines: + self.register(RegexEngine) @property def is_finalized(self) -> bool: @@ -236,9 +238,7 @@ def _resolve_registration( def create_builtin_registry() -> EngineRegistry: """Build the finalized registry shipped by the base package.""" - registry = EngineRegistry() - registry.register(RegexEngine) - return registry.finalize() + return EngineRegistry(include_builtin_engines=True).finalize() @dataclass(frozen=True) diff --git a/projects/privacy-guard/tests/examples/test_custom_engine.py b/projects/privacy-guard/tests/examples/test_custom_engine.py index 039b1b3..320f89b 100644 --- a/projects/privacy-guard/tests/examples/test_custom_engine.py +++ b/projects/privacy-guard/tests/examples/test_custom_engine.py @@ -94,9 +94,10 @@ def test_custom_registry_drives_cli_discovery_and_schema() -> None: env=environment, ) - assert engines.stdout.startswith("keyword-tool\tdetect\t") - assert "regex" not in engines.stdout + assert engines.stdout.startswith("regex\tdetect,replace\t") + assert "keyword-tool\tdetect\t" in engines.stdout serialized_schema = json.loads(schema.stdout) + assert "RegexEngineConfig" in serialized_schema["$defs"] assert "KeywordEngineConfig" in serialized_schema["$defs"] keyword_properties = serialized_schema["$defs"]["KeywordEngineConfig"]["properties"] assert set(keyword_properties) == { @@ -118,6 +119,7 @@ def test_openshell_walkthrough_uses_the_custom_registry_and_current_policy() -> assert isinstance(policy, dict) assert isinstance(config, dict) assert not (EXAMPLE_DIRECTORY / "privacy_guard_app.py").exists() + assert "EngineRegistry(include_builtin_engines=True)" in implementation assert "def create_registry() -> EngineRegistry:" in implementation middleware_config = policy["network_middlewares"]["privacy_guard_detect"] assert middleware_config["middleware"] == "privacy-guard-custom-engine" diff --git a/projects/privacy-guard/tests/test_engine_registry.py b/projects/privacy-guard/tests/test_engine_registry.py index 7116673..b603552 100644 --- a/projects/privacy-guard/tests/test_engine_registry.py +++ b/projects/privacy-guard/tests/test_engine_registry.py @@ -127,10 +127,17 @@ def test_builtin_registry_contains_the_builtin_regex_engine() -> None: ) +def test_registry_can_include_builtin_engines_before_custom_registration() -> None: + registry = EngineRegistry(include_builtin_engines=True) + registry.register(AcmeEngine, resources=AcmeResources(prefix="token")) + registry.finalize() + + assert registry.engine_names == ("regex", "acme-pii") + + def test_custom_engine_config_joins_the_exact_discriminated_union() -> None: resources = AcmeResources(prefix="token") - registry = EngineRegistry() - registry.register(RegexEngine) + registry = EngineRegistry(include_builtin_engines=True) registry.register(AcmeEngine, resources=resources) registry.finalize() From 03f8dac510ae4f6ec06496f7614c4de6d42f0813 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 25 Jul 2026 16:17:22 +0000 Subject: [PATCH 40/82] Clarify Privacy Guard model types --- .../architecture/entity-processing-engines.md | 9 ++++---- .../architecture/service-boundary.md | 3 +-- .../privacy-guard/src/privacy_guard/cli.py | 4 +++- .../privacy-guard/src/privacy_guard/config.py | 22 +++++++------------ .../src/privacy_guard/engine_registry.py | 19 ++++++++-------- .../src/privacy_guard/engines/__init__.py | 4 ---- .../src/privacy_guard/engines/base.py | 15 +------------ .../src/privacy_guard/request_processor.py | 13 ++++++----- .../src/privacy_guard/service/servicer.py | 13 ++++------- .../privacy-guard/tests/engines/test_base.py | 22 +++++++------------ .../tests/service/test_servicer.py | 4 +++- .../tests/test_engine_registry.py | 4 ++-- 12 files changed, 51 insertions(+), 81 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md index e17fc88..88f5432 100644 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -141,7 +141,7 @@ Each `EntityDetection` contains: | `entity` | Bounded entity label | | `start` | Inclusive Unicode code-point offset in the stage input | | `end` | Exclusive Unicode code-point offset in the stage input | -| `confidence` | Optional `low`, `medium`, `high`, or strict value from 0 through 1 | +| `confidence` | Optional `low`, `medium`, or `high` certainty | | `metadata` | Optional bounded engine-specific attribution retained inside the processing boundary | A detection span is non-empty and must fall within the stage input. The public @@ -154,9 +154,8 @@ successful return is the engine's authoritative completed result. Text may not change without at least one detection. Engines must raise on native partial failure instead of returning partial output. -Confidence remains in the representation supplied by the engine. Privacy Guard -does not invent numeric values for categorical confidence or compare confidence -across engines. +Privacy Guard preserves categorical confidence as supplied by the engine and +does not compare confidence across engines. ## Custom engine example @@ -302,7 +301,7 @@ Test engines without gRPC: - detection-only immutability - replacement behavior and native partial failure - Unicode offsets and invalid spans -- categorical and numeric confidence +- categorical confidence - output and detection limits - timeout propagation and expiration - deterministic ordering where relevant diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index 1cd1b75..1c6ace1 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -102,8 +102,7 @@ confidence. For each `EntityDetectionSummary`, the service emits: - `type`: `detected_entity` - `label`: `entity (source-stage)` -- `confidence`: the categorical value, a bounded numeric representation, or - empty when absent +- `confidence`: the categorical value or empty when absent - `count`: aggregate occurrence count The current OpenShell `Finding` message has no dedicated source field. diff --git a/projects/privacy-guard/src/privacy_guard/cli.py b/projects/privacy-guard/src/privacy_guard/cli.py index 8219c91..8d72440 100644 --- a/projects/privacy-guard/src/privacy_guard/cli.py +++ b/projects/privacy-guard/src/privacy_guard/cli.py @@ -102,7 +102,9 @@ def engines(context: typer.Context) -> None: for strategy in EntityProcessingStrategy if strategy in description.supported_strategies ) - typer.echo(f"{description.engine}\t{strategies}\t{description.description}") + typer.echo( + f"{description.engine_name}\t{strategies}\t{description.description}" + ) _LOGGER = logging.getLogger(__name__) diff --git a/projects/privacy-guard/src/privacy_guard/config.py b/projects/privacy-guard/src/privacy_guard/config.py index e2888ba..6e7e8f9 100644 --- a/projects/privacy-guard/src/privacy_guard/config.py +++ b/projects/privacy-guard/src/privacy_guard/config.py @@ -13,7 +13,7 @@ from functools import reduce from hashlib import sha256 from operator import or_ -from typing import Annotated, Generic, Self, TypeAlias, TypeVar +from typing import Annotated, Generic, Self, TypeVar from pydantic import ( Field, @@ -80,7 +80,7 @@ def diagnostic_name(self, stage_number: int) -> str: return f"{engine}[{stage_number}]" -class EntityProcessing( +class EntityProcessingStages( StrictDomainModel, Generic[_EngineConfigT], ): @@ -112,17 +112,13 @@ class PrivacyGuardConfig( ): """Complete validated Privacy Guard behavior for one OpenShell policy.""" - entity_processing: EntityProcessing[_EngineConfigT] = Field(repr=False) + entity_processing: EntityProcessingStages[_EngineConfigT] = Field(repr=False) on_detection: OnDetection = Field(repr=False) -FinalizedPrivacyGuardConfig: TypeAlias = PrivacyGuardConfig[EngineConfig] -FinalizedPrivacyGuardConfigType = type[FinalizedPrivacyGuardConfig] - - def build_privacy_guard_config_type( config_types: Sequence[type[EngineConfig]], -) -> FinalizedPrivacyGuardConfigType: +) -> type[PrivacyGuardConfig[EngineConfig]]: """Build the exact registry-dependent discriminated policy model.""" if not config_types: raise ValueError("at least one engine config type must be registered") @@ -143,16 +139,16 @@ def build_privacy_guard_config_type( def build_privacy_guard_config_adapter( config_types: Sequence[type[EngineConfig]], -) -> TypeAdapter[FinalizedPrivacyGuardConfig]: +) -> TypeAdapter[PrivacyGuardConfig[EngineConfig]]: """Build the registry-dependent adapter used for validation and schemas.""" config_type = build_privacy_guard_config_type(config_types) return TypeAdapter(config_type) def parse_privacy_guard_config( - adapter: TypeAdapter[FinalizedPrivacyGuardConfig], + adapter: TypeAdapter[PrivacyGuardConfig[EngineConfig]], values: object, -) -> FinalizedPrivacyGuardConfig: +) -> PrivacyGuardConfig[EngineConfig]: """Parse an expanded mapping without exposing rejected values in errors.""" if not isinstance(values, Mapping): raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) @@ -183,10 +179,8 @@ def configuration_fingerprint( __all__ = [ - "EntityProcessing", "EntityProcessingStage", - "FinalizedPrivacyGuardConfig", - "FinalizedPrivacyGuardConfigType", + "EntityProcessingStages", "OnDetection", "PolicyAction", "PrivacyGuardConfig", diff --git a/projects/privacy-guard/src/privacy_guard/engine_registry.py b/projects/privacy-guard/src/privacy_guard/engine_registry.py index 14e705f..6648432 100644 --- a/projects/privacy-guard/src/privacy_guard/engine_registry.py +++ b/projects/privacy-guard/src/privacy_guard/engine_registry.py @@ -12,9 +12,8 @@ from pydantic_core import PydanticUndefined from privacy_guard.config import ( - FinalizedPrivacyGuardConfig, - FinalizedPrivacyGuardConfigType, PolicyAction, + PrivacyGuardConfig, build_privacy_guard_config_type, parse_privacy_guard_config, ) @@ -37,7 +36,7 @@ class EngineDescription: """Safe discovery metadata for one registered engine.""" - engine: str + engine_name: str description: str supported_strategies: frozenset[EntityProcessingStrategy] configuration_schema: dict[str, object] @@ -48,8 +47,10 @@ class EngineRegistry: def __init__(self, *, include_builtin_engines: bool = False) -> None: self._registrations: dict[str, _Registration] = {} - self._config_type: FinalizedPrivacyGuardConfigType | None = None - self._config_adapter: TypeAdapter[FinalizedPrivacyGuardConfig] | None = None + self._config_type: type[PrivacyGuardConfig[EngineConfig]] | None = None + self._config_adapter: TypeAdapter[PrivacyGuardConfig[EngineConfig]] | None = ( + None + ) if include_builtin_engines: self.register(RegexEngine) @@ -62,13 +63,13 @@ def engine_names(self) -> tuple[str, ...]: return tuple(self._registrations) @property - def config_type(self) -> FinalizedPrivacyGuardConfigType: + def config_type(self) -> type[PrivacyGuardConfig[EngineConfig]]: if self._config_type is None: raise EngineRegistryError("engine registry is not finalized") return self._config_type @property - def config_adapter(self) -> TypeAdapter[FinalizedPrivacyGuardConfig]: + def config_adapter(self) -> TypeAdapter[PrivacyGuardConfig[EngineConfig]]: if self._config_adapter is None: raise EngineRegistryError("engine registry is not finalized") return self._config_adapter @@ -170,7 +171,7 @@ def finalize(self) -> Self: self._config_adapter = TypeAdapter(config_type) return self - def validate_config(self, values: object) -> FinalizedPrivacyGuardConfig: + def validate_config(self, values: object) -> PrivacyGuardConfig[EngineConfig]: """Purely parse and validate an expanded Privacy Guard configuration.""" config = parse_privacy_guard_config(self.config_adapter, values) required_strategy = ( @@ -212,7 +213,7 @@ def describe_engines(self) -> tuple[EngineDescription, ...]: """Return safe engine metadata without constructing runtime engines.""" return tuple( EngineDescription( - engine=engine, + engine_name=engine, description=_engine_description(registration.engine_type), supported_strategies=registration.supported_strategies, configuration_schema=registration.config_type.model_json_schema(), diff --git a/projects/privacy-guard/src/privacy_guard/engines/__init__.py b/projects/privacy-guard/src/privacy_guard/engines/__init__.py index 06e33e7..a673331 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/__init__.py +++ b/projects/privacy-guard/src/privacy_guard/engines/__init__.py @@ -5,7 +5,6 @@ from privacy_guard.engines.base import ( BoundedMetadata, ConfidenceLevel, - DetectionConfidence, EngineConfig, EngineResources, EntityDetection, @@ -13,7 +12,6 @@ EntityProcessingEngine, EntityProcessingStrategy, TextProcessingResult, - UnitInterval, ) from privacy_guard.engines.regex import ( RegexEngine, @@ -34,7 +32,6 @@ __all__ = [ "BoundedMetadata", "ConfidenceLevel", - "DetectionConfidence", "EngineConfig", "EngineConfigurationError", "EngineContractError", @@ -53,5 +50,4 @@ "RegexPatternCatalog", "RegexReplacement", "TextProcessingResult", - "UnitInterval", ] diff --git a/projects/privacy-guard/src/privacy_guard/engines/base.py b/projects/privacy-guard/src/privacy_guard/engines/base.py index 286c5e1..d0b5395 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/base.py +++ b/projects/privacy-guard/src/privacy_guard/engines/base.py @@ -61,17 +61,6 @@ class ConfidenceLevel(StrEnum): HIGH = "high" -def _parse_unit_interval(value: object) -> float: - if isinstance(value, bool) or not isinstance(value, int | float): - raise ValueError("confidence must be a number from zero through one") - result = float(value) - if not 0.0 <= result <= 1.0: - raise ValueError("confidence must be a number from zero through one") - return result - - -UnitInterval = Annotated[float, BeforeValidator(_parse_unit_interval)] -DetectionConfidence: TypeAlias = ConfidenceLevel | UnitInterval EntityName = Annotated[str, BeforeValidator(validate_bounded_metadata_string)] MetadataString = Annotated[str, BeforeValidator(validate_bounded_metadata_string)] BoundedMetadata: TypeAlias = Mapping[MetadataString, MetadataString] @@ -83,7 +72,7 @@ class EntityDetection(StrictDomainModel): entity: EntityName start: int = Field(ge=0) end: int - confidence: DetectionConfidence | None = None + confidence: ConfidenceLevel | None = None metadata: BoundedMetadata = Field(default_factory=dict, repr=False) @field_validator("confidence", mode="before") @@ -368,7 +357,6 @@ def _validate_result( __all__ = [ "BoundedMetadata", "ConfidenceLevel", - "DetectionConfidence", "EngineConfig", "EngineConfigurationError", "EngineContractError", @@ -381,5 +369,4 @@ def _validate_result( "EntityProcessingError", "EntityProcessingStrategy", "TextProcessingResult", - "UnitInterval", ] diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index b5458f8..1f0df9d 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -12,7 +12,7 @@ from pydantic import Field from privacy_guard.base import StrictDomainModel -from privacy_guard.config import FinalizedPrivacyGuardConfig, PolicyAction +from privacy_guard.config import PolicyAction, PrivacyGuardConfig from privacy_guard.constants import ( BLOCK_REASON_CODE, DEFAULT_TIMEOUT_SECONDS, @@ -23,7 +23,8 @@ MAX_TIMEOUT_SECONDS, ) from privacy_guard.engines import ( - DetectionConfidence, + ConfidenceLevel, + EngineConfig, EntityProcessingStrategy, TextProcessingResult, ) @@ -50,7 +51,7 @@ class EntityDetectionSummary(StrictDomainModel): entity: str source_stage: str - confidence: DetectionConfidence | None = None + confidence: ConfidenceLevel | None = None count: int = Field(ge=1) @@ -68,7 +69,7 @@ class RequestProcessor: def __init__( self, - config: FinalizedPrivacyGuardConfig, + config: PrivacyGuardConfig[EngineConfig], stages: Sequence[tuple[str, _RunnableEngine]], *, timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, @@ -96,7 +97,7 @@ def __init__( self._log_request_content = log_request_content @property - def config(self) -> FinalizedPrivacyGuardConfig: + def config(self) -> PrivacyGuardConfig[EngineConfig]: """Return the exact validated configuration retained by this processor.""" return self._config @@ -184,7 +185,7 @@ def _aggregate_detections( stage_results: Sequence[tuple[str, TextProcessingResult]], ) -> tuple[EntityDetectionSummary, ...]: groups: OrderedDict[ - tuple[str, str, DetectionConfidence | None], + tuple[str, str, ConfidenceLevel | None], int, ] = OrderedDict() for source, result in stage_results: diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index b9d4dc9..ca4c1db 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -18,7 +18,7 @@ from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc -from privacy_guard.config import FinalizedPrivacyGuardConfig, configuration_fingerprint +from privacy_guard.config import PrivacyGuardConfig, configuration_fingerprint from privacy_guard.constants import ( BLOCK_REASON, BLOCK_REASON_CODE, @@ -33,7 +33,7 @@ SERVICE_VERSION, ) from privacy_guard.engine_registry import EngineRegistry -from privacy_guard.engines import ConfidenceLevel +from privacy_guard.engines import EngineConfig from privacy_guard.errors import ( EngineRegistryError, ErrorCode, @@ -261,7 +261,7 @@ def _prepare(self, values: object) -> tuple[str, RequestProcessor]: def _build_processor( self, - config: FinalizedPrivacyGuardConfig, + config: PrivacyGuardConfig[EngineConfig], ) -> RequestProcessor: stages = tuple( ( @@ -365,12 +365,7 @@ def _result_to_proto(result: RequestProcessingResult) -> pb2.HttpRequestResult: def _detection_to_proto(detection: EntityDetectionSummary) -> pb2.Finding: confidence = detection.confidence - if isinstance(confidence, ConfidenceLevel): - confidence_text = confidence.value - elif confidence is None: - confidence_text = "" - else: - confidence_text = format(confidence, ".12g") + confidence_text = confidence.value if confidence is not None else "" result = pb2.Finding( type="detected_entity", label=f"{detection.entity} ({detection.source_stage})", diff --git a/projects/privacy-guard/tests/engines/test_base.py b/projects/privacy-guard/tests/engines/test_base.py index 5673923..8e2c82e 100644 --- a/projects/privacy-guard/tests/engines/test_base.py +++ b/projects/privacy-guard/tests/engines/test_base.py @@ -56,7 +56,7 @@ def _run( entity="token", start=0, end=len(text), - confidence=0.75, + confidence=ConfidenceLevel.HIGH, metadata={"provider": "custom"}, ) output = ( @@ -105,22 +105,16 @@ def test_detection_confidence_and_metadata_are_strict_bounded_values() -> None: "metadata": {"pattern": "email.patterns[0]"}, } ) - numeric = EntityDetection( - entity="email", - start=0, - end=1, - confidence=0.25, - ) - assert categorical.confidence is ConfidenceLevel.HIGH - assert numeric.confidence == 0.25 assert type(categorical.metadata).__name__ == "mappingproxy" with pytest.raises(ValidationError): - EntityDetection( - entity="email", - start=0, - end=1, - confidence=1.01, + EntityDetection.model_validate( + { + "entity": "email", + "start": 0, + "end": 1, + "confidence": 0.25, + } ) diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index 1e9e07e..eb3ad2f 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -10,7 +10,9 @@ from google.protobuf.message import Message from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 +from privacy_guard.config import PrivacyGuardConfig from privacy_guard.engine_registry import create_builtin_registry +from privacy_guard.engines import EngineConfig from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.request_processor import RequestProcessor from privacy_guard.service import servicer as servicer_module @@ -147,7 +149,7 @@ def test_evaluation_revalidates_configuration_before_reusing_cached_processor( def record_validation( registry: servicer_module.EngineRegistry, values: object, - ) -> servicer_module.FinalizedPrivacyGuardConfig: + ) -> PrivacyGuardConfig[EngineConfig]: nonlocal validation_count validation_count += 1 return original_validate(registry, values) diff --git a/projects/privacy-guard/tests/test_engine_registry.py b/projects/privacy-guard/tests/test_engine_registry.py index b603552..d456a8f 100644 --- a/projects/privacy-guard/tests/test_engine_registry.py +++ b/projects/privacy-guard/tests/test_engine_registry.py @@ -118,7 +118,7 @@ def test_builtin_registry_contains_the_builtin_regex_engine() -> None: assert registry.is_finalized is True assert registry.engine_names == ("regex",) description = registry.describe_engines()[0] - assert description.engine == "regex" + assert description.engine_name == "regex" assert description.supported_strategies == frozenset( { EntityProcessingStrategy.DETECT, @@ -280,7 +280,7 @@ def _run( descriptions = registry.describe_engines() assert CountingEngine.initialized == 0 - assert descriptions[0].engine == "detect-only" + assert descriptions[0].engine_name == "detect-only" assert descriptions[0].supported_strategies == frozenset( {EntityProcessingStrategy.DETECT} ) From 907632b29e0fdb9c3f2ddc517df5eed86567f116 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 25 Jul 2026 16:34:11 +0000 Subject: [PATCH 41/82] Reduce Privacy Guard abstraction overhead --- .../architecture/request-lifecycle.md | 11 +- .../architecture/safety-and-limits.md | 6 +- .../privacy-guard/src/privacy_guard/config.py | 70 ++---------- .../src/privacy_guard/constants.py | 8 -- .../src/privacy_guard/engine_registry.py | 100 ++++++++++-------- .../src/privacy_guard/engines/base.py | 7 -- .../src/privacy_guard/engines/regex.py | 13 ++- .../privacy-guard/src/privacy_guard/errors.py | 28 ++--- .../src/privacy_guard/request_processor.py | 16 +-- .../src/privacy_guard/service/servicer.py | 54 ++++------ .../tests/test_engine_registry.py | 25 ++--- 11 files changed, 120 insertions(+), 218 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/request-lifecycle.md b/docs/documentation/privacy-guard/architecture/request-lifecycle.md index f0a13ea..fc5b2fd 100644 --- a/docs/documentation/privacy-guard/architecture/request-lifecycle.md +++ b/docs/documentation/privacy-guard/architecture/request-lifecycle.md @@ -36,9 +36,8 @@ on_detection: action: detect ``` -`entity_processing` is an object so concrete pipeline-wide settings can be -added later without changing the stage-list shape. V0 defines only `stages`. -The list must be non-empty. +`entity_processing` groups the ordered stages and owns their non-empty-list and +unique-diagnostic-name validation. V0 defines only `stages`. Each `EntityProcessingStage` contains: @@ -86,8 +85,8 @@ A non-empty body must decode as strict UTF-8. The decoded `str` is the only request input passed to `RequestProcessor`; headers, content type, request ID, target, and protobuf messages do not cross that boundary. -The processor validates both character and encoded-byte bounds before running -the pipeline. +The processor validates the encoded UTF-8 byte bound before running the +pipeline. ## Ordered stage execution @@ -107,7 +106,7 @@ The processor then: 2. calls each stage exactly once in policy order 3. passes the current text, invocation strategy, and shared timeout to `engine.run()` -4. validates intermediate character, byte, and detection limits +4. validates intermediate UTF-8 byte and detection limits 5. passes the returned text to the next stage 6. checks the same timeout after the final result diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index d556a42..0a334c6 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -34,7 +34,6 @@ reimplement either. | Limit | Constant | Value | Owner | | --- | --- | ---: | --- | | Incoming request body | `MAX_BODY_BYTES` | 4 MiB | Service | -| Input or intermediate text characters | `MAX_SCANNED_CHARACTERS` | 4,194,304 | Processor | | Input or intermediate UTF-8 bytes | `MAX_BODY_BYTES` | 4 MiB | Processor | | Engine output UTF-8 bytes | `MAX_BODY_BYTES` | 4 MiB | Engine wrapper | | Encoded replacement body | `MAX_BODY_BYTES` | 4 MiB | Service | @@ -81,7 +80,6 @@ deny with no partial findings. | Entities per catalog | `MAX_REGEX_ENTITIES_PER_CATALOG` | 2,000 | | Patterns per catalog | `MAX_REGEX_PATTERNS_PER_CATALOG` | 10,000 | | Pattern string | `MAX_REGEX_PATTERN_BYTES` | 16 KiB | -| Matches per pattern | `MAX_MATCHES_PER_PATTERN` | 256 | Entity and pattern names use `[A-Za-z_][A-Za-z0-9_-]*`. Pattern names are optional; their deterministic @@ -104,7 +102,7 @@ configuration. It: failure - evaluates patterns independently to retain overlaps - uses the timeout-capable third-party `regex` backend -- caps matches per pattern and detections per stage +- caps detections per stage - projects exact UTF-8 replacement size before rendering No timeout, limit, or pattern failure returns partial stage detections or @@ -117,7 +115,7 @@ The processor returns a successful deny with - the shared timeout expires - an engine exceeds its detection or output limit -- intermediate text exceeds the character or byte limit +- intermediate text exceeds the UTF-8 byte limit - aggregate request detections exceed the limit The service returns the same bounded deny when: diff --git a/projects/privacy-guard/src/privacy_guard/config.py b/projects/privacy-guard/src/privacy_guard/config.py index 6e7e8f9..e9d67ec 100644 --- a/projects/privacy-guard/src/privacy_guard/config.py +++ b/projects/privacy-guard/src/privacy_guard/config.py @@ -8,24 +8,18 @@ from __future__ import annotations import json -from collections.abc import Mapping, Sequence from enum import StrEnum -from functools import reduce from hashlib import sha256 -from operator import or_ -from typing import Annotated, Generic, Self, TypeVar +from typing import Generic, Self, TypeVar from pydantic import ( Field, - TypeAdapter, - ValidationError, field_validator, model_validator, ) from privacy_guard.base import StrictDomainModel from privacy_guard.engines import EngineConfig -from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.string_validators import ( BoundedMetadataString, validate_scalar_string, @@ -116,66 +110,18 @@ class PrivacyGuardConfig( on_detection: OnDetection = Field(repr=False) -def build_privacy_guard_config_type( - config_types: Sequence[type[EngineConfig]], -) -> type[PrivacyGuardConfig[EngineConfig]]: - """Build the exact registry-dependent discriminated policy model.""" - if not config_types: - raise ValueError("at least one engine config type must be registered") - registered_union = reduce(or_, config_types) - registered_config = Annotated[ - registered_union, # ty: ignore[invalid-type-form] - Field(discriminator="engine"), - ] - config_type = PrivacyGuardConfig.__class_getitem__( - registered_config # ty: ignore[invalid-argument-type] - ) - if not isinstance(config_type, type) or not issubclass( - config_type, PrivacyGuardConfig - ): - raise TypeError("Pydantic did not construct a policy config type") - return config_type # ty: ignore[invalid-return-type] - - -def build_privacy_guard_config_adapter( - config_types: Sequence[type[EngineConfig]], -) -> TypeAdapter[PrivacyGuardConfig[EngineConfig]]: - """Build the registry-dependent adapter used for validation and schemas.""" - config_type = build_privacy_guard_config_type(config_types) - return TypeAdapter(config_type) - - -def parse_privacy_guard_config( - adapter: TypeAdapter[PrivacyGuardConfig[EngineConfig]], - values: object, -) -> PrivacyGuardConfig[EngineConfig]: - """Parse an expanded mapping without exposing rejected values in errors.""" - if not isinstance(values, Mapping): - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) - try: - return adapter.validate_python(dict(values)) - except (TypeError, ValueError, ValidationError): - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None - - -def canonical_config_json( +def configuration_fingerprint( config: PrivacyGuardConfig[EngineConfig], -) -> bytes: - """Serialize every concrete engine field deterministically for hashing.""" - return json.dumps( +) -> str: + """Return the canonical SHA-256 fingerprint of an expanded policy config.""" + serialized = json.dumps( config.model_dump(mode="json"), allow_nan=False, ensure_ascii=False, separators=(",", ":"), sort_keys=True, ).encode("utf-8") - - -def configuration_fingerprint( - config: PrivacyGuardConfig[EngineConfig], -) -> str: - """Return the canonical SHA-256 fingerprint of an expanded policy config.""" - return sha256(canonical_config_json(config)).hexdigest() + return sha256(serialized).hexdigest() __all__ = [ @@ -184,9 +130,5 @@ def configuration_fingerprint( "OnDetection", "PolicyAction", "PrivacyGuardConfig", - "build_privacy_guard_config_adapter", - "build_privacy_guard_config_type", - "canonical_config_json", "configuration_fingerprint", - "parse_privacy_guard_config", ] diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index 5c4fa90..52d14e4 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -18,7 +18,6 @@ LIMIT_REASON_CODE = "privacy_guard_limit_exceeded" # Text input limits. MAX_BODY_BYTES = 4 * 1024 * 1024 -MAX_SCANNED_CHARACTERS = 4 * 1024 * 1024 # Engine and result limits. MAX_DETECTIONS_PER_STAGE = 256 @@ -35,7 +34,6 @@ MAX_REGEX_PATTERN_BYTES = 16 * 1024 MAX_REGEX_CATALOG_FILE_BYTES = 16 * 1024 * 1024 MAX_REGEX_CATALOG_PATH_BYTES = 1024 -MAX_MATCHES_PER_PATTERN = 256 DEFAULT_TIMEOUT_SECONDS = 1.0 MAX_TIMEOUT_SECONDS = 30.0 @@ -46,10 +44,4 @@ MAX_RECEIVE_MESSAGE_BYTES = MAX_BODY_BYTES + PROTOBUF_ENVELOPE_ALLOWANCE_BYTES # Protocol validation values. -UINT32_MAX = 2**32 - 1 REASON_CODE_PATTERN = re.compile(r"[a-z][a-z0-9_]{0,63}\Z") -CONFIDENCE_RANK = { - "low": 0, - "medium": 1, - "high": 2, -} diff --git a/projects/privacy-guard/src/privacy_guard/engine_registry.py b/projects/privacy-guard/src/privacy_guard/engine_registry.py index 6648432..3a2e520 100644 --- a/projects/privacy-guard/src/privacy_guard/engine_registry.py +++ b/projects/privacy-guard/src/privacy_guard/engine_registry.py @@ -4,18 +4,19 @@ import inspect import re +from collections.abc import Mapping, Sequence from dataclasses import dataclass +from functools import reduce +from operator import or_ from types import NoneType -from typing import Literal, Self, get_args, get_origin +from typing import Annotated, Literal, Self, get_args, get_origin -from pydantic import TypeAdapter +from pydantic import Field, TypeAdapter, ValidationError from pydantic_core import PydanticUndefined from privacy_guard.config import ( PolicyAction, PrivacyGuardConfig, - build_privacy_guard_config_type, - parse_privacy_guard_config, ) from privacy_guard.engines import ( EngineConfig, @@ -39,7 +40,6 @@ class EngineDescription: engine_name: str description: str supported_strategies: frozenset[EntityProcessingStrategy] - configuration_schema: dict[str, object] class EngineRegistry: @@ -47,7 +47,6 @@ class EngineRegistry: def __init__(self, *, include_builtin_engines: bool = False) -> None: self._registrations: dict[str, _Registration] = {} - self._config_type: type[PrivacyGuardConfig[EngineConfig]] | None = None self._config_adapter: TypeAdapter[PrivacyGuardConfig[EngineConfig]] | None = ( None ) @@ -58,22 +57,6 @@ def __init__(self, *, include_builtin_engines: bool = False) -> None: def is_finalized(self) -> bool: return self._config_adapter is not None - @property - def engine_names(self) -> tuple[str, ...]: - return tuple(self._registrations) - - @property - def config_type(self) -> type[PrivacyGuardConfig[EngineConfig]]: - if self._config_type is None: - raise EngineRegistryError("engine registry is not finalized") - return self._config_type - - @property - def config_adapter(self) -> TypeAdapter[PrivacyGuardConfig[EngineConfig]]: - if self._config_adapter is None: - raise EngineRegistryError("engine registry is not finalized") - return self._config_adapter - def register( self, engine_type: type[object], @@ -121,16 +104,7 @@ def register( ): raise EngineRegistryError("engine config type is already registered") - supported_strategies = getattr(engine_type, "supported_strategies", None) - if ( - not isinstance(supported_strategies, frozenset) - or not supported_strategies - or any( - not isinstance(strategy, EntityProcessingStrategy) - for strategy in supported_strategies - ) - ): - raise EngineRegistryError("engine supported strategies are invalid") + _supported_strategies(engine_type) if resources_runtime_type is NoneType: if resources is not None: raise EngineRegistryError("resource-free engine received resources") @@ -147,9 +121,7 @@ def register( self._registrations[engine_name] = _Registration( engine_type=engine_type, config_type=config_type, - resources_type=resources_type, resources=resources, - supported_strategies=supported_strategies, ) def finalize(self) -> Self: @@ -157,7 +129,7 @@ def finalize(self) -> Self: if self.is_finalized: return self try: - config_type = build_privacy_guard_config_type( + config_type = _build_privacy_guard_config_type( tuple( registration.config_type for registration in self._registrations.values() @@ -167,13 +139,17 @@ def finalize(self) -> Self: raise EngineRegistryError( "cannot finalize an empty engine registry" ) from None - self._config_type = config_type self._config_adapter = TypeAdapter(config_type) return self def validate_config(self, values: object) -> PrivacyGuardConfig[EngineConfig]: """Purely parse and validate an expanded Privacy Guard configuration.""" - config = parse_privacy_guard_config(self.config_adapter, values) + if not isinstance(values, Mapping): + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) + try: + config = self._require_config_adapter().validate_python(dict(values)) + except (TypeError, ValueError, ValidationError): + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None required_strategy = ( EntityProcessingStrategy.REPLACE if config.on_detection.action is PolicyAction.REPLACE @@ -207,7 +183,7 @@ def create_engine( def configuration_json_schema(self) -> dict[str, object]: """Return the finalized complete policy JSON Schema.""" - return self.config_adapter.json_schema() + return self._require_config_adapter().json_schema() def describe_engines(self) -> tuple[EngineDescription, ...]: """Return safe engine metadata without constructing runtime engines.""" @@ -215,8 +191,7 @@ def describe_engines(self) -> tuple[EngineDescription, ...]: EngineDescription( engine_name=engine, description=_engine_description(registration.engine_type), - supported_strategies=registration.supported_strategies, - configuration_schema=registration.config_type.model_json_schema(), + supported_strategies=_supported_strategies(registration.engine_type), ) for engine, registration in self._registrations.items() ) @@ -236,6 +211,13 @@ def _resolve_registration( raise EngineRegistryError("engine config is not registered") from None return registration + def _require_config_adapter( + self, + ) -> TypeAdapter[PrivacyGuardConfig[EngineConfig]]: + if self._config_adapter is None: + raise EngineRegistryError("engine registry is not finalized") + return self._config_adapter + def create_builtin_registry() -> EngineRegistry: """Build the finalized registry shipped by the base package.""" @@ -246,9 +228,43 @@ def create_builtin_registry() -> EngineRegistry: class _Registration: engine_type: type[object] config_type: type[EngineConfig] - resources_type: object resources: EngineResources | None - supported_strategies: frozenset[EntityProcessingStrategy] + + +def _build_privacy_guard_config_type( + config_types: Sequence[type[EngineConfig]], +) -> type[PrivacyGuardConfig[EngineConfig]]: + if not config_types: + raise ValueError("at least one engine config type must be registered") + registered_union = reduce(or_, config_types) + registered_config = Annotated[ + registered_union, # ty: ignore[invalid-type-form] + Field(discriminator="engine"), + ] + config_type = PrivacyGuardConfig.__class_getitem__( + registered_config # ty: ignore[invalid-argument-type] + ) + if not isinstance(config_type, type) or not issubclass( + config_type, PrivacyGuardConfig + ): + raise TypeError("Pydantic did not construct a policy config type") + return config_type # ty: ignore[invalid-return-type] + + +def _supported_strategies( + engine_type: type[object], +) -> frozenset[EntityProcessingStrategy]: + supported_strategies = getattr(engine_type, "supported_strategies", None) + if ( + not isinstance(supported_strategies, frozenset) + or not supported_strategies + or any( + not isinstance(strategy, EntityProcessingStrategy) + for strategy in supported_strategies + ) + ): + raise EngineRegistryError("engine supported strategies are invalid") + return supported_strategies def _engine_discriminator( diff --git a/projects/privacy-guard/src/privacy_guard/engines/base.py b/projects/privacy-guard/src/privacy_guard/engines/base.py index d0b5395..4b87d77 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/base.py +++ b/projects/privacy-guard/src/privacy_guard/engines/base.py @@ -34,9 +34,7 @@ from privacy_guard.errors import ( EngineConfigurationError, EngineContractError, - EngineExecutionError, EngineLimitExceeded, - EntityProcessingError, ) from privacy_guard.string_validators import ( ScalarString, @@ -358,15 +356,10 @@ def _validate_result( "BoundedMetadata", "ConfidenceLevel", "EngineConfig", - "EngineConfigurationError", - "EngineContractError", - "EngineExecutionError", - "EngineLimitExceeded", "EngineResources", "EntityDetection", "EntityName", "EntityProcessingEngine", - "EntityProcessingError", "EntityProcessingStrategy", "TextProcessingResult", ] diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index 5916199..3435a30 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -23,11 +23,9 @@ from privacy_guard.base import StrictDomainModel from privacy_guard.constants import ( - CONFIDENCE_RANK, MAX_BODY_BYTES, MAX_DETECTIONS_PER_STAGE, MAX_DIAGNOSTIC_TEXT_BYTES, - MAX_MATCHES_PER_PATTERN, MAX_REGEX_CATALOG_FILE_BYTES, MAX_REGEX_CATALOG_PATH_BYTES, MAX_REGEX_ENTITIES_PER_CATALOG, @@ -240,7 +238,6 @@ def _run( detections_with_identity: list[tuple[EntityDetection, str]] = [] try: for rule in self._rules: - match_count = 0 next_position = 0 while next_position <= len(text): match = rule.compiled.search( @@ -256,9 +253,6 @@ def _run( raise EngineConfigurationError( "regex engine configuration is invalid" ) - match_count += 1 - if match_count > MAX_MATCHES_PER_PATTERN: - raise EngineLimitExceeded("regex match count exceeds the limit") detection = EntityDetection( entity=rule.entity, start=start, @@ -633,7 +627,7 @@ def _resolve_overlaps( def _categorical_confidence_rank(confidence: object) -> int: if not isinstance(confidence, ConfidenceLevel): raise EngineContractError("regex detection confidence is invalid") - return CONFIDENCE_RANK[confidence.value] + return _CONFIDENCE_RANK[confidence] def _render_bounded_replacement( @@ -675,6 +669,11 @@ def _rendered_template_size(template: str, entity: str) -> int: _NAME_PATTERN = regex.compile(r"[A-Za-z_][A-Za-z0-9_-]*\Z") _INLINE_FLAG_PATTERN = regex.compile(r"[A-Za-z0-9-]+(?=[:)])") +_CONFIDENCE_RANK = { + ConfidenceLevel.LOW: 0, + ConfidenceLevel.MEDIUM: 1, + ConfidenceLevel.HIGH: 2, +} _PATTERN_METADATA_KEY = "pattern" _MAX_CACHED_PATTERN_CATALOGS = 64 _PATTERN_CATALOG_CACHE: OrderedDict[ diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py index bcb1cbb..c08839f 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -32,13 +32,12 @@ class ErrorCode(StrEnum): BODY_ENCODING_INVALID = "body_encoding_invalid" ENGINE_OUTPUT_INVALID = "engine_output_invalid" ENGINE_EXECUTION_FAILED = "engine_execution_failed" - RESULT_LIMIT_EXCEEDED = "result_limit_exceeded" SERVER_BIND_FAILED = "server_bind_failed" UNEXPECTED_SERVICE_FAILURE = "unexpected_service_failure" @dataclass(frozen=True) -class ErrorSpec: +class _ErrorSpec: """Immutable, developer-authored classification and remediation text.""" kind: ErrorKind @@ -107,8 +106,8 @@ class EngineRegistryError(Exception): """A content-safe engine registration or registry lifecycle failure.""" -_ERROR_SPECS: dict[ErrorCode, ErrorSpec] = { - ErrorCode.CONFIG_INVALID: ErrorSpec( +_ERROR_SPECS: dict[ErrorCode, _ErrorSpec] = { + ErrorCode.CONFIG_INVALID: _ErrorSpec( ErrorKind.INVALID_INPUT, ErrorComponent.CONFIG, "parse", @@ -116,56 +115,49 @@ class EngineRegistryError(Exception): "Check entity-processing stages, engine configuration, pattern catalogs, " "replacement recipes, and the on-detection action.", ), - ErrorCode.REQUEST_PHASE_INVALID: ErrorSpec( + ErrorCode.REQUEST_PHASE_INVALID: _ErrorSpec( ErrorKind.INVALID_INPUT, ErrorComponent.SERVICE, "validate_phase", "Request evaluation phase is invalid.", "Use the advertised pre-credentials phase.", ), - ErrorCode.REQUEST_BODY_TOO_LARGE: ErrorSpec( + ErrorCode.REQUEST_BODY_TOO_LARGE: _ErrorSpec( ErrorKind.INVALID_INPUT, ErrorComponent.SERVICE, "validate_body_size", "Request body exceeds the advertised size limit.", "Reduce the request body to the maximum size in the middleware manifest.", ), - ErrorCode.BODY_ENCODING_INVALID: ErrorSpec( + ErrorCode.BODY_ENCODING_INVALID: _ErrorSpec( ErrorKind.INVALID_INPUT, ErrorComponent.SERVICE, "decode_text", "Request body encoding is invalid.", "Supply a valid UTF-8 request body.", ), - ErrorCode.ENGINE_OUTPUT_INVALID: ErrorSpec( + ErrorCode.ENGINE_OUTPUT_INVALID: _ErrorSpec( ErrorKind.INTERNAL, ErrorComponent.PROCESSOR, "validate_engine", "An entity-processing engine returned an invalid result.", "Check the engine run contract, result model, spans, strategy, and limits.", ), - ErrorCode.ENGINE_EXECUTION_FAILED: ErrorSpec( + ErrorCode.ENGINE_EXECUTION_FAILED: _ErrorSpec( ErrorKind.INTERNAL, ErrorComponent.ENGINE, "run", "An entity-processing engine failed.", "Run the engine's focused configuration and single-text tests.", ), - ErrorCode.RESULT_LIMIT_EXCEEDED: ErrorSpec( - ErrorKind.INTERNAL, - ErrorComponent.SERVICE, - "serialize_result", - "A safe middleware result could not be represented.", - "Tune replacement size or entity-detection cardinality.", - ), - ErrorCode.SERVER_BIND_FAILED: ErrorSpec( + ErrorCode.SERVER_BIND_FAILED: _ErrorSpec( ErrorKind.INTERNAL, ErrorComponent.SERVER, "bind", "Server could not bind its listen address.", "Check the listen address and port availability.", ), - ErrorCode.UNEXPECTED_SERVICE_FAILURE: ErrorSpec( + ErrorCode.UNEXPECTED_SERVICE_FAILURE: _ErrorSpec( ErrorKind.INTERNAL, ErrorComponent.SERVICE, "evaluate_http_request", diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index 1f0df9d..37934a1 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -19,7 +19,6 @@ LIMIT_REASON_CODE, MAX_BODY_BYTES, MAX_DETECTIONS_PER_REQUEST, - MAX_SCANNED_CHARACTERS, MAX_TIMEOUT_SECONDS, ) from privacy_guard.engines import ( @@ -96,21 +95,13 @@ def __init__( self._timeout_seconds = float(timeout_seconds) self._log_request_content = log_request_content - @property - def config(self) -> PrivacyGuardConfig[EngineConfig]: - """Return the exact validated configuration retained by this processor.""" - return self._config - def process(self, text: str) -> RequestProcessingResult: """Process one complete request text and apply the user-facing action.""" try: input_text = validate_scalar_string(text) except ValueError: raise PrivacyGuardError(ErrorCode.BODY_ENCODING_INVALID) from None - if ( - len(input_text) > MAX_SCANNED_CHARACTERS - or len(input_text.encode("utf-8")) > MAX_BODY_BYTES - ): + if len(input_text.encode("utf-8")) > MAX_BODY_BYTES: raise PrivacyGuardError(ErrorCode.REQUEST_BODY_TOO_LARGE) if self._log_request_content: _LOGGER.debug("privacy_guard_text_input text=%r", input_text) @@ -136,10 +127,7 @@ def process(self, text: str) -> RequestProcessingResult: strategy=strategy, timeout=timeout, ) - if ( - len(result.text) > MAX_SCANNED_CHARACTERS - or len(result.text.encode("utf-8")) > MAX_BODY_BYTES - ): + if len(result.text.encode("utf-8")) > MAX_BODY_BYTES: raise EngineLimitExceeded("intermediate text exceeds the limit") if ( sum(len(item.detections) for _, item in stage_results) diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index ca4c1db..f1014a7 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -81,7 +81,17 @@ async def Describe( context: grpc.aio.ServicerContext[object, pb2.MiddlewareManifest], ) -> pb2.MiddlewareManifest: """Advertise the binding and its finalized policy schema.""" - return self._describe() + return pb2.MiddlewareManifest( + name=SERVICE_NAME, + service_version=SERVICE_VERSION, + bindings=[ + pb2.MiddlewareBinding( + operation=pb2.SUPERVISOR_MIDDLEWARE_OPERATION_HTTP_REQUEST, + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + max_body_bytes=MAX_BODY_BYTES, + ) + ], + ) @override async def ValidateConfig( @@ -107,19 +117,6 @@ async def EvaluateHttpRequest( """Resolve the prepared config, decode one text, and process it.""" return await self._evaluate_rpc(request, context) - def _describe(self) -> pb2.MiddlewareManifest: - return pb2.MiddlewareManifest( - name=SERVICE_NAME, - service_version=SERVICE_VERSION, - bindings=[ - pb2.MiddlewareBinding( - operation=pb2.SUPERVISOR_MIDDLEWARE_OPERATION_HTTP_REQUEST, - phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, - max_body_bytes=MAX_BODY_BYTES, - ) - ], - ) - def _validate_config( self, request: pb2.ValidateConfigRequest, @@ -189,19 +186,16 @@ async def _evaluate_http_request( result = await self._run_in_worker( lambda: self._prepare_and_process(values, request.body) ) - if result is None: - return pb2.HttpRequestResult(decision=pb2.DECISION_ALLOW) - return _result_to_proto(result) def _prepare_and_process( self, values: object, body: bytes, - ) -> RequestProcessingResult | None: + ) -> RequestProcessingResult: processor = self._processors.resolve(values) if not body: - return None + return RequestProcessingResult(decision=RequestDecision.ALLOW) try: text = body.decode("utf-8", errors="strict") except UnicodeDecodeError: @@ -240,24 +234,20 @@ def __init__( def resolve(self, values: object) -> RequestProcessor: """Return the cached or newly prepared processor for expanded config.""" - _, processor = self._prepare(values) - return processor - - def _prepare(self, values: object) -> tuple[str, RequestProcessor]: config = self._registry.validate_config(values) fingerprint = configuration_fingerprint(config) with self._lock: cached = self._processors.get(fingerprint) if cached is not None: self._processors.move_to_end(fingerprint) - return fingerprint, cached + return cached processor = self._build_processor(config) with self._lock: self._processors[fingerprint] = processor self._processors.move_to_end(fingerprint) while len(self._processors) > _MAX_CACHED_PROCESSORS: self._processors.popitem(last=False) - return fingerprint, processor + return processor def _build_processor( self, @@ -327,14 +317,12 @@ def _mapping_from_proto(config: Message) -> dict[str, object]: def _result_to_proto(result: RequestProcessingResult) -> pb2.HttpRequestResult: - try: - findings = [ - _detection_to_proto(detection) for detection in result.detection_summaries - ] - except PrivacyGuardError as error: - if error.code is ErrorCode.RESULT_LIMIT_EXCEEDED: + findings: list[pb2.Finding] = [] + for detection in result.detection_summaries: + finding = _detection_to_proto(detection) + if finding.ByteSize() > MAX_PROTO_FINDING_BYTES: return _limit_deny() - raise + findings.append(finding) if len(findings) > MAX_PROTO_FINDING_GROUPS: return _limit_deny() if result.decision is RequestDecision.ALLOW: @@ -372,8 +360,6 @@ def _detection_to_proto(detection: EntityDetectionSummary) -> pb2.Finding: confidence=confidence_text, count=detection.count, ) - if result.ByteSize() > MAX_PROTO_FINDING_BYTES: - raise PrivacyGuardError(ErrorCode.RESULT_LIMIT_EXCEEDED) return result diff --git a/projects/privacy-guard/tests/test_engine_registry.py b/projects/privacy-guard/tests/test_engine_registry.py index d456a8f..61326ea 100644 --- a/projects/privacy-guard/tests/test_engine_registry.py +++ b/projects/privacy-guard/tests/test_engine_registry.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Mapping from dataclasses import dataclass from typing import Literal @@ -116,8 +115,9 @@ def test_builtin_registry_contains_the_builtin_regex_engine() -> None: registry = create_builtin_registry() assert registry.is_finalized is True - assert registry.engine_names == ("regex",) - description = registry.describe_engines()[0] + descriptions = registry.describe_engines() + assert tuple(item.engine_name for item in descriptions) == ("regex",) + description = descriptions[0] assert description.engine_name == "regex" assert description.supported_strategies == frozenset( { @@ -132,7 +132,10 @@ def test_registry_can_include_builtin_engines_before_custom_registration() -> No registry.register(AcmeEngine, resources=AcmeResources(prefix="token")) registry.finalize() - assert registry.engine_names == ("regex", "acme-pii") + assert tuple(item.engine_name for item in registry.describe_engines()) == ( + "regex", + "acme-pii", + ) def test_custom_engine_config_joins_the_exact_discriminated_union() -> None: @@ -148,7 +151,10 @@ def test_custom_engine_config_joins_the_exact_discriminated_union() -> None: assert type(engine) is AcmeEngine assert engine.config is config.entity_processing.stages[0].config assert engine.resources is resources - assert registry.engine_names == ("regex", "acme-pii") + assert tuple(item.engine_name for item in registry.describe_engines()) == ( + "regex", + "acme-pii", + ) def test_detection_only_engine_is_rejected_for_replace_action() -> None: @@ -284,15 +290,6 @@ def _run( assert descriptions[0].supported_strategies == frozenset( {EntityProcessingStrategy.DETECT} ) - properties_value = descriptions[0].configuration_schema["properties"] - assert isinstance(properties_value, Mapping) - properties = { - key: value for key, value in properties_value.items() if isinstance(key, str) - } - engine_value = properties["engine"] - assert isinstance(engine_value, Mapping) - engine = {key: value for key, value in engine_value.items() if isinstance(key, str)} - assert engine["const"] == "detect-only" def test_registry_requires_at_least_one_engine() -> None: From a18291cf61ce1382373b66fe8bcea0a09f4e00ca Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 25 Jul 2026 16:48:11 +0000 Subject: [PATCH 42/82] Standardize Privacy Guard error names --- .../src/privacy_guard/engines/__init__.py | 4 ++-- .../src/privacy_guard/engines/base.py | 8 ++++---- .../src/privacy_guard/engines/regex.py | 13 +++++++------ .../privacy-guard/src/privacy_guard/errors.py | 6 +++++- .../src/privacy_guard/request_processor.py | 15 ++++++++++----- .../privacy-guard/src/privacy_guard/timeout.py | 13 +++++-------- projects/privacy-guard/tests/engines/test_base.py | 9 +++++---- .../privacy-guard/tests/engines/test_regex.py | 9 +++++---- 8 files changed, 43 insertions(+), 34 deletions(-) diff --git a/projects/privacy-guard/src/privacy_guard/engines/__init__.py b/projects/privacy-guard/src/privacy_guard/engines/__init__.py index a673331..a928d55 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/__init__.py +++ b/projects/privacy-guard/src/privacy_guard/engines/__init__.py @@ -25,7 +25,7 @@ EngineConfigurationError, EngineContractError, EngineExecutionError, - EngineLimitExceeded, + EngineLimitExceededError, EntityProcessingError, ) @@ -36,7 +36,7 @@ "EngineConfigurationError", "EngineContractError", "EngineExecutionError", - "EngineLimitExceeded", + "EngineLimitExceededError", "EngineResources", "EntityDetection", "EntityName", diff --git a/projects/privacy-guard/src/privacy_guard/engines/base.py b/projects/privacy-guard/src/privacy_guard/engines/base.py index 4b87d77..39593e0 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/base.py +++ b/projects/privacy-guard/src/privacy_guard/engines/base.py @@ -34,7 +34,7 @@ from privacy_guard.errors import ( EngineConfigurationError, EngineContractError, - EngineLimitExceeded, + EngineLimitExceededError, ) from privacy_guard.string_validators import ( ScalarString, @@ -123,7 +123,7 @@ def from_detections( bounded: list[EntityDetection] = [] for detection in detections: if len(bounded) >= MAX_DETECTIONS_PER_STAGE: - raise EngineLimitExceeded("engine returned too many detections") + raise EngineLimitExceededError("engine returned too many detections") bounded.append(detection) return cls(text=text, detections=tuple(bounded)) @@ -339,9 +339,9 @@ def _validate_result( if not isinstance(result, TextProcessingResult): raise EngineContractError("engine output is invalid") if len(result.detections) > MAX_DETECTIONS_PER_STAGE: - raise EngineLimitExceeded("engine returned too many detections") + raise EngineLimitExceededError("engine returned too many detections") if len(result.text.encode("utf-8")) > MAX_BODY_BYTES: - raise EngineLimitExceeded("engine output text exceeds the size limit") + raise EngineLimitExceededError("engine output text exceeds the size limit") for detection in result.detections: if detection.end > len(input_text): raise EngineContractError("engine detection span is invalid") diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index 3435a30..ec58d65 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -44,10 +44,11 @@ from privacy_guard.errors import ( EngineConfigurationError, EngineContractError, - EngineLimitExceeded, + EngineLimitExceededError, + TimeoutExpiredError, ) from privacy_guard.string_validators import ScalarString, validate_scalar_string -from privacy_guard.timeout import Timeout, TimeoutExpired +from privacy_guard.timeout import Timeout class RegexPattern(StrictDomainModel): @@ -262,12 +263,12 @@ def _run( ) detections_with_identity.append((detection, rule.pattern_identity)) if len(detections_with_identity) > MAX_DETECTIONS_PER_STAGE: - raise EngineLimitExceeded( + raise EngineLimitExceededError( "regex detection count exceeds the limit" ) next_position = start + 1 except TimeoutError: - raise TimeoutExpired from None + raise TimeoutExpiredError from None detections_with_identity.sort( key=lambda item: ( @@ -641,11 +642,11 @@ def _render_bounded_replacement( projected_size += len(text[cursor : detection.start].encode("utf-8")) projected_size += _rendered_template_size(template, detection.entity) if projected_size > MAX_BODY_BYTES: - raise EngineLimitExceeded("regex replacement exceeds the size limit") + raise EngineLimitExceededError("regex replacement exceeds the size limit") cursor = detection.end projected_size += len(text[cursor:].encode("utf-8")) if projected_size > MAX_BODY_BYTES: - raise EngineLimitExceeded("regex replacement exceeds the size limit") + raise EngineLimitExceededError("regex replacement exceeds the size limit") parts: list[str] = [] cursor = 0 diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py index c08839f..8144afc 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -98,10 +98,14 @@ class EngineExecutionError(EntityProcessingError): """An engine's configured runtime failed to complete one text input.""" -class EngineLimitExceeded(EntityProcessingError): +class EngineLimitExceededError(EntityProcessingError): """An engine exceeded a bounded configuration or output limit.""" +class TimeoutExpiredError(EntityProcessingError): + """The shared entity-processing timeout expired.""" + + class EngineRegistryError(Exception): """A content-safe engine registration or registry lifecycle failure.""" diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index 37934a1..9f295f8 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -29,13 +29,14 @@ ) from privacy_guard.errors import ( EngineContractError, - EngineLimitExceeded, + EngineLimitExceededError, EntityProcessingError, ErrorCode, PrivacyGuardError, + TimeoutExpiredError, ) from privacy_guard.string_validators import validate_scalar_string -from privacy_guard.timeout import Timeout, TimeoutExpired +from privacy_guard.timeout import Timeout class RequestDecision(StrEnum): @@ -128,17 +129,21 @@ def process(self, text: str) -> RequestProcessingResult: timeout=timeout, ) if len(result.text.encode("utf-8")) > MAX_BODY_BYTES: - raise EngineLimitExceeded("intermediate text exceeds the limit") + raise EngineLimitExceededError( + "intermediate text exceeds the limit" + ) if ( sum(len(item.detections) for _, item in stage_results) + len(result.detections) > MAX_DETECTIONS_PER_REQUEST ): - raise EngineLimitExceeded("request detections exceed the limit") + raise EngineLimitExceededError( + "request detections exceed the limit" + ) stage_results.append((source, result)) current_text = result.text timeout.raise_if_expired() - except (EngineLimitExceeded, TimeoutExpired): + except (EngineLimitExceededError, TimeoutExpiredError): return RequestProcessingResult( decision=RequestDecision.DENY, reason_code=LIMIT_REASON_CODE, diff --git a/projects/privacy-guard/src/privacy_guard/timeout.py b/projects/privacy-guard/src/privacy_guard/timeout.py index 1a30d32..d0f164e 100644 --- a/projects/privacy-guard/src/privacy_guard/timeout.py +++ b/projects/privacy-guard/src/privacy_guard/timeout.py @@ -10,10 +10,7 @@ from privacy_guard.base import StrictDomainModel from privacy_guard.constants import MAX_TIMEOUT_SECONDS - - -class TimeoutExpired(Exception): - """Signal that the shared entity-processing timeout has expired.""" +from privacy_guard.errors import TimeoutExpiredError class Timeout(StrictDomainModel): @@ -35,15 +32,15 @@ def from_seconds(cls, seconds: float) -> Self: return cls(deadline=monotonic() + seconds) def remaining_seconds(self) -> float: - """Return the positive duration remaining or raise ``TimeoutExpired``.""" + """Return the positive duration remaining or raise ``TimeoutExpiredError``.""" remaining = self.deadline - monotonic() if remaining <= 0: - raise TimeoutExpired + raise TimeoutExpiredError return remaining def raise_if_expired(self) -> None: - """Raise ``TimeoutExpired`` when no time remains.""" + """Raise ``TimeoutExpiredError`` when no time remains.""" self.remaining_seconds() -__all__ = ["Timeout", "TimeoutExpired"] +__all__ = ["Timeout"] diff --git a/projects/privacy-guard/tests/engines/test_base.py b/projects/privacy-guard/tests/engines/test_base.py index 8e2c82e..989bb2c 100644 --- a/projects/privacy-guard/tests/engines/test_base.py +++ b/projects/privacy-guard/tests/engines/test_base.py @@ -13,14 +13,15 @@ ConfidenceLevel, EngineConfig, EngineContractError, - EngineLimitExceeded, + EngineLimitExceededError, EngineResources, EntityDetection, EntityProcessingEngine, EntityProcessingStrategy, TextProcessingResult, ) -from privacy_guard.timeout import Timeout, TimeoutExpired +from privacy_guard.errors import TimeoutExpiredError +from privacy_guard.timeout import Timeout class _Replacement(StrictDomainModel): @@ -127,7 +128,7 @@ def detections() -> Iterator[EntityDetection]: produced += 1 yield EntityDetection(entity="token", start=index, end=index + 1) - with pytest.raises(EngineLimitExceeded): + with pytest.raises(EngineLimitExceededError): TextProcessingResult.from_detections( text="x" * 1_000, detections=detections(), @@ -257,5 +258,5 @@ def test_timeout_duration_is_strict_positive_and_bounded( def test_expired_timeout_raises_typed_signal() -> None: timeout = Timeout(deadline=monotonic() - 1) - with pytest.raises(TimeoutExpired): + with pytest.raises(TimeoutExpiredError): timeout.raise_if_expired() diff --git a/projects/privacy-guard/tests/engines/test_regex.py b/projects/privacy-guard/tests/engines/test_regex.py index 1c8a1bb..c8f35e3 100644 --- a/projects/privacy-guard/tests/engines/test_regex.py +++ b/projects/privacy-guard/tests/engines/test_regex.py @@ -8,12 +8,13 @@ import privacy_guard.engines.regex as regex_module from privacy_guard.engines import ( EngineConfigurationError, - EngineLimitExceeded, + EngineLimitExceededError, EntityProcessingStrategy, RegexEngine, RegexEngineConfig, ) -from privacy_guard.timeout import Timeout, TimeoutExpired +from privacy_guard.errors import TimeoutExpiredError +from privacy_guard.timeout import Timeout def _config( @@ -223,7 +224,7 @@ def test_replacement_size_is_projected_before_rendering( replacement={"strategy": "template", "template": "[{entity}]"}, ) - with pytest.raises(EngineLimitExceeded): + with pytest.raises(EngineLimitExceededError): _run(config, "x", EntityProcessingStrategy.REPLACE) @@ -231,7 +232,7 @@ def test_pattern_search_has_an_enforceable_timeout() -> None: config = _config([{"pattern": "(a+)+$", "confidence": "high"}]) engine = RegexEngine(config, None) - with pytest.raises(TimeoutExpired): + with pytest.raises(TimeoutExpiredError): engine.run( "a" * 100_000 + "!", strategy=EntityProcessingStrategy.DETECT, From f7d28bdfb4b0035313a6f751d6533073b69164fd Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 25 Jul 2026 16:52:12 +0000 Subject: [PATCH 43/82] Move engine registry into engine package --- docs/documentation/privacy-guard/architecture/index.md | 9 ++++----- .../privacy-guard/architecture/service-boundary.md | 2 +- projects/privacy-guard/AGENTS.md | 3 +-- projects/privacy-guard/README.md | 6 +++--- .../examples/custom-engine/custom_engine.py | 2 +- projects/privacy-guard/src/privacy_guard/cli.py | 2 +- .../{engine_registry.py => engines/registry.py} | 4 +++- .../privacy-guard/src/privacy_guard/service/server.py | 2 +- .../privacy-guard/src/privacy_guard/service/servicer.py | 2 +- .../test_registry.py} | 2 +- .../privacy-guard/tests/examples/test_regex_engine.py | 2 +- projects/privacy-guard/tests/service/test_server.py | 2 +- projects/privacy-guard/tests/service/test_servicer.py | 2 +- projects/privacy-guard/tests/test_cli.py | 2 +- projects/privacy-guard/tests/test_config.py | 2 +- projects/privacy-guard/tests/test_request_processor.py | 2 +- 16 files changed, 23 insertions(+), 23 deletions(-) rename projects/privacy-guard/src/privacy_guard/{engine_registry.py => engines/registry.py} (99%) rename projects/privacy-guard/tests/{test_engine_registry.py => engines/test_registry.py} (99%) diff --git a/docs/documentation/privacy-guard/architecture/index.md b/docs/documentation/privacy-guard/architecture/index.md index ce0c4af..c4c5aa0 100644 --- a/docs/documentation/privacy-guard/architecture/index.md +++ b/docs/documentation/privacy-guard/architecture/index.md @@ -60,11 +60,10 @@ Source paths on these pages are relative to - `request_processor.py` runs configured stages over one text value, shares one timeout across them, aggregates detections, and applies the user-facing policy action. It does not import gRPC or implement an engine's algorithms. -- `engines/` defines the custom-engine contract and the built-in Regex - implementation. Each engine owns its detection and replacement algorithms. -- `engine_registry.py` registers engine implementations and operator-owned - resources, builds the exact Pydantic discriminated union, validates - configurations, and constructs configured engines. +- `engines/` defines the custom-engine contract, registers engine + implementations and operator-owned resources, builds the exact Pydantic + discriminated union, and contains the built-in Regex implementation. Each + engine owns its detection and replacement algorithms. - `config.py` defines ordered stages, the required policy action, canonical configuration serialization, and fingerprints. - top-level `base.py` defines the package-wide strict immutable domain-model diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index 1c6ace1..c6982d3 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -201,7 +201,7 @@ different engine inventories or runtime resources. Synchronous applications use: ```python -from privacy_guard.engine_registry import create_builtin_registry +from privacy_guard.engines.registry import create_builtin_registry from privacy_guard.service import PrivacyGuardServer server = PrivacyGuardServer(create_builtin_registry()) diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md index fa3d559..6880806 100644 --- a/projects/privacy-guard/AGENTS.md +++ b/projects/privacy-guard/AGENTS.md @@ -30,9 +30,8 @@ Run focused tests while working and `make check` before handoff. ## Project map -- `src/privacy_guard/engines/`: engine contract and built-in implementations +- `src/privacy_guard/engines/`: engine contract, registry, and built-in implementations - `src/privacy_guard/config.py`: policy action and ordered stage configuration -- `src/privacy_guard/engine_registry.py`: registration and finalized config union - `src/privacy_guard/request_processor.py`: stage execution and policy disposition - `src/privacy_guard/cli.py`: command parsing, discovery, schema output, and server adapter - `src/privacy_guard/base.py`: package-wide strict immutable domain-model base diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index d692b0b..f43d10b 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -117,7 +117,7 @@ Application startup registers engines and operator-owned resources, then returns one finalized registry: ```python -from privacy_guard.engine_registry import EngineRegistry +from privacy_guard.engines.registry import EngineRegistry def create_registry() -> EngineRegistry: @@ -156,7 +156,7 @@ The base installation has an explicit built-in registry containing `RegexEngine`: ```python -from privacy_guard.engine_registry import create_builtin_registry +from privacy_guard.engines.registry import create_builtin_registry registry = create_builtin_registry() ``` @@ -166,7 +166,7 @@ registry = create_builtin_registry() The server is a library API independent of the command-line application: ```python -from privacy_guard.engine_registry import create_builtin_registry +from privacy_guard.engines.registry import create_builtin_registry from privacy_guard.service import PrivacyGuardServer server = PrivacyGuardServer(create_builtin_registry()) diff --git a/projects/privacy-guard/examples/custom-engine/custom_engine.py b/projects/privacy-guard/examples/custom-engine/custom_engine.py index 532de02..fb6d8f1 100644 --- a/projects/privacy-guard/examples/custom-engine/custom_engine.py +++ b/projects/privacy-guard/examples/custom-engine/custom_engine.py @@ -7,7 +7,6 @@ from pydantic import Field -from privacy_guard.engine_registry import EngineRegistry from privacy_guard.engines import ( ConfidenceLevel, EngineConfig, @@ -17,6 +16,7 @@ EntityProcessingStrategy, TextProcessingResult, ) +from privacy_guard.engines.registry import EngineRegistry from privacy_guard.timeout import Timeout diff --git a/projects/privacy-guard/src/privacy_guard/cli.py b/projects/privacy-guard/src/privacy_guard/cli.py index 8d72440..796b8d8 100644 --- a/projects/privacy-guard/src/privacy_guard/cli.py +++ b/projects/privacy-guard/src/privacy_guard/cli.py @@ -10,8 +10,8 @@ import typer -from privacy_guard.engine_registry import EngineRegistry, create_builtin_registry from privacy_guard.engines import EntityProcessingStrategy +from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry from privacy_guard.service.server import DEFAULT_LISTEN_ADDRESS, PrivacyGuardServer app = typer.Typer( diff --git a/projects/privacy-guard/src/privacy_guard/engine_registry.py b/projects/privacy-guard/src/privacy_guard/engines/registry.py similarity index 99% rename from projects/privacy-guard/src/privacy_guard/engine_registry.py rename to projects/privacy-guard/src/privacy_guard/engines/registry.py index 3a2e520..9afc834 100644 --- a/projects/privacy-guard/src/privacy_guard/engine_registry.py +++ b/projects/privacy-guard/src/privacy_guard/engines/registry.py @@ -18,11 +18,13 @@ PolicyAction, PrivacyGuardConfig, ) -from privacy_guard.engines import ( +from privacy_guard.engines.base import ( EngineConfig, EngineResources, EntityProcessingEngine, EntityProcessingStrategy, +) +from privacy_guard.engines.regex import ( RegexEngine, ) from privacy_guard.errors import ( diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 530a14c..0882b10 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -9,7 +9,7 @@ from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES -from privacy_guard.engine_registry import EngineRegistry +from privacy_guard.engines.registry import EngineRegistry from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.service.servicer import PrivacyGuardMiddleware diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index f1014a7..6f171f4 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -32,8 +32,8 @@ SERVICE_NAME, SERVICE_VERSION, ) -from privacy_guard.engine_registry import EngineRegistry from privacy_guard.engines import EngineConfig +from privacy_guard.engines.registry import EngineRegistry from privacy_guard.errors import ( EngineRegistryError, ErrorCode, diff --git a/projects/privacy-guard/tests/test_engine_registry.py b/projects/privacy-guard/tests/engines/test_registry.py similarity index 99% rename from projects/privacy-guard/tests/test_engine_registry.py rename to projects/privacy-guard/tests/engines/test_registry.py index 61326ea..086efd5 100644 --- a/projects/privacy-guard/tests/test_engine_registry.py +++ b/projects/privacy-guard/tests/engines/test_registry.py @@ -9,7 +9,6 @@ from pydantic import field_validator from privacy_guard.base import StrictDomainModel -from privacy_guard.engine_registry import EngineRegistry, create_builtin_registry from privacy_guard.engines import ( EngineConfig, EngineConfigurationError, @@ -19,6 +18,7 @@ RegexEngine, TextProcessingResult, ) +from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry from privacy_guard.errors import EngineRegistryError, PrivacyGuardError from privacy_guard.timeout import Timeout diff --git a/projects/privacy-guard/tests/examples/test_regex_engine.py b/projects/privacy-guard/tests/examples/test_regex_engine.py index 4369338..e554ef1 100644 --- a/projects/privacy-guard/tests/examples/test_regex_engine.py +++ b/projects/privacy-guard/tests/examples/test_regex_engine.py @@ -14,8 +14,8 @@ from google.protobuf import json_format from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 -from privacy_guard.engine_registry import create_builtin_registry from privacy_guard.engines import RegexPatternCatalog +from privacy_guard.engines.registry import create_builtin_registry from privacy_guard.service.servicer import PrivacyGuardMiddleware EXAMPLE_DIRECTORY = Path(__file__).parents[2] / "examples" / "regex-engine" diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 9b4a047..e2a285d 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -10,7 +10,7 @@ import pytest from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES -from privacy_guard.engine_registry import EngineRegistry, create_builtin_registry +from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry from privacy_guard.errors import EngineRegistryError, ErrorCode, PrivacyGuardError from privacy_guard.service import server as server_module from privacy_guard.service.server import PrivacyGuardServer diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index eb3ad2f..105c0cc 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -11,8 +11,8 @@ from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.config import PrivacyGuardConfig -from privacy_guard.engine_registry import create_builtin_registry from privacy_guard.engines import EngineConfig +from privacy_guard.engines.registry import create_builtin_registry from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.request_processor import RequestProcessor from privacy_guard.service import servicer as servicer_module diff --git a/projects/privacy-guard/tests/test_cli.py b/projects/privacy-guard/tests/test_cli.py index 892ba1d..ccf1bb1 100644 --- a/projects/privacy-guard/tests/test_cli.py +++ b/projects/privacy-guard/tests/test_cli.py @@ -13,7 +13,7 @@ from privacy_guard import cli as cli_module from privacy_guard.cli import app -from privacy_guard.engine_registry import EngineRegistry, create_builtin_registry +from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry from privacy_guard.service.server import PrivacyGuardServer diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py index b16cc99..578d37a 100644 --- a/projects/privacy-guard/tests/test_config.py +++ b/projects/privacy-guard/tests/test_config.py @@ -16,12 +16,12 @@ PolicyAction, configuration_fingerprint, ) -from privacy_guard.engine_registry import EngineRegistry from privacy_guard.engines import ( RegexEngine, RegexEngineConfig, RegexPatternCatalog, ) +from privacy_guard.engines.registry import EngineRegistry from privacy_guard.errors import ErrorCode, PrivacyGuardError diff --git a/projects/privacy-guard/tests/test_request_processor.py b/projects/privacy-guard/tests/test_request_processor.py index 7fe65ef..367c704 100644 --- a/projects/privacy-guard/tests/test_request_processor.py +++ b/projects/privacy-guard/tests/test_request_processor.py @@ -3,8 +3,8 @@ from __future__ import annotations from privacy_guard.config import PolicyAction -from privacy_guard.engine_registry import EngineRegistry from privacy_guard.engines import RegexEngine +from privacy_guard.engines.registry import EngineRegistry from privacy_guard.request_processor import RequestDecision, RequestProcessor From ebfbe1f5d23e1996dda966c4e6ac112f99cfb53a Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 25 Jul 2026 17:00:54 +0000 Subject: [PATCH 44/82] Clarify engine result enforcement boundary --- .../architecture/entity-processing-engines.md | 8 +++-- projects/privacy-guard/README.md | 5 +-- .../src/privacy_guard/engines/base.py | 2 +- .../privacy-guard/tests/engines/test_base.py | 32 +++++++++++++++++++ 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md index 88f5432..f466121 100644 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -145,9 +145,11 @@ Each `EntityDetection` contains: | `metadata` | Optional bounded engine-specific attribution retained inside the processing boundary | A detection span is non-empty and must fall within the stage input. The public -wrapper also enforces stage detection and output-size limits. Engines that -produce detections lazily should use `TextProcessingResult.from_detections()`; -it stops consuming the iterable as soon as the stage limit is exceeded. +wrapper independently enforces stage detection and output-size limits for every +returned result. Engines that produce detections lazily should use +`TextProcessingResult.from_detections()`; this optional convenience stops +consuming the iterable as soon as the stage limit is exceeded, but it is not +the enforcement boundary. For `DETECT`, output text must exactly equal input text. For `REPLACE`, a successful return is the engine's authoritative completed result. Text may not diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index f43d10b..c231d0d 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -100,8 +100,9 @@ Custom engines are a first-class extension point. Authors declare one typed config, optional typed resources, `supported_strategies`, and `_run`. They do not write `__init__`; `_initialize` is optional, and `@override` is not required. The public wrapper validates strategy support, input, timeout, spans, output -size, mutation behavior, and result cardinality. Lazy detection streams use -`TextProcessingResult.from_detections()` for bounded materialization. +size, mutation behavior, and result cardinality for every returned result. +`TextProcessingResult.from_detections()` is an optional safe materializer for +lazy detection streams, not the enforcement boundary. Resource-backed engines define an `EngineResources` subclass containing their operator-owned runtime dependencies. Resource bundles may contain initialized diff --git a/projects/privacy-guard/src/privacy_guard/engines/base.py b/projects/privacy-guard/src/privacy_guard/engines/base.py index 39593e0..2d1d08e 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/base.py +++ b/projects/privacy-guard/src/privacy_guard/engines/base.py @@ -119,7 +119,7 @@ def from_detections( text: str, detections: Iterable[EntityDetection], ) -> Self: - """Build a result while bounding a lazily produced detection stream.""" + """Safely materialize a lazy stream; ``run()`` still validates the result.""" bounded: list[EntityDetection] = [] for detection in detections: if len(bounded) >= MAX_DETECTIONS_PER_STAGE: diff --git a/projects/privacy-guard/tests/engines/test_base.py b/projects/privacy-guard/tests/engines/test_base.py index 989bb2c..a6e04e4 100644 --- a/projects/privacy-guard/tests/engines/test_base.py +++ b/projects/privacy-guard/tests/engines/test_base.py @@ -9,6 +9,7 @@ from pydantic import ValidationError from privacy_guard.base import StrictDomainModel +from privacy_guard.constants import MAX_DETECTIONS_PER_STAGE from privacy_guard.engines import ( ConfidenceLevel, EngineConfig, @@ -137,6 +138,37 @@ def detections() -> Iterator[EntityDetection]: assert produced == 257 +class _OversizedResultEngine(EntityProcessingEngine[_Config]): + supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + del strategy, timeout + return TextProcessingResult( + text=text, + detections=tuple( + EntityDetection(entity="token", start=0, end=1) + for _ in range(MAX_DETECTIONS_PER_STAGE + 1) + ), + ) + + +def test_engine_boundary_bounds_results_built_without_lazy_helper() -> None: + engine = _OversizedResultEngine(_Config(), None) + + with pytest.raises(EngineLimitExceededError): + engine.run( + "text", + strategy=EntityProcessingStrategy.DETECT, + timeout=Timeout.from_seconds(1), + ) + + class _DetectOnlyEngine(EntityProcessingEngine[_Config]): supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) From 8ca3e75b369769cc3639982d719fe8a0bb32a8b2 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 25 Jul 2026 17:04:03 +0000 Subject: [PATCH 45/82] Simplify lazy detection bounding --- .../privacy-guard/src/privacy_guard/engines/base.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/projects/privacy-guard/src/privacy_guard/engines/base.py b/projects/privacy-guard/src/privacy_guard/engines/base.py index 2d1d08e..980d899 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/base.py +++ b/projects/privacy-guard/src/privacy_guard/engines/base.py @@ -5,6 +5,7 @@ from abc import ABC, abstractmethod from collections.abc import Iterable, Mapping from enum import StrEnum +from itertools import islice from types import MappingProxyType from typing import ( Annotated, @@ -120,12 +121,10 @@ def from_detections( detections: Iterable[EntityDetection], ) -> Self: """Safely materialize a lazy stream; ``run()`` still validates the result.""" - bounded: list[EntityDetection] = [] - for detection in detections: - if len(bounded) >= MAX_DETECTIONS_PER_STAGE: - raise EngineLimitExceededError("engine returned too many detections") - bounded.append(detection) - return cls(text=text, detections=tuple(bounded)) + bounded = tuple(islice(detections, MAX_DETECTIONS_PER_STAGE + 1)) + if len(bounded) > MAX_DETECTIONS_PER_STAGE: + raise EngineLimitExceededError("engine returned too many detections") + return cls(text=text, detections=bounded) class EngineConfig(StrictDomainModel): From 68a842b612a86d8a407180412bfd08ae0a713549 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 14:32:54 +0000 Subject: [PATCH 46/82] Clarify overlapping regex matches --- projects/privacy-guard/src/privacy_guard/engines/regex.py | 2 +- projects/privacy-guard/tests/test_cli.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index ec58d65..0c8f8e5 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -198,7 +198,7 @@ def _patterns_are_valid(self) -> Self: class RegexEngine(EntityProcessingEngine[RegexEngineConfig]): - """Detect overlapping regex matches and optionally replace deterministic winners.""" + """Detect every regex match, including matches that share input characters.""" supported_strategies = frozenset( { diff --git a/projects/privacy-guard/tests/test_cli.py b/projects/privacy-guard/tests/test_cli.py index ccf1bb1..7a36cc9 100644 --- a/projects/privacy-guard/tests/test_cli.py +++ b/projects/privacy-guard/tests/test_cli.py @@ -48,7 +48,10 @@ def test_cli_engines_describes_the_installed_engine() -> None: assert result.exit_code == 0 assert result.output.startswith("regex\tdetect,replace\t") - assert "Detect overlapping regex matches" in result.output + description = ( + "Detect every regex match, including matches that share input characters" + ) + assert description in result.output def test_cli_schema_prints_the_finalized_discriminated_policy_schema() -> None: From d314ba86079ad4af3d4c70181336ab90c5a5274e Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 14:37:48 +0000 Subject: [PATCH 47/82] Add timeout translation context manager --- .../architecture/entity-processing-engines.md | 8 +++ .../src/privacy_guard/engines/regex.py | 50 +++++++++---------- .../src/privacy_guard/timeout.py | 12 +++++ .../privacy-guard/tests/engines/test_base.py | 17 ------- projects/privacy-guard/tests/test_timeout.py | 37 ++++++++++++++ 5 files changed, 80 insertions(+), 44 deletions(-) create mode 100644 projects/privacy-guard/tests/test_timeout.py diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md index f466121..c515e17 100644 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -240,6 +240,14 @@ When a delegated API accepts a timeout, pass the remaining duration: remaining = timeout.remaining_seconds() ``` +When that API raises Python's `TimeoutError` on expiration, use the shared +context manager to translate it into Privacy Guard's domain error: + +```python +with timeout.translate_errors(): + result = client.process(text, timeout=timeout.remaining_seconds()) +``` + Unique long-running loops may also call `timeout.raise_if_expired()`. Operations that cannot be interrupted must be documented and bounded independently. diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index 0c8f8e5..36811f5 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -45,7 +45,6 @@ EngineConfigurationError, EngineContractError, EngineLimitExceededError, - TimeoutExpiredError, ) from privacy_guard.string_validators import ScalarString, validate_scalar_string from privacy_guard.timeout import Timeout @@ -237,38 +236,35 @@ def _run( timeout: Timeout, ) -> TextProcessingResult: detections_with_identity: list[tuple[EntityDetection, str]] = [] - try: - for rule in self._rules: - next_position = 0 - while next_position <= len(text): + for rule in self._rules: + next_position = 0 + while next_position <= len(text): + with timeout.translate_errors(): match = rule.compiled.search( text, next_position, timeout=timeout.remaining_seconds(), ) - timeout.raise_if_expired() - if match is None: - break - start, end = match.span() - if start == end or match.span(rule.marker) != (end, end): - raise EngineConfigurationError( - "regex engine configuration is invalid" - ) - detection = EntityDetection( - entity=rule.entity, - start=start, - end=end, - confidence=rule.confidence, - metadata={_PATTERN_METADATA_KEY: rule.pattern_identity}, + if match is None: + break + start, end = match.span() + if start == end or match.span(rule.marker) != (end, end): + raise EngineConfigurationError( + "regex engine configuration is invalid" + ) + detection = EntityDetection( + entity=rule.entity, + start=start, + end=end, + confidence=rule.confidence, + metadata={_PATTERN_METADATA_KEY: rule.pattern_identity}, + ) + detections_with_identity.append((detection, rule.pattern_identity)) + if len(detections_with_identity) > MAX_DETECTIONS_PER_STAGE: + raise EngineLimitExceededError( + "regex detection count exceeds the limit" ) - detections_with_identity.append((detection, rule.pattern_identity)) - if len(detections_with_identity) > MAX_DETECTIONS_PER_STAGE: - raise EngineLimitExceededError( - "regex detection count exceeds the limit" - ) - next_position = start + 1 - except TimeoutError: - raise TimeoutExpiredError from None + next_position = start + 1 detections_with_identity.sort( key=lambda item: ( diff --git a/projects/privacy-guard/src/privacy_guard/timeout.py b/projects/privacy-guard/src/privacy_guard/timeout.py index d0f164e..901951c 100644 --- a/projects/privacy-guard/src/privacy_guard/timeout.py +++ b/projects/privacy-guard/src/privacy_guard/timeout.py @@ -3,6 +3,8 @@ from __future__ import annotations import math +from collections.abc import Iterator +from contextlib import contextmanager from time import monotonic from typing import Self @@ -42,5 +44,15 @@ def raise_if_expired(self) -> None: """Raise ``TimeoutExpiredError`` when no time remains.""" self.remaining_seconds() + @contextmanager + def translate_errors(self) -> Iterator[None]: + """Translate a delegated ``TimeoutError`` and enforce this deadline.""" + self.raise_if_expired() + try: + yield + except TimeoutError: + raise TimeoutExpiredError from None + self.raise_if_expired() + __all__ = ["Timeout"] diff --git a/projects/privacy-guard/tests/engines/test_base.py b/projects/privacy-guard/tests/engines/test_base.py index a6e04e4..e7262c6 100644 --- a/projects/privacy-guard/tests/engines/test_base.py +++ b/projects/privacy-guard/tests/engines/test_base.py @@ -2,7 +2,6 @@ from collections.abc import Iterator from dataclasses import dataclass -from time import monotonic from typing import Literal import pytest @@ -21,7 +20,6 @@ EntityProcessingStrategy, TextProcessingResult, ) -from privacy_guard.errors import TimeoutExpiredError from privacy_guard.timeout import Timeout @@ -277,18 +275,3 @@ def test_engine_boundary_rejects_spans_outside_stage_input() -> None: strategy=EntityProcessingStrategy.DETECT, timeout=Timeout.from_seconds(1), ) - - -@pytest.mark.parametrize("seconds", [True, 0, -1, float("inf"), 31]) -def test_timeout_duration_is_strict_positive_and_bounded( - seconds: bool | int | float, -) -> None: - with pytest.raises(ValueError): - Timeout.from_seconds(seconds) - - -def test_expired_timeout_raises_typed_signal() -> None: - timeout = Timeout(deadline=monotonic() - 1) - - with pytest.raises(TimeoutExpiredError): - timeout.raise_if_expired() diff --git a/projects/privacy-guard/tests/test_timeout.py b/projects/privacy-guard/tests/test_timeout.py new file mode 100644 index 0000000..ba6a8bd --- /dev/null +++ b/projects/privacy-guard/tests/test_timeout.py @@ -0,0 +1,37 @@ +"""Tests for the shared entity-processing timeout.""" + +from time import monotonic + +import pytest + +from privacy_guard.errors import TimeoutExpiredError +from privacy_guard.timeout import Timeout + + +@pytest.mark.parametrize("seconds", [True, 0, -1, float("inf"), 31]) +def test_timeout_duration_is_strict_positive_and_bounded( + seconds: bool | int | float, +) -> None: + with pytest.raises(ValueError): + Timeout.from_seconds(seconds) + + +def test_expired_timeout_raises_typed_signal() -> None: + timeout = Timeout(deadline=monotonic() - 1) + + with pytest.raises(TimeoutExpiredError): + timeout.raise_if_expired() + + +def test_timeout_context_translates_delegated_timeout_error() -> None: + timeout = Timeout.from_seconds(1) + + with pytest.raises(TimeoutExpiredError), timeout.translate_errors(): + raise TimeoutError + + +def test_timeout_context_preserves_unrelated_errors() -> None: + timeout = Timeout.from_seconds(1) + + with pytest.raises(RuntimeError), timeout.translate_errors(): + raise RuntimeError From b84153c4057d2c12723b887b9c3dda38d182f011 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 14:40:15 +0000 Subject: [PATCH 48/82] Rename timeout enforcement context --- .../privacy-guard/architecture/entity-processing-engines.md | 2 +- projects/privacy-guard/src/privacy_guard/engines/regex.py | 2 +- projects/privacy-guard/src/privacy_guard/timeout.py | 4 ++-- projects/privacy-guard/tests/test_timeout.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md index c515e17..4d57898 100644 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -244,7 +244,7 @@ When that API raises Python's `TimeoutError` on expiration, use the shared context manager to translate it into Privacy Guard's domain error: ```python -with timeout.translate_errors(): +with timeout.enforce(): result = client.process(text, timeout=timeout.remaining_seconds()) ``` diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index 36811f5..7491ac9 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -239,7 +239,7 @@ def _run( for rule in self._rules: next_position = 0 while next_position <= len(text): - with timeout.translate_errors(): + with timeout.enforce(): match = rule.compiled.search( text, next_position, diff --git a/projects/privacy-guard/src/privacy_guard/timeout.py b/projects/privacy-guard/src/privacy_guard/timeout.py index 901951c..0fc797a 100644 --- a/projects/privacy-guard/src/privacy_guard/timeout.py +++ b/projects/privacy-guard/src/privacy_guard/timeout.py @@ -45,8 +45,8 @@ def raise_if_expired(self) -> None: self.remaining_seconds() @contextmanager - def translate_errors(self) -> Iterator[None]: - """Translate a delegated ``TimeoutError`` and enforce this deadline.""" + def enforce(self) -> Iterator[None]: + """Enforce this deadline around one delegated operation.""" self.raise_if_expired() try: yield diff --git a/projects/privacy-guard/tests/test_timeout.py b/projects/privacy-guard/tests/test_timeout.py index ba6a8bd..522939f 100644 --- a/projects/privacy-guard/tests/test_timeout.py +++ b/projects/privacy-guard/tests/test_timeout.py @@ -26,12 +26,12 @@ def test_expired_timeout_raises_typed_signal() -> None: def test_timeout_context_translates_delegated_timeout_error() -> None: timeout = Timeout.from_seconds(1) - with pytest.raises(TimeoutExpiredError), timeout.translate_errors(): + with pytest.raises(TimeoutExpiredError), timeout.enforce(): raise TimeoutError def test_timeout_context_preserves_unrelated_errors() -> None: timeout = Timeout.from_seconds(1) - with pytest.raises(RuntimeError), timeout.translate_errors(): + with pytest.raises(RuntimeError), timeout.enforce(): raise RuntimeError From 4f8815ac11b710cca0594931d21b1822ba1376bb Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 14:45:26 +0000 Subject: [PATCH 49/82] Make timeout failures actionable --- .../architecture/safety-and-limits.md | 5 +++++ .../src/privacy_guard/constants.py | 5 ++++- .../privacy-guard/src/privacy_guard/errors.py | 6 ++++++ .../tests/service/test_servicer.py | 20 ++++++++++++++++++- projects/privacy-guard/tests/test_timeout.py | 7 ++++++- 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 0a334c6..57116f1 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -124,6 +124,11 @@ The service returns the same bounded deny when: - replacement text cannot be encoded within the body limit - a deny reason code is not safely representable +The deny reason directs users to reduce the request or replacement size, or +simplify the configured stages and patterns before retrying. These options +reduce both bounded output pressure and timeout risk without weakening the +fail-closed behavior. + Malformed input and engine contract or execution failures instead abort the RPC with a cataloged error. OpenShell then applies the middleware registration's failure behavior. diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index 52d14e4..cb89b1a 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -14,7 +14,10 @@ SERVICE_VERSION = version("privacy-guard") BLOCK_REASON = "Privacy Guard blocked the request" BLOCK_REASON_CODE = "privacy_guard_blocked" -LIMIT_REASON = "Privacy Guard denied a result that exceeded a safety limit" +LIMIT_REASON = ( + "Privacy Guard exceeded a processing safety limit. Reduce the request or " + "replacement size, or simplify the configured stages and patterns, then retry." +) LIMIT_REASON_CODE = "privacy_guard_limit_exceeded" # Text input limits. MAX_BODY_BYTES = 4 * 1024 * 1024 diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py index 8144afc..001c812 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -105,6 +105,12 @@ class EngineLimitExceededError(EntityProcessingError): class TimeoutExpiredError(EntityProcessingError): """The shared entity-processing timeout expired.""" + def __init__(self) -> None: + super().__init__( + "Privacy Guard processing timed out. Reduce the request size or simplify " + "the configured stages and patterns, then retry." + ) + class EngineRegistryError(Exception): """A content-safe engine registration or registry lifecycle failure.""" diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index 105c0cc..94ce638 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -11,10 +11,15 @@ from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.config import PrivacyGuardConfig +from privacy_guard.constants import LIMIT_REASON, LIMIT_REASON_CODE from privacy_guard.engines import EngineConfig from privacy_guard.engines.registry import create_builtin_registry from privacy_guard.errors import ErrorCode, PrivacyGuardError -from privacy_guard.request_processor import RequestProcessor +from privacy_guard.request_processor import ( + RequestDecision, + RequestProcessingResult, + RequestProcessor, +) from privacy_guard.service import servicer as servicer_module from privacy_guard.service.servicer import PrivacyGuardMiddleware @@ -89,6 +94,19 @@ def test_validate_config_is_pure_and_reports_invalid_config() -> None: assert "config_invalid" in invalid.reason +def test_limit_deny_explains_recovery_options() -> None: + result = servicer_module._result_to_proto( + RequestProcessingResult( + decision=RequestDecision.DENY, + reason_code=LIMIT_REASON_CODE, + ) + ) + + assert result.reason == LIMIT_REASON + assert "Reduce the request or replacement size" in result.reason + assert "simplify the configured stages and patterns" in result.reason + + def test_evaluation_decodes_one_utf8_text_and_encodes_replacement() -> None: async def evaluate() -> pb2.HttpRequestResult: middleware = PrivacyGuardMiddleware(create_builtin_registry()) diff --git a/projects/privacy-guard/tests/test_timeout.py b/projects/privacy-guard/tests/test_timeout.py index 522939f..2e06cc1 100644 --- a/projects/privacy-guard/tests/test_timeout.py +++ b/projects/privacy-guard/tests/test_timeout.py @@ -19,9 +19,14 @@ def test_timeout_duration_is_strict_positive_and_bounded( def test_expired_timeout_raises_typed_signal() -> None: timeout = Timeout(deadline=monotonic() - 1) - with pytest.raises(TimeoutExpiredError): + with pytest.raises(TimeoutExpiredError) as captured: timeout.raise_if_expired() + assert str(captured.value) == ( + "Privacy Guard processing timed out. Reduce the request size or simplify " + "the configured stages and patterns, then retry." + ) + def test_timeout_context_translates_delegated_timeout_error() -> None: timeout = Timeout.from_seconds(1) From b278e8cce3fa676f1e6d0cf4654ea8c9495e948e Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 14:57:08 +0000 Subject: [PATCH 50/82] Make Privacy Guard errors actionable --- .../architecture/safety-and-limits.md | 23 +++++-- .../architecture/service-boundary.md | 19 ++++-- projects/privacy-guard/README.md | 15 ++++- .../privacy-guard/src/privacy_guard/cli.py | 35 ++++++++-- .../src/privacy_guard/constants.py | 10 ++- .../privacy-guard/src/privacy_guard/errors.py | 5 +- .../src/privacy_guard/request_processor.py | 14 +--- .../src/privacy_guard/service/server.py | 8 ++- .../src/privacy_guard/service/servicer.py | 7 ++ .../src/privacy_guard/timeout.py | 28 +++++--- .../tests/service/test_server.py | 21 +++++- .../tests/service/test_servicer.py | 15 +++++ projects/privacy-guard/tests/test_cli.py | 64 +++++++++++++++---- projects/privacy-guard/tests/test_timeout.py | 8 ++- 14 files changed, 212 insertions(+), 60 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 57116f1..3dd1666 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -40,8 +40,8 @@ reimplement either. | gRPC receive message | `MAX_RECEIVE_MESSAGE_BYTES` | 5 MiB | gRPC server | | Detections returned by one stage | `MAX_DETECTIONS_PER_STAGE` | 256 | Engine wrapper | | Detections across one request | `MAX_DETECTIONS_PER_REQUEST` | 4,096 | Processor | -| Default shared timeout | `DEFAULT_TIMEOUT_SECONDS` | 1 second | Processor | -| Maximum configured timeout | `MAX_TIMEOUT_SECONDS` | 30 seconds | Processor and `Timeout` | +| Default shared timeout | `DEFAULT_TIMEOUT_SECONDS` | 1 second | Server and processor | +| Maximum configured timeout | `MAX_TIMEOUT_SECONDS` | 30 seconds | Server, processor, and `Timeout` | | Active processor workers | `MAX_CONCURRENT_PROCESSING` | 4 | Service | | Concurrent gRPC calls | `MAX_CONCURRENT_RPCS` | 16 | gRPC server | @@ -54,6 +54,10 @@ Third-party calls that accept a timeout should receive the same remaining duration; calls that cannot be interrupted continue to occupy a service worker until they exit. +Operators configure the internal bound with `--timeout-seconds` or the +programmatic `PrivacyGuardServer(timeout_seconds=...)` argument. OpenShell's +outer middleware timeout must be longer than this internal bound. + ## Diagnostic and result limits | Limit | Constant | Value | @@ -124,10 +128,11 @@ The service returns the same bounded deny when: - replacement text cannot be encoded within the body limit - a deny reason code is not safely representable -The deny reason directs users to reduce the request or replacement size, or -simplify the configured stages and patterns before retrying. These options -reduce both bounded output pressure and timeout risk without weakening the -fail-closed behavior. +The deny reason directs users to reduce the request or replacement size, +simplify the configured stages and patterns, or, when processing timed out, +increase `--timeout-seconds` up to 30 and ensure OpenShell's middleware timeout +is longer before retrying. These options reduce bounded output pressure or +provide appropriate time without weakening the fail-closed behavior. Malformed input and engine contract or execution failures instead abort the RPC with a cataloged error. OpenShell then applies the middleware registration's @@ -150,6 +155,12 @@ error spec defines: - content-safe summary - remediation hint +Successful policy and safety-limit denials use stable reason codes plus +content-safe recovery guidance. CLI validation failures name the rejected +option and state the accepted form or next action. Internal engine and +third-party failures remain concise because the owning boundary translates +them before they reach users. + Caught extension and third-party exceptions are translated at their trust boundary. Raw exception messages and chains are not exposed. Configuration errors do not include rejected pattern strings, paths, credentials, provider diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index c6982d3..9ef56e2 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -64,6 +64,13 @@ The cache is protected for concurrent access and is not correctness-relevant. Eviction or process restart simply causes reconstruction from a later evaluation's expanded config. +Each cached processor receives the server's operational processing timeout. +The default is 1 second shared across every configured stage. Operators may set +up to 30 seconds with `privacy-guard serve --timeout-seconds`; programmatic +applications pass the same `timeout_seconds` argument to `PrivacyGuardServer`. +OpenShell's outer middleware `timeout` must be longer than Privacy Guard's +processing timeout so the supervisor does not end the RPC first. + ## Incoming requests For each evaluation, the service: @@ -204,7 +211,10 @@ Synchronous applications use: from privacy_guard.engines.registry import create_builtin_registry from privacy_guard.service import PrivacyGuardServer -server = PrivacyGuardServer(create_builtin_registry()) +server = PrivacyGuardServer( + create_builtin_registry(), + timeout_seconds=5, +) server.run("127.0.0.1:50051") ``` @@ -247,7 +257,7 @@ option. They create the registry normally and pass it to the server: ```python registry = create_registry() -PrivacyGuardServer(registry).run() +PrivacyGuardServer(registry, timeout_seconds=5).run() ``` The CLI exposes: @@ -255,12 +265,13 @@ The CLI exposes: ```bash privacy-guard engines privacy-guard schema -privacy-guard serve --listen 127.0.0.1:50051 +privacy-guard serve --listen 127.0.0.1:50051 --timeout-seconds 5 ``` Entity behavior comes from policy configuration, not server startup flags. Engine implementations and operational resources come from the selected -application registry. +application registry. The processing timeout is an operational server bound, +not policy-controlled behavior. ## Upstream protocol work diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index c231d0d..d4ceb0f 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -170,7 +170,10 @@ The server is a library API independent of the command-line application: from privacy_guard.engines.registry import create_builtin_registry from privacy_guard.service import PrivacyGuardServer -server = PrivacyGuardServer(create_builtin_registry()) +server = PrivacyGuardServer( + create_builtin_registry(), + timeout_seconds=5, +) server.run("127.0.0.1:50051") ``` @@ -191,13 +194,19 @@ modules, select engines, generate schemas, or parse command-line options. ```bash uv run privacy-guard engines uv run privacy-guard schema -uv run privacy-guard serve --listen 127.0.0.1:50051 +uv run privacy-guard serve \ + --listen 127.0.0.1:50051 \ + --timeout-seconds 5 ``` Entity behavior is supplied by OpenShell policy config, not server startup flags. Deployment startup owns only installed engine implementations and operator resources such as model profiles, endpoints, clients, and credentials. -Use `--registry-factory module:factory` for a custom engine installation. +The processing timeout is an operational bound shared by every stage; it +defaults to 1 second and may be increased to at most 30 seconds. When increasing +it, configure OpenShell's middleware `timeout` to a longer duration so the +supervisor does not end the RPC first. Use `--registry-factory module:factory` +for a custom engine installation. Registry factories execute operator Python code; load only trusted modules. The `privacy_guard.cli` module owns the executable application and adapts its `serve` command to the programmatic server API. diff --git a/projects/privacy-guard/src/privacy_guard/cli.py b/projects/privacy-guard/src/privacy_guard/cli.py index 796b8d8..144605f 100644 --- a/projects/privacy-guard/src/privacy_guard/cli.py +++ b/projects/privacy-guard/src/privacy_guard/cli.py @@ -10,9 +10,11 @@ import typer +from privacy_guard.constants import DEFAULT_TIMEOUT_SECONDS, MAX_TIMEOUT_SECONDS from privacy_guard.engines import EntityProcessingStrategy from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry from privacy_guard.service.server import DEFAULT_LISTEN_ADDRESS, PrivacyGuardServer +from privacy_guard.timeout import validate_timeout_seconds app = typer.Typer( name="privacy-guard", @@ -71,11 +73,28 @@ def serve( str, typer.Option(help="Address on which the middleware server listens."), ] = DEFAULT_LISTEN_ADDRESS, + timeout_seconds: Annotated[ + float, + typer.Option( + help=( + "Maximum seconds shared by all processing stages in one request; " + f"must be greater than 0 and at most {MAX_TIMEOUT_SECONDS:g}." + ), + ), + ] = DEFAULT_TIMEOUT_SECONDS, ) -> None: """Run the middleware server with the selected engine inventory.""" options = _command_options(context) + try: + validated_timeout_seconds = validate_timeout_seconds(timeout_seconds) + except ValueError as error: + raise typer.BadParameter( + str(error), + param_hint="--timeout-seconds", + ) from None PrivacyGuardServer( options.registry, + timeout_seconds=validated_timeout_seconds, log_request_content=options.log_request_content, ).run(listen) @@ -122,7 +141,7 @@ def _load_registry(factory_reference: str | None) -> EngineRegistry: module_name, separator, factory_name = factory_reference.partition(":") if not separator or not module_name or not factory_name: raise typer.BadParameter( - "registry factory must use module:factory", + "Use module:factory, for example my_engines:create_registry.", param_hint="--registry-factory", ) try: @@ -130,29 +149,33 @@ def _load_registry(factory_reference: str | None) -> EngineRegistry: factory = getattr(module, factory_name) except Exception: raise typer.BadParameter( - "registry factory could not be loaded", + "Registry factory could not be loaded. Verify the module is installed " + "and the module:factory reference is correct.", param_hint="--registry-factory", ) from None if not callable(factory): raise typer.BadParameter( - "registry factory is not callable", + "Registry factory is not callable. Export a callable that returns a " + "finalized EngineRegistry.", param_hint="--registry-factory", ) try: registry = factory() except Exception: raise typer.BadParameter( - "registry factory failed", + "Registry factory failed. Run the factory directly with content-safe " + "diagnostics and fix its startup error.", param_hint="--registry-factory", ) from None if not isinstance(registry, EngineRegistry): raise typer.BadParameter( - "registry factory returned an invalid object", + "Registry factory returned an invalid object. Return an EngineRegistry.", param_hint="--registry-factory", ) if not registry.is_finalized: raise typer.BadParameter( - "registry factory returned an unfinalized registry", + "Registry factory returned an unfinalized registry. Call finalize() " + "before returning it.", param_hint="--registry-factory", ) return registry diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index cb89b1a..e7cd4b7 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -9,6 +9,10 @@ import re from importlib.metadata import version +# Configurable processing timeout. +DEFAULT_TIMEOUT_SECONDS = 1.0 +MAX_TIMEOUT_SECONDS = 30.0 + # Middleware identity and stable response values. SERVICE_NAME = "privacy-guard" SERVICE_VERSION = version("privacy-guard") @@ -16,7 +20,9 @@ BLOCK_REASON_CODE = "privacy_guard_blocked" LIMIT_REASON = ( "Privacy Guard exceeded a processing safety limit. Reduce the request or " - "replacement size, or simplify the configured stages and patterns, then retry." + "replacement size, simplify the configured stages and patterns, or, for a " + f"timeout, increase --timeout-seconds up to {MAX_TIMEOUT_SECONDS:g} and ensure " + "the OpenShell middleware timeout is longer, then retry." ) LIMIT_REASON_CODE = "privacy_guard_limit_exceeded" # Text input limits. @@ -37,8 +43,6 @@ MAX_REGEX_PATTERN_BYTES = 16 * 1024 MAX_REGEX_CATALOG_FILE_BYTES = 16 * 1024 * 1024 MAX_REGEX_CATALOG_PATH_BYTES = 1024 -DEFAULT_TIMEOUT_SECONDS = 1.0 -MAX_TIMEOUT_SECONDS = 30.0 # Service concurrency and transport limits. MAX_CONCURRENT_PROCESSING = 4 diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py index 001c812..1da6256 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -5,6 +5,8 @@ from dataclasses import dataclass from enum import StrEnum +from privacy_guard.constants import MAX_TIMEOUT_SECONDS + class ErrorKind(StrEnum): """Whether a failure is attributable to input or middleware internals.""" @@ -108,7 +110,8 @@ class TimeoutExpiredError(EntityProcessingError): def __init__(self) -> None: super().__init__( "Privacy Guard processing timed out. Reduce the request size or simplify " - "the configured stages and patterns, then retry." + "the configured stages and patterns, or increase the processing timeout " + f"to at most {MAX_TIMEOUT_SECONDS:g} seconds, then retry." ) diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index 9f295f8..9d3c7f6 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import math from collections import OrderedDict from collections.abc import Sequence from enum import StrEnum @@ -19,7 +18,6 @@ LIMIT_REASON_CODE, MAX_BODY_BYTES, MAX_DETECTIONS_PER_REQUEST, - MAX_TIMEOUT_SECONDS, ) from privacy_guard.engines import ( ConfidenceLevel, @@ -36,7 +34,7 @@ TimeoutExpiredError, ) from privacy_guard.string_validators import validate_scalar_string -from privacy_guard.timeout import Timeout +from privacy_guard.timeout import Timeout, validate_timeout_seconds class RequestDecision(StrEnum): @@ -75,14 +73,6 @@ def __init__( timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, log_request_content: bool = False, ) -> None: - if ( - isinstance(timeout_seconds, bool) - or not isinstance(timeout_seconds, int | float) - or not math.isfinite(timeout_seconds) - or timeout_seconds <= 0 - or timeout_seconds > MAX_TIMEOUT_SECONDS - ): - raise ValueError("timeout must be finite, positive, and bounded") configured_stages = tuple(stages) if len(configured_stages) != len(config.entity_processing.stages): raise ValueError("configured stages do not match the policy") @@ -93,7 +83,7 @@ def __init__( raise ValueError("stage sources must be non-empty and unique") self._config = config self._stages = configured_stages - self._timeout_seconds = float(timeout_seconds) + self._timeout_seconds = validate_timeout_seconds(timeout_seconds) self._log_request_content = log_request_content def process(self, text: str) -> RequestProcessingResult: diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 0882b10..2cccd2c 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -8,7 +8,11 @@ import grpc from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc -from privacy_guard.constants import MAX_CONCURRENT_RPCS, MAX_RECEIVE_MESSAGE_BYTES +from privacy_guard.constants import ( + DEFAULT_TIMEOUT_SECONDS, + MAX_CONCURRENT_RPCS, + MAX_RECEIVE_MESSAGE_BYTES, +) from privacy_guard.engines.registry import EngineRegistry from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.service.servicer import PrivacyGuardMiddleware @@ -23,10 +27,12 @@ def __init__( self, registry: EngineRegistry, *, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, log_request_content: bool = False, ) -> None: self._middleware = PrivacyGuardMiddleware( registry, + timeout_seconds=timeout_seconds, log_request_content=log_request_content, ) diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index 6f171f4..baba477 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -22,6 +22,7 @@ from privacy_guard.constants import ( BLOCK_REASON, BLOCK_REASON_CODE, + DEFAULT_TIMEOUT_SECONDS, LIMIT_REASON, LIMIT_REASON_CODE, MAX_BODY_BYTES, @@ -46,6 +47,7 @@ RequestProcessingResult, RequestProcessor, ) +from privacy_guard.timeout import validate_timeout_seconds class PrivacyGuardMiddleware(pb2_grpc.SupervisorMiddlewareServicer): @@ -55,6 +57,7 @@ def __init__( self, registry: EngineRegistry, *, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, log_request_content: bool = False, ) -> None: if not registry.is_finalized: @@ -62,6 +65,7 @@ def __init__( self._registry = registry self._processors = _RequestProcessorCache( registry, + timeout_seconds=validate_timeout_seconds(timeout_seconds), log_request_content=log_request_content, ) self._processing_slots = asyncio.Semaphore(MAX_CONCURRENT_PROCESSING) @@ -225,9 +229,11 @@ def __init__( self, registry: EngineRegistry, *, + timeout_seconds: float, log_request_content: bool, ) -> None: self._registry = registry + self._timeout_seconds = timeout_seconds self._log_request_content = log_request_content self._processors: OrderedDict[str, RequestProcessor] = OrderedDict() self._lock = RLock() @@ -266,6 +272,7 @@ def _build_processor( return RequestProcessor( config, stages, + timeout_seconds=self._timeout_seconds, log_request_content=self._log_request_content, ) diff --git a/projects/privacy-guard/src/privacy_guard/timeout.py b/projects/privacy-guard/src/privacy_guard/timeout.py index 0fc797a..763e131 100644 --- a/projects/privacy-guard/src/privacy_guard/timeout.py +++ b/projects/privacy-guard/src/privacy_guard/timeout.py @@ -15,6 +15,22 @@ from privacy_guard.errors import TimeoutExpiredError +def validate_timeout_seconds(seconds: object) -> float: + """Return a finite supported processing timeout in seconds.""" + if ( + isinstance(seconds, bool) + or not isinstance(seconds, int | float) + or not math.isfinite(seconds) + or seconds <= 0 + or seconds > MAX_TIMEOUT_SECONDS + ): + raise ValueError( + "timeout seconds must be a finite number greater than 0 and at most " + f"{MAX_TIMEOUT_SECONDS:g}" + ) + return float(seconds) + + class Timeout(StrictDomainModel): """An immutable monotonic deadline shared across processing stages.""" @@ -23,15 +39,7 @@ class Timeout(StrictDomainModel): @classmethod def from_seconds(cls, seconds: float) -> Self: """Create a timeout from a finite, positive bounded duration.""" - if ( - isinstance(seconds, bool) - or not isinstance(seconds, int | float) - or not math.isfinite(seconds) - or seconds <= 0 - or seconds > MAX_TIMEOUT_SECONDS - ): - raise ValueError("timeout must be finite, positive, and within the limit") - return cls(deadline=monotonic() + seconds) + return cls(deadline=monotonic() + validate_timeout_seconds(seconds)) def remaining_seconds(self) -> float: """Return the positive duration remaining or raise ``TimeoutExpiredError``.""" @@ -55,4 +63,4 @@ def enforce(self) -> Iterator[None]: self.raise_if_expired() -__all__ = ["Timeout"] +__all__ = ["Timeout", "validate_timeout_seconds"] diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index e2a285d..57eebfe 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -78,11 +78,16 @@ async def record_serve(self: PrivacyGuardServer, listen: str) -> None: monkeypatch.setattr(PrivacyGuardServer, "serve", record_serve) - server = PrivacyGuardServer(registry=registry, log_request_content=True) + server = PrivacyGuardServer( + registry=registry, + timeout_seconds=4.5, + log_request_content=True, + ) server.run() assert served == [(server, "127.0.0.1:50051")] assert server._middleware._registry is registry + assert server._middleware._processors._timeout_seconds == 4.5 assert server._middleware._processors._log_request_content is True @@ -91,6 +96,20 @@ def test_programmatic_server_requires_an_explicit_finalized_registry() -> None: PrivacyGuardServer(EngineRegistry()) +@pytest.mark.parametrize("timeout_seconds", [True, 0, 31, float("inf")]) +def test_programmatic_server_rejects_invalid_processing_timeout( + timeout_seconds: bool | int | float, +) -> None: + with pytest.raises( + ValueError, + match="finite number greater than 0 and at most 30", + ): + PrivacyGuardServer( + create_builtin_registry(), + timeout_seconds=timeout_seconds, + ) + + def test_synchronous_server_exits_cleanly_after_keyboard_interrupt( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index 94ce638..df5e40c 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -105,6 +105,21 @@ def test_limit_deny_explains_recovery_options() -> None: assert result.reason == LIMIT_REASON assert "Reduce the request or replacement size" in result.reason assert "simplify the configured stages and patterns" in result.reason + assert "increase --timeout-seconds up to 30" in result.reason + assert "OpenShell middleware timeout is longer" in result.reason + + +def test_middleware_applies_configured_timeout_to_cached_processors() -> None: + middleware = PrivacyGuardMiddleware( + create_builtin_registry(), + timeout_seconds=4.5, + ) + try: + processor = middleware._processors.resolve(_values()) + finally: + asyncio.run(middleware.close()) + + assert processor._timeout_seconds == 4.5 def test_evaluation_decodes_one_utf8_text_and_encodes_replacement() -> None: diff --git a/projects/privacy-guard/tests/test_cli.py b/projects/privacy-guard/tests/test_cli.py index 7a36cc9..d3f1d3d 100644 --- a/projects/privacy-guard/tests/test_cli.py +++ b/projects/privacy-guard/tests/test_cli.py @@ -43,6 +43,16 @@ def test_console_script_targets_the_cli_module() -> None: assert console_script.value == "privacy_guard.cli:app" +def test_cli_serve_help_explains_the_processing_timeout() -> None: + result = CliRunner().invoke(app, ["serve", "--help"]) + + assert result.exit_code == 0 + output = _normalized_output(result) + assert "--timeout-seconds" in output + assert "shared by all processing stages" in output + assert "at most 30" in output + + def test_cli_engines_describes_the_installed_engine() -> None: result = CliRunner().invoke(app, ["engines"]) @@ -99,12 +109,12 @@ def create_registry() -> EngineRegistry: @pytest.mark.parametrize( ("factory_reference", "reason"), [ - ("missing-separator", "module:factory"), - ("operator_engines:missing", "could not be loaded"), - ("operator_engines:not_callable", "not callable"), - ("operator_engines:failed", "factory failed"), - ("operator_engines:wrong_type", "invalid"), - ("operator_engines:unfinished", "unfinalized"), + ("missing-separator", "my_engines:create_registry"), + ("operator_engines:missing", "Verify the module is installed"), + ("operator_engines:not_callable", "Export a callable"), + ("operator_engines:failed", "Run the factory directly"), + ("operator_engines:wrong_type", "Return an EngineRegistry"), + ("operator_engines:unfinished", "Call finalize()"), ], ) def test_cli_rejects_invalid_registry_factories( @@ -130,10 +140,11 @@ def fail() -> EngineRegistry: result = CliRunner().invoke( app, ["--registry-factory", factory_reference, "engines"], + terminal_width=240, ) assert result.exit_code == 2 - assert reason in _plain_output(result) + assert reason in _normalized_output(result) assert "sensitive factory failure" not in _plain_output(result) @@ -141,24 +152,55 @@ def test_cli_serve_adapts_operational_options_to_the_programmatic_server( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: - calls: list[tuple[str, bool]] = [] + calls: list[tuple[str, float, bool]] = [] def record_run(self: PrivacyGuardServer, listen: str) -> None: - calls.append((listen, self._middleware._processors._log_request_content)) + calls.append( + ( + listen, + self._middleware._processors._timeout_seconds, + self._middleware._processors._log_request_content, + ) + ) monkeypatch.setattr(PrivacyGuardServer, "run", record_run) with caplog.at_level(logging.WARNING, logger="privacy_guard.cli"): result = CliRunner().invoke( app, - ["--debug-log-content", "serve", "--listen", "127.0.0.1:50052"], + [ + "--debug-log-content", + "serve", + "--listen", + "127.0.0.1:50052", + "--timeout-seconds", + "4.5", + ], ) assert result.exit_code == 0 - assert calls == [("127.0.0.1:50052", True)] + assert calls == [("127.0.0.1:50052", 4.5, True)] assert "privacy_guard_request_content_logging_enabled" in caplog.text +@pytest.mark.parametrize("timeout_seconds", ["0", "31", "nan"]) +def test_cli_rejects_invalid_processing_timeout(timeout_seconds: str) -> None: + result = CliRunner().invoke( + app, + ["serve", "--timeout-seconds", timeout_seconds], + terminal_width=240, + ) + + assert result.exit_code == 2 + output = _normalized_output(result) + assert "--timeout-seconds" in output + assert "greater than 0 and at most 30" in output + + +def _normalized_output(result: Result) -> str: + return " ".join(_plain_output(result).replace("│", " ").split()) + + def _plain_output(result: Result) -> str: return _ANSI_STYLE_PATTERN.sub("", result.output) diff --git a/projects/privacy-guard/tests/test_timeout.py b/projects/privacy-guard/tests/test_timeout.py index 2e06cc1..7860375 100644 --- a/projects/privacy-guard/tests/test_timeout.py +++ b/projects/privacy-guard/tests/test_timeout.py @@ -12,7 +12,10 @@ def test_timeout_duration_is_strict_positive_and_bounded( seconds: bool | int | float, ) -> None: - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match="finite number greater than 0 and at most 30", + ): Timeout.from_seconds(seconds) @@ -24,7 +27,8 @@ def test_expired_timeout_raises_typed_signal() -> None: assert str(captured.value) == ( "Privacy Guard processing timed out. Reduce the request size or simplify " - "the configured stages and patterns, then retry." + "the configured stages and patterns, or increase the processing timeout " + "to at most 30 seconds, then retry." ) From 9035335e70894ba307f161a1a3609001600d3c9a Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 15:03:47 +0000 Subject: [PATCH 51/82] Clarify Privacy Guard recovery guidance --- .../architecture/safety-and-limits.md | 16 +++++++----- .../architecture/service-boundary.md | 6 +++-- projects/privacy-guard/README.md | 7 +++--- .../privacy-guard/src/privacy_guard/cli.py | 14 ++++++++--- .../src/privacy_guard/constants.py | 8 +++--- .../src/privacy_guard/request_processor.py | 9 ++++++- .../tests/service/test_servicer.py | 9 +++++-- projects/privacy-guard/tests/test_cli.py | 23 ++++++++++++++++- .../tests/test_request_processor.py | 25 +++++++++++++++++++ 9 files changed, 96 insertions(+), 21 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 3dd1666..5af3476 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -56,7 +56,8 @@ until they exit. Operators configure the internal bound with `--timeout-seconds` or the programmatic `PrivacyGuardServer(timeout_seconds=...)` argument. OpenShell's -outer middleware timeout must be longer than this internal bound. +outer middleware timeout must include additional headroom for worker queueing +and configuration or engine preparation beyond this internal bound. ## Diagnostic and result limits @@ -128,11 +129,14 @@ The service returns the same bounded deny when: - replacement text cannot be encoded within the body limit - a deny reason code is not safely representable -The deny reason directs users to reduce the request or replacement size, -simplify the configured stages and patterns, or, when processing timed out, -increase `--timeout-seconds` up to 30 and ensure OpenShell's middleware timeout -is longer before retrying. These options reduce bounded output pressure or -provide appropriate time without weakening the fail-closed behavior. +The deny reason directs users to reduce the request or replacement size and +simplify the configured stages and patterns. Privacy Guard also emits a +content-safe limit kind in its logs. When the log reports a timeout, users can +increase `--timeout-seconds` or +`PrivacyGuardServer(timeout_seconds=...)` up to 30 seconds and give OpenShell's +middleware timeout additional headroom for worker queueing and configuration or +engine preparation. These options reduce bounded output pressure or provide +appropriate time without weakening the fail-closed behavior. Malformed input and engine contract or execution failures instead abort the RPC with a cataloged error. OpenShell then applies the middleware registration's diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index 9ef56e2..df8ec70 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -68,8 +68,10 @@ Each cached processor receives the server's operational processing timeout. The default is 1 second shared across every configured stage. Operators may set up to 30 seconds with `privacy-guard serve --timeout-seconds`; programmatic applications pass the same `timeout_seconds` argument to `PrivacyGuardServer`. -OpenShell's outer middleware `timeout` must be longer than Privacy Guard's -processing timeout so the supervisor does not end the RPC first. +OpenShell's outer middleware `timeout` must include the processing timeout plus +additional headroom for worker queueing, repeated configuration validation, +cache resolution, and engine construction so the supervisor does not end the +RPC first. ## Incoming requests diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index d4ceb0f..f346d90 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -204,9 +204,10 @@ flags. Deployment startup owns only installed engine implementations and operator resources such as model profiles, endpoints, clients, and credentials. The processing timeout is an operational bound shared by every stage; it defaults to 1 second and may be increased to at most 30 seconds. When increasing -it, configure OpenShell's middleware `timeout` to a longer duration so the -supervisor does not end the RPC first. Use `--registry-factory module:factory` -for a custom engine installation. +it, configure OpenShell's middleware `timeout` with additional headroom for +worker queueing and configuration or engine preparation so the supervisor does +not end the RPC first. Use `--registry-factory module:factory` for a custom +engine installation. Registry factories execute operator Python code; load only trusted modules. The `privacy_guard.cli` module owns the executable application and adapts its `serve` command to the programmatic server API. diff --git a/projects/privacy-guard/src/privacy_guard/cli.py b/projects/privacy-guard/src/privacy_guard/cli.py index 144605f..2662f56 100644 --- a/projects/privacy-guard/src/privacy_guard/cli.py +++ b/projects/privacy-guard/src/privacy_guard/cli.py @@ -146,11 +146,19 @@ def _load_registry(factory_reference: str | None) -> EngineRegistry: ) try: module = importlib.import_module(module_name) - factory = getattr(module, factory_name) except Exception: raise typer.BadParameter( - "Registry factory could not be loaded. Verify the module is installed " - "and the module:factory reference is correct.", + "Registry module could not be imported. Verify the module:factory " + "reference, then import the module directly with content-safe " + "diagnostics to find missing dependencies or startup failures.", + param_hint="--registry-factory", + ) from None + try: + factory = getattr(module, factory_name) + except AttributeError: + raise typer.BadParameter( + "Registry module does not export the named factory. Correct the " + "module:factory reference or export that callable.", param_hint="--registry-factory", ) from None if not callable(factory): diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index e7cd4b7..5f684bf 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -20,9 +20,11 @@ BLOCK_REASON_CODE = "privacy_guard_blocked" LIMIT_REASON = ( "Privacy Guard exceeded a processing safety limit. Reduce the request or " - "replacement size, simplify the configured stages and patterns, or, for a " - f"timeout, increase --timeout-seconds up to {MAX_TIMEOUT_SECONDS:g} and ensure " - "the OpenShell middleware timeout is longer, then retry." + "replacement size or simplify the configured stages and patterns, then retry. " + "If Privacy Guard logs report a timeout, increase the processing timeout with " + "--timeout-seconds or PrivacyGuardServer(timeout_seconds=...) to at most " + f"{MAX_TIMEOUT_SECONDS:g} seconds, and give OpenShell's middleware timeout " + "additional headroom for queueing and configuration preparation." ) LIMIT_REASON_CODE = "privacy_guard_limit_exceeded" # Text input limits. diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index 9d3c7f6..fe54546 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -133,7 +133,14 @@ def process(self, text: str) -> RequestProcessingResult: stage_results.append((source, result)) current_text = result.text timeout.raise_if_expired() - except (EngineLimitExceededError, TimeoutExpiredError): + except TimeoutExpiredError: + _LOGGER.info("privacy_guard_processing_limit kind=timeout") + return RequestProcessingResult( + decision=RequestDecision.DENY, + reason_code=LIMIT_REASON_CODE, + ) + except EngineLimitExceededError: + _LOGGER.info("privacy_guard_processing_limit kind=resource") return RequestProcessingResult( decision=RequestDecision.DENY, reason_code=LIMIT_REASON_CODE, diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index df5e40c..da4318c 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -105,8 +105,13 @@ def test_limit_deny_explains_recovery_options() -> None: assert result.reason == LIMIT_REASON assert "Reduce the request or replacement size" in result.reason assert "simplify the configured stages and patterns" in result.reason - assert "increase --timeout-seconds up to 30" in result.reason - assert "OpenShell middleware timeout is longer" in result.reason + assert "If Privacy Guard logs report a timeout" in result.reason + assert "--timeout-seconds or PrivacyGuardServer(timeout_seconds=...)" in ( + result.reason + ) + assert "additional headroom for queueing and configuration preparation" in ( + result.reason + ) def test_middleware_applies_configured_timeout_to_cached_processors() -> None: diff --git a/projects/privacy-guard/tests/test_cli.py b/projects/privacy-guard/tests/test_cli.py index d3f1d3d..945d84b 100644 --- a/projects/privacy-guard/tests/test_cli.py +++ b/projects/privacy-guard/tests/test_cli.py @@ -110,7 +110,7 @@ def create_registry() -> EngineRegistry: ("factory_reference", "reason"), [ ("missing-separator", "my_engines:create_registry"), - ("operator_engines:missing", "Verify the module is installed"), + ("operator_engines:missing", "does not export the named factory"), ("operator_engines:not_callable", "Export a callable"), ("operator_engines:failed", "Run the factory directly"), ("operator_engines:wrong_type", "Return an EngineRegistry"), @@ -148,6 +148,27 @@ def fail() -> EngineRegistry: assert "sensitive factory failure" not in _plain_output(result) +def test_cli_explains_registry_module_import_failures_without_leaking_details( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fail_import(_: str) -> object: + raise RuntimeError("sensitive import failure") + + monkeypatch.setattr(cli_module.importlib, "import_module", fail_import) + + result = CliRunner().invoke( + app, + ["--registry-factory", "operator_engines:create_registry", "engines"], + terminal_width=240, + ) + + assert result.exit_code == 2 + output = _normalized_output(result) + assert "Registry module could not be imported" in output + assert "import the module directly with content-safe diagnostics" in output + assert "sensitive import failure" not in _plain_output(result) + + def test_cli_serve_adapts_operational_options_to_the_programmatic_server( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, diff --git a/projects/privacy-guard/tests/test_request_processor.py b/projects/privacy-guard/tests/test_request_processor.py index 367c704..8f3691c 100644 --- a/projects/privacy-guard/tests/test_request_processor.py +++ b/projects/privacy-guard/tests/test_request_processor.py @@ -2,10 +2,16 @@ from __future__ import annotations +import logging +from time import monotonic + +import pytest + from privacy_guard.config import PolicyAction from privacy_guard.engines import RegexEngine from privacy_guard.engines.registry import EngineRegistry from privacy_guard.request_processor import RequestDecision, RequestProcessor +from privacy_guard.timeout import Timeout def _values(action: PolicyAction) -> dict[str, object]: @@ -107,3 +113,22 @@ def test_block_is_a_processor_disposition_not_an_engine_strategy() -> None: assert result.replacement_text is None assert result.reason_code == "privacy_guard_blocked" assert tuple(item.entity for item in result.detection_summaries) == ("person",) + + +def test_timeout_deny_logs_a_content_safe_diagnostic_kind( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr( + Timeout, + "from_seconds", + classmethod(lambda cls, seconds: cls(deadline=monotonic() - 1)), + ) + + with caplog.at_level(logging.INFO, logger="privacy_guard.request_processor"): + result = _processor(PolicyAction.DETECT).process("Hello Alice") + + assert result.decision is RequestDecision.DENY + assert result.reason_code == "privacy_guard_limit_exceeded" + assert "privacy_guard_processing_limit kind=timeout" in caplog.text + assert "Alice" not in caplog.text From bd5d399cd72ccf58fa0de5cf2c10dd8a2df323e6 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 15:10:26 +0000 Subject: [PATCH 52/82] Correlate Privacy Guard limit diagnostics --- .../architecture/safety-and-limits.md | 6 +- .../privacy-guard/src/privacy_guard/cli.py | 7 +++ .../src/privacy_guard/request_processor.py | 9 ++- .../src/privacy_guard/service/servicer.py | 57 ++++++++++++------- .../tests/examples/test_custom_engine.py | 2 +- .../tests/examples/test_regex_engine.py | 2 +- .../tests/service/test_servicer.py | 56 +++++++++++++++++- projects/privacy-guard/tests/test_cli.py | 26 +++++++++ .../tests/test_request_processor.py | 22 +++++++ 9 files changed, 159 insertions(+), 28 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 5af3476..31696d3 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -130,9 +130,9 @@ The service returns the same bounded deny when: - a deny reason code is not safely representable The deny reason directs users to reduce the request or replacement size and -simplify the configured stages and patterns. Privacy Guard also emits a -content-safe limit kind in its logs. When the log reports a timeout, users can -increase `--timeout-seconds` or +simplify the configured stages and patterns. The request-ID-bearing service +evaluation log also emits a content-safe `timeout` or `resource` limit kind. +When the log reports a timeout, users can increase `--timeout-seconds` or `PrivacyGuardServer(timeout_seconds=...)` up to 30 seconds and give OpenShell's middleware timeout additional headroom for worker queueing and configuration or engine preparation. These options reduce bounded output pressure or provide diff --git a/projects/privacy-guard/src/privacy_guard/cli.py b/projects/privacy-guard/src/privacy_guard/cli.py index 2662f56..4c7c9db 100644 --- a/projects/privacy-guard/src/privacy_guard/cli.py +++ b/projects/privacy-guard/src/privacy_guard/cli.py @@ -161,6 +161,13 @@ def _load_registry(factory_reference: str | None) -> EngineRegistry: "module:factory reference or export that callable.", param_hint="--registry-factory", ) from None + except Exception: + raise typer.BadParameter( + "Registry factory could not be resolved. Import the module and access " + "the factory directly with content-safe diagnostics, then fix its " + "dynamic attribute lookup.", + param_hint="--registry-factory", + ) from None if not callable(factory): raise typer.BadParameter( "Registry factory is not callable. Export a callable that returns a " diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index fe54546..07b937f 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -6,7 +6,7 @@ from collections import OrderedDict from collections.abc import Sequence from enum import StrEnum -from typing import Protocol +from typing import Literal, Protocol from pydantic import Field @@ -60,6 +60,11 @@ class RequestProcessingResult(StrictDomainModel): replacement_text: str | None = Field(default=None, repr=False) detection_summaries: tuple[EntityDetectionSummary, ...] = () reason_code: str | None = None + diagnostic_limit_kind: Literal["timeout", "resource"] | None = Field( + default=None, + exclude=True, + repr=False, + ) class RequestProcessor: @@ -138,12 +143,14 @@ def process(self, text: str) -> RequestProcessingResult: return RequestProcessingResult( decision=RequestDecision.DENY, reason_code=LIMIT_REASON_CODE, + diagnostic_limit_kind="timeout", ) except EngineLimitExceededError: _LOGGER.info("privacy_guard_processing_limit kind=resource") return RequestProcessingResult( decision=RequestDecision.DENY, reason_code=LIMIT_REASON_CODE, + diagnostic_limit_kind="resource", ) except EngineContractError: raise PrivacyGuardError(ErrorCode.ENGINE_OUTPUT_INVALID) from None diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index baba477..ca2b453 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -144,8 +144,9 @@ async def _evaluate_rpc( failure: PrivacyGuardError | None = None action = "error" finding_count = 0 + limit_kind: str | None = None try: - response = await self._evaluate_http_request(request) + response, limit_kind = await self._evaluate_http_request(request) action = "allow" if response.decision == pb2.DECISION_ALLOW else "deny" finding_count = sum(finding.count for finding in response.findings) return response @@ -159,15 +160,17 @@ async def _evaluate_rpc( started=started, action=action, finding_count=finding_count, + limit_kind=limit_kind, failure=failure, ) _LOGGER.info( "privacy_guard_evaluation request_id=%s duration_ms=%.3f " - "action=%s finding_count=%d error_code=%s", + "action=%s finding_count=%d limit_kind=%s error_code=%s", log_extra["request_id"], log_extra["duration_ms"], log_extra["action"], log_extra["finding_count"], + log_extra["limit_kind"] or "none", log_extra["error_code"] or "none", extra=log_extra, ) @@ -181,7 +184,7 @@ async def _evaluate_rpc( async def _evaluate_http_request( self, request: pb2.HttpRequestEvaluation, - ) -> pb2.HttpRequestResult: + ) -> tuple[pb2.HttpRequestResult, str | None]: if request.phase != pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS: raise PrivacyGuardError(ErrorCode.REQUEST_PHASE_INVALID) if len(request.body) > MAX_BODY_BYTES: @@ -296,6 +299,7 @@ class _EvaluationLogExtra(TypedDict): duration_ms: float action: str finding_count: int + limit_kind: str | None error_code: str | None @@ -305,6 +309,7 @@ def _evaluation_log_extra( started: float, action: str, finding_count: int, + limit_kind: str | None, failure: PrivacyGuardError | None, ) -> _EvaluationLogExtra: return { @@ -312,6 +317,7 @@ def _evaluation_log_extra( "duration_ms": round((time.monotonic() - started) * 1000, 3), "action": action, "finding_count": finding_count, + "limit_kind": limit_kind, "error_code": failure.code.value if failure is not None else None, } @@ -323,7 +329,9 @@ def _mapping_from_proto(config: Message) -> dict[str, object]: raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None -def _result_to_proto(result: RequestProcessingResult) -> pb2.HttpRequestResult: +def _result_to_proto( + result: RequestProcessingResult, +) -> tuple[pb2.HttpRequestResult, str | None]: findings: list[pb2.Finding] = [] for detection in result.detection_summaries: finding = _detection_to_proto(detection) @@ -339,21 +347,29 @@ def _result_to_proto(result: RequestProcessingResult) -> pb2.HttpRequestResult: ) if len(replacement_body) > MAX_BODY_BYTES: return _limit_deny() - return pb2.HttpRequestResult( - decision=pb2.DECISION_ALLOW, - body=replacement_body, - has_body=replacement is not None, - findings=findings, + return ( + pb2.HttpRequestResult( + decision=pb2.DECISION_ALLOW, + body=replacement_body, + has_body=replacement is not None, + findings=findings, + ), + None, ) if result.decision is RequestDecision.DENY: reason_code = result.reason_code or BLOCK_REASON_CODE if REASON_CODE_PATTERN.fullmatch(reason_code) is None: return _limit_deny() - return pb2.HttpRequestResult( - decision=pb2.DECISION_DENY, - reason=LIMIT_REASON if reason_code == LIMIT_REASON_CODE else BLOCK_REASON, - reason_code=reason_code, - findings=findings, + return ( + pb2.HttpRequestResult( + decision=pb2.DECISION_DENY, + reason=LIMIT_REASON + if reason_code == LIMIT_REASON_CODE + else BLOCK_REASON, + reason_code=reason_code, + findings=findings, + ), + result.diagnostic_limit_kind, ) raise PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) @@ -370,11 +386,14 @@ def _detection_to_proto(detection: EntityDetectionSummary) -> pb2.Finding: return result -def _limit_deny() -> pb2.HttpRequestResult: - return pb2.HttpRequestResult( - decision=pb2.DECISION_DENY, - reason=LIMIT_REASON, - reason_code=LIMIT_REASON_CODE, +def _limit_deny() -> tuple[pb2.HttpRequestResult, str]: + return ( + pb2.HttpRequestResult( + decision=pb2.DECISION_DENY, + reason=LIMIT_REASON, + reason_code=LIMIT_REASON_CODE, + ), + "resource", ) diff --git a/projects/privacy-guard/tests/examples/test_custom_engine.py b/projects/privacy-guard/tests/examples/test_custom_engine.py index 320f89b..8738dcd 100644 --- a/projects/privacy-guard/tests/examples/test_custom_engine.py +++ b/projects/privacy-guard/tests/examples/test_custom_engine.py @@ -35,7 +35,7 @@ def test_custom_engine_runs_through_the_middleware_boundary() -> None: async def evaluate() -> None: middleware = PrivacyGuardMiddleware(create_registry()) try: - result = await middleware._evaluate_http_request( + result, _ = await middleware._evaluate_http_request( pb2.HttpRequestEvaluation( phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, config=config, diff --git a/projects/privacy-guard/tests/examples/test_regex_engine.py b/projects/privacy-guard/tests/examples/test_regex_engine.py index e554ef1..9f2111b 100644 --- a/projects/privacy-guard/tests/examples/test_regex_engine.py +++ b/projects/privacy-guard/tests/examples/test_regex_engine.py @@ -35,7 +35,7 @@ def test_regex_example_runs_through_the_middleware_boundary( async def evaluate() -> None: middleware = PrivacyGuardMiddleware(create_builtin_registry()) try: - result = await middleware._evaluate_http_request( + result, _ = await middleware._evaluate_http_request( pb2.HttpRequestEvaluation( phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, config=config, diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index da4318c..9ebd53b 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -3,8 +3,11 @@ from __future__ import annotations import asyncio +import logging from threading import get_ident +from typing import Never +import grpc import pytest from google.protobuf import json_format from google.protobuf.message import Message @@ -70,6 +73,11 @@ def _request(body: bytes, *, action: str = "replace") -> pb2.HttpRequestEvaluati ) +class _UnusedAbortContext: + async def abort(self, code: grpc.StatusCode, details: str) -> Never: + raise AssertionError(f"unexpected abort: {code.name}: {details}") + + def test_copied_proto_remains_the_current_openshell_contract() -> None: evaluation = pb2.HttpRequestEvaluation() finding = pb2.Finding() @@ -95,13 +103,15 @@ def test_validate_config_is_pure_and_reports_invalid_config() -> None: def test_limit_deny_explains_recovery_options() -> None: - result = servicer_module._result_to_proto( + result, limit_kind = servicer_module._result_to_proto( RequestProcessingResult( decision=RequestDecision.DENY, reason_code=LIMIT_REASON_CODE, + diagnostic_limit_kind="timeout", ) ) + assert limit_kind == "timeout" assert result.reason == LIMIT_REASON assert "Reduce the request or replacement size" in result.reason assert "simplify the configured stages and patterns" in result.reason @@ -131,7 +141,10 @@ def test_evaluation_decodes_one_utf8_text_and_encodes_replacement() -> None: async def evaluate() -> pb2.HttpRequestResult: middleware = PrivacyGuardMiddleware(create_builtin_registry()) try: - return await middleware._evaluate_http_request(_request(b"email a@b.com")) + result, _ = await middleware._evaluate_http_request( + _request(b"email a@b.com") + ) + return result finally: await middleware.close() @@ -229,9 +242,10 @@ def test_detect_returns_no_body_mutation() -> None: async def evaluate() -> pb2.HttpRequestResult: middleware = PrivacyGuardMiddleware(create_builtin_registry()) try: - return await middleware._evaluate_http_request( + result, _ = await middleware._evaluate_http_request( _request(b"a@b.com", action="detect") ) + return result finally: await middleware.close() @@ -240,3 +254,39 @@ async def evaluate() -> pb2.HttpRequestResult: assert result.decision == pb2.DECISION_ALLOW assert result.has_body is False assert result.body == b"" + + +def test_service_resource_limit_log_is_content_safe_and_request_correlated( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + async def evaluate() -> pb2.HttpRequestResult: + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + + async def return_resource_limit( + _: pb2.HttpRequestEvaluation, + ) -> tuple[pb2.HttpRequestResult, str]: + return servicer_module._limit_deny() + + monkeypatch.setattr( + middleware, + "_evaluate_http_request", + return_resource_limit, + ) + request = _request(b"sensitive request content") + request.context.request_id = "request-123" + try: + return await middleware._evaluate_rpc( + request, + _UnusedAbortContext(), + ) + finally: + await middleware.close() + + with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): + result = asyncio.run(evaluate()) + + assert result.reason_code == LIMIT_REASON_CODE + assert "request_id=request-123" in caplog.text + assert "limit_kind=resource" in caplog.text + assert "sensitive request content" not in caplog.text diff --git a/projects/privacy-guard/tests/test_cli.py b/projects/privacy-guard/tests/test_cli.py index 945d84b..a358a27 100644 --- a/projects/privacy-guard/tests/test_cli.py +++ b/projects/privacy-guard/tests/test_cli.py @@ -169,6 +169,32 @@ def fail_import(_: str) -> object: assert "sensitive import failure" not in _plain_output(result) +def test_cli_translates_dynamic_registry_factory_lookup_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class DynamicModule: + def __getattr__(self, _: str) -> object: + raise RuntimeError("sensitive dynamic lookup failure") + + monkeypatch.setattr( + cli_module.importlib, + "import_module", + lambda _: DynamicModule(), + ) + + result = CliRunner().invoke( + app, + ["--registry-factory", "operator_engines:create_registry", "engines"], + terminal_width=240, + ) + + assert result.exit_code == 2 + output = _normalized_output(result) + assert "Registry factory could not be resolved" in output + assert "fix its dynamic attribute lookup" in output + assert "sensitive dynamic lookup failure" not in _plain_output(result) + + def test_cli_serve_adapts_operational_options_to_the_programmatic_server( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, diff --git a/projects/privacy-guard/tests/test_request_processor.py b/projects/privacy-guard/tests/test_request_processor.py index 8f3691c..7f8dd79 100644 --- a/projects/privacy-guard/tests/test_request_processor.py +++ b/projects/privacy-guard/tests/test_request_processor.py @@ -10,6 +10,7 @@ from privacy_guard.config import PolicyAction from privacy_guard.engines import RegexEngine from privacy_guard.engines.registry import EngineRegistry +from privacy_guard.errors import EngineLimitExceededError from privacy_guard.request_processor import RequestDecision, RequestProcessor from privacy_guard.timeout import Timeout @@ -130,5 +131,26 @@ def test_timeout_deny_logs_a_content_safe_diagnostic_kind( assert result.decision is RequestDecision.DENY assert result.reason_code == "privacy_guard_limit_exceeded" + assert result.diagnostic_limit_kind == "timeout" assert "privacy_guard_processing_limit kind=timeout" in caplog.text assert "Alice" not in caplog.text + + +def test_resource_limit_deny_logs_a_content_safe_diagnostic_kind( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + def exceed_limit(*_: object, **__: object) -> object: + raise EngineLimitExceededError("sensitive resource detail") + + monkeypatch.setattr(RegexEngine, "_run", exceed_limit) + + with caplog.at_level(logging.INFO, logger="privacy_guard.request_processor"): + result = _processor(PolicyAction.DETECT).process("Hello Alice") + + assert result.decision is RequestDecision.DENY + assert result.reason_code == "privacy_guard_limit_exceeded" + assert result.diagnostic_limit_kind == "resource" + assert "privacy_guard_processing_limit kind=resource" in caplog.text + assert "Alice" not in caplog.text + assert "sensitive resource detail" not in caplog.text From 5cd330031d1bbe827189358769c1d50e62003f0d Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 15:25:36 +0000 Subject: [PATCH 53/82] Simplify Privacy Guard error and command surfaces --- .../privacy-guard/architecture/index.md | 4 +- .../architecture/safety-and-limits.md | 25 ++++---- .../architecture/service-boundary.md | 15 ++--- projects/privacy-guard/AGENTS.md | 2 +- projects/privacy-guard/README.md | 12 ++-- .../examples/custom-engine/README.md | 2 +- .../examples/regex-engine/README.md | 2 +- .../privacy-guard/src/privacy_guard/cli.py | 35 ++++++----- .../src/privacy_guard/constants.py | 11 ++-- .../privacy-guard/src/privacy_guard/errors.py | 17 +++--- .../src/privacy_guard/request_processor.py | 11 +--- .../src/privacy_guard/service/server.py | 14 ++--- .../src/privacy_guard/service/servicer.py | 61 ++++++------------- .../tests/examples/test_custom_engine.py | 4 +- .../tests/examples/test_regex_engine.py | 4 +- .../tests/service/test_server.py | 33 +++++----- .../tests/service/test_servicer.py | 57 +---------------- projects/privacy-guard/tests/test_cli.py | 60 +++++++++--------- .../tests/test_request_processor.py | 20 ++---- 19 files changed, 149 insertions(+), 240 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/index.md b/docs/documentation/privacy-guard/architecture/index.md index c4c5aa0..0614b50 100644 --- a/docs/documentation/privacy-guard/architecture/index.md +++ b/docs/documentation/privacy-guard/architecture/index.md @@ -55,8 +55,8 @@ Source paths on these pages are relative to Outside generated `bindings/`, no other package imports gRPC or generated bindings. - `cli.py` owns command parsing, registry-factory loading, engine discovery, - schema output, logging options, and the adapter that starts the programmatic - server. + configuration-schema output, logging options, and the adapter that starts + the programmatic server. - `request_processor.py` runs configured stages over one text value, shares one timeout across them, aggregates detections, and applies the user-facing policy action. It does not import gRPC or implement an engine's algorithms. diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 31696d3..95e7cf9 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -129,14 +129,13 @@ The service returns the same bounded deny when: - replacement text cannot be encoded within the body limit - a deny reason code is not safely representable -The deny reason directs users to reduce the request or replacement size and -simplify the configured stages and patterns. The request-ID-bearing service -evaluation log also emits a content-safe `timeout` or `resource` limit kind. -When the log reports a timeout, users can increase `--timeout-seconds` or -`PrivacyGuardServer(timeout_seconds=...)` up to 30 seconds and give OpenShell's -middleware timeout additional headroom for worker queueing and configuration or -engine preparation. These options reduce bounded output pressure or provide -appropriate time without weakening the fail-closed behavior. +The deny reason directs users to reduce the request or replacement size, +simplify the configured stages and patterns, or increase `--timeout-seconds` or +`PrivacyGuardServer(timeout_seconds=...)` up to 30 seconds. When increasing the +processing timeout, users must give OpenShell's middleware timeout additional +headroom for worker queueing and configuration or engine preparation. These +options reduce bounded output pressure or provide appropriate time without +weakening the fail-closed behavior. Malformed input and engine contract or execution failures instead abort the RPC with a cataloged error. OpenShell then applies the middleware registration's @@ -160,10 +159,12 @@ error spec defines: - remediation hint Successful policy and safety-limit denials use stable reason codes plus -content-safe recovery guidance. CLI validation failures name the rejected -option and state the accepted form or next action. Internal engine and -third-party failures remain concise because the owning boundary translates -them before they reach users. +content-safe recovery guidance; they do not add domain fields solely for +diagnostic logging. CLI validation failures name the rejected option and state +the accepted form or next action. Cataloged server-startup failures print one +content-safe message and exit nonzero rather than exposing a traceback. +Internal engine and third-party failures remain concise because the owning +boundary translates them before they reach users. Caught extension and third-party exceptions are translated at their trust boundary. Raw exception messages and chains are not exposed. Configuration diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index df8ec70..a5147e8 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -31,7 +31,7 @@ are invalid input. The current manifest message has no field for the finalized policy schema or engine discovery metadata. Those are available through the local -`privacy-guard schema` and `privacy-guard engines` commands. +`privacy-guard configuration-schema` and `privacy-guard engines` commands. ## Configuration lifecycle @@ -217,11 +217,12 @@ server = PrivacyGuardServer( create_builtin_registry(), timeout_seconds=5, ) -server.run("127.0.0.1:50051") +server.serve_sync("127.0.0.1:50051") ``` -Async applications call `await server.serve(address)` instead. Both entry -points use the same server instance and lifecycle. +Async applications call `await server.serve_async(address)` instead. The +explicit suffixes make the blocking behavior visible at each call site. Both +entry points use the same server instance and lifecycle. The server: @@ -245,7 +246,7 @@ imports the module, calls the function once, and uses the returned finalized ```bash privacy-guard --registry-factory my_engines:create_registry engines -privacy-guard --registry-factory my_engines:create_registry schema +privacy-guard --registry-factory my_engines:create_registry configuration-schema privacy-guard --registry-factory my_engines:create_registry serve ``` @@ -259,14 +260,14 @@ option. They create the registry normally and pass it to the server: ```python registry = create_registry() -PrivacyGuardServer(registry, timeout_seconds=5).run() +PrivacyGuardServer(registry, timeout_seconds=5).serve_sync() ``` The CLI exposes: ```bash privacy-guard engines -privacy-guard schema +privacy-guard configuration-schema privacy-guard serve --listen 127.0.0.1:50051 --timeout-seconds 5 ``` diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md index 6880806..b742ebd 100644 --- a/projects/privacy-guard/AGENTS.md +++ b/projects/privacy-guard/AGENTS.md @@ -33,7 +33,7 @@ Run focused tests while working and `make check` before handoff. - `src/privacy_guard/engines/`: engine contract, registry, and built-in implementations - `src/privacy_guard/config.py`: policy action and ordered stage configuration - `src/privacy_guard/request_processor.py`: stage execution and policy disposition -- `src/privacy_guard/cli.py`: command parsing, discovery, schema output, and server adapter +- `src/privacy_guard/cli.py`: command parsing, discovery, configuration-schema output, and server adapter - `src/privacy_guard/base.py`: package-wide strict immutable domain-model base - `src/privacy_guard/string_validators.py`: shared string validators and field types - `src/privacy_guard/service/`: gRPC lifecycle and protobuf adapter diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index f346d90..4682332 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -137,7 +137,7 @@ or otherwise present on Python's import path: ```bash uv run privacy-guard --registry-factory my_engines:create_registry engines -uv run privacy-guard --registry-factory my_engines:create_registry schema +uv run privacy-guard --registry-factory my_engines:create_registry configuration-schema uv run privacy-guard --registry-factory my_engines:create_registry serve ``` @@ -174,14 +174,14 @@ server = PrivacyGuardServer( create_builtin_registry(), timeout_seconds=5, ) -server.run("127.0.0.1:50051") +server.serve_sync("127.0.0.1:50051") ``` -`run()` is the blocking synchronous entry point. Async applications can use the -same server directly: +`serve_sync()` is the blocking entry point. Async applications use the +explicitly named asynchronous counterpart: ```python -await server.serve("127.0.0.1:50051") +await server.serve_async("127.0.0.1:50051") ``` Custom applications pass their own finalized registry to @@ -193,7 +193,7 @@ modules, select engines, generate schemas, or parse command-line options. ```bash uv run privacy-guard engines -uv run privacy-guard schema +uv run privacy-guard configuration-schema uv run privacy-guard serve \ --listen 127.0.0.1:50051 \ --timeout-seconds 5 diff --git a/projects/privacy-guard/examples/custom-engine/README.md b/projects/privacy-guard/examples/custom-engine/README.md index a8b2dd8..7d9d538 100644 --- a/projects/privacy-guard/examples/custom-engine/README.md +++ b/projects/privacy-guard/examples/custom-engine/README.md @@ -70,7 +70,7 @@ uv run privacy-guard \ uv run privacy-guard \ --registry-factory custom_engine:create_registry \ - schema + configuration-schema ``` The first command should print the built-in `regex` row with `detect,replace` diff --git a/projects/privacy-guard/examples/regex-engine/README.md b/projects/privacy-guard/examples/regex-engine/README.md index cec2b74..f1ea81a 100644 --- a/projects/privacy-guard/examples/regex-engine/README.md +++ b/projects/privacy-guard/examples/regex-engine/README.md @@ -46,7 +46,7 @@ uv sync --locked ```bash uv run privacy-guard engines -uv run privacy-guard schema +uv run privacy-guard configuration-schema ``` The first command should print one `regex` row with `detect,replace`. The schema diff --git a/projects/privacy-guard/src/privacy_guard/cli.py b/projects/privacy-guard/src/privacy_guard/cli.py index 4c7c9db..4a33451 100644 --- a/projects/privacy-guard/src/privacy_guard/cli.py +++ b/projects/privacy-guard/src/privacy_guard/cli.py @@ -13,6 +13,7 @@ from privacy_guard.constants import DEFAULT_TIMEOUT_SECONDS, MAX_TIMEOUT_SECONDS from privacy_guard.engines import EntityProcessingStrategy from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry +from privacy_guard.errors import PrivacyGuardError from privacy_guard.service.server import DEFAULT_LISTEN_ADDRESS, PrivacyGuardServer from privacy_guard.timeout import validate_timeout_seconds @@ -92,15 +93,19 @@ def serve( str(error), param_hint="--timeout-seconds", ) from None - PrivacyGuardServer( - options.registry, - timeout_seconds=validated_timeout_seconds, - log_request_content=options.log_request_content, - ).run(listen) - - -@app.command("schema") -def schema(context: typer.Context) -> None: + try: + PrivacyGuardServer( + options.registry, + timeout_seconds=validated_timeout_seconds, + log_request_content=options.log_request_content, + ).serve_sync(listen) + except PrivacyGuardError as error: + typer.echo(str(error), err=True) + raise typer.Exit(code=1) from None + + +@app.command("configuration-schema") +def configuration_schema(context: typer.Context) -> None: """Print the exact finalized policy JSON Schema.""" typer.echo( json.dumps( @@ -155,17 +160,11 @@ def _load_registry(factory_reference: str | None) -> EngineRegistry: ) from None try: factory = getattr(module, factory_name) - except AttributeError: - raise typer.BadParameter( - "Registry module does not export the named factory. Correct the " - "module:factory reference or export that callable.", - param_hint="--registry-factory", - ) from None except Exception: raise typer.BadParameter( - "Registry factory could not be resolved. Import the module and access " - "the factory directly with content-safe diagnostics, then fix its " - "dynamic attribute lookup.", + "Registry factory could not be resolved. Verify the module:factory " + "reference and exported callable, then access it directly with " + "content-safe diagnostics.", param_hint="--registry-factory", ) from None if not callable(factory): diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index 5f684bf..8957d45 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -20,11 +20,12 @@ BLOCK_REASON_CODE = "privacy_guard_blocked" LIMIT_REASON = ( "Privacy Guard exceeded a processing safety limit. Reduce the request or " - "replacement size or simplify the configured stages and patterns, then retry. " - "If Privacy Guard logs report a timeout, increase the processing timeout with " - "--timeout-seconds or PrivacyGuardServer(timeout_seconds=...) to at most " - f"{MAX_TIMEOUT_SECONDS:g} seconds, and give OpenShell's middleware timeout " - "additional headroom for queueing and configuration preparation." + "replacement size, simplify the configured stages and patterns, or increase " + "the processing timeout with --timeout-seconds or " + "PrivacyGuardServer(timeout_seconds=...) to at most " + f"{MAX_TIMEOUT_SECONDS:g} seconds. If increasing it, give OpenShell's " + "middleware timeout additional headroom for queueing and configuration " + "preparation, then retry." ) LIMIT_REASON_CODE = "privacy_guard_limit_exceeded" # Text input limits. diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py index 1da6256..0e43b22 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -125,8 +125,8 @@ class EngineRegistryError(Exception): ErrorComponent.CONFIG, "parse", "Policy configuration is invalid.", - "Check entity-processing stages, engine configuration, pattern catalogs, " - "replacement recipes, and the on-detection action.", + "Compare it with `privacy-guard configuration-schema`, then check the " + "stages, engine settings, pattern catalogs, replacements, and action.", ), ErrorCode.REQUEST_PHASE_INVALID: _ErrorSpec( ErrorKind.INVALID_INPUT, @@ -154,27 +154,30 @@ class EngineRegistryError(Exception): ErrorComponent.PROCESSOR, "validate_engine", "An entity-processing engine returned an invalid result.", - "Check the engine run contract, result model, spans, strategy, and limits.", + "Custom engine developers should check the run contract, result model, " + "spans, processing strategy, and output limits.", ), ErrorCode.ENGINE_EXECUTION_FAILED: _ErrorSpec( ErrorKind.INTERNAL, ErrorComponent.ENGINE, "run", "An entity-processing engine failed.", - "Run the engine's focused configuration and single-text tests.", + "Check the request ID and error code in service logs, then run the " + "configured engine's focused configuration and single-text tests.", ), ErrorCode.SERVER_BIND_FAILED: _ErrorSpec( ErrorKind.INTERNAL, ErrorComponent.SERVER, "bind", - "Server could not bind its listen address.", - "Check the listen address and port availability.", + "Server could not start on its listen address.", + "Choose an available listen address and port, then retry.", ), ErrorCode.UNEXPECTED_SERVICE_FAILURE: _ErrorSpec( ErrorKind.INTERNAL, ErrorComponent.SERVICE, "evaluate_http_request", "The middleware encountered an unexpected failure.", - "Reproduce with focused service and processor tests.", + "Retry once; if it recurs, report the request ID and error code from the " + "service log.", ), } diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index 07b937f..8c11b45 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -6,7 +6,7 @@ from collections import OrderedDict from collections.abc import Sequence from enum import StrEnum -from typing import Literal, Protocol +from typing import Protocol from pydantic import Field @@ -60,11 +60,6 @@ class RequestProcessingResult(StrictDomainModel): replacement_text: str | None = Field(default=None, repr=False) detection_summaries: tuple[EntityDetectionSummary, ...] = () reason_code: str | None = None - diagnostic_limit_kind: Literal["timeout", "resource"] | None = Field( - default=None, - exclude=True, - repr=False, - ) class RequestProcessor: @@ -139,18 +134,14 @@ def process(self, text: str) -> RequestProcessingResult: current_text = result.text timeout.raise_if_expired() except TimeoutExpiredError: - _LOGGER.info("privacy_guard_processing_limit kind=timeout") return RequestProcessingResult( decision=RequestDecision.DENY, reason_code=LIMIT_REASON_CODE, - diagnostic_limit_kind="timeout", ) except EngineLimitExceededError: - _LOGGER.info("privacy_guard_processing_limit kind=resource") return RequestProcessingResult( decision=RequestDecision.DENY, reason_code=LIMIT_REASON_CODE, - diagnostic_limit_kind="resource", ) except EngineContractError: raise PrivacyGuardError(ErrorCode.ENGINE_OUTPUT_INVALID) from None diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index 2cccd2c..de09195 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -36,25 +36,25 @@ def __init__( log_request_content=log_request_content, ) - def run(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: - """Run synchronously until termination.""" + def serve_sync(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: + """Serve synchronously until termination.""" try: - asyncio.run(self.serve(listen)) + asyncio.run(self.serve_async(listen)) except KeyboardInterrupt: return - async def serve(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: + async def serve_async(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: """Serve asynchronously until termination, then close owned resources.""" server = _create_grpc_server(self._middleware) _LOGGER.info("privacy_guard_server_starting listen=%s", listen) try: try: bound_port = server.add_insecure_port(listen) + if bound_port == 0: + raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) + await server.start() except RuntimeError: raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) from None - if bound_port == 0: - raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) - await server.start() await server.wait_for_termination() finally: try: diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index ca2b453..1c87ae4 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -14,7 +14,6 @@ import grpc from google.protobuf import json_format from google.protobuf.message import Message -from typing_extensions import override from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc @@ -78,7 +77,6 @@ async def close(self) -> None: """Wait for in-flight synchronous engines during shutdown.""" self._processing_executor.shutdown(wait=True, cancel_futures=True) - @override async def Describe( self, request: object, @@ -97,7 +95,6 @@ async def Describe( ], ) - @override async def ValidateConfig( self, request: pb2.ValidateConfigRequest, @@ -109,7 +106,6 @@ async def ValidateConfig( """Validate expanded configuration without preparing runtime state.""" return await self._run_in_worker(lambda: self._validate_config(request)) - @override async def EvaluateHttpRequest( self, request: pb2.HttpRequestEvaluation, @@ -144,9 +140,8 @@ async def _evaluate_rpc( failure: PrivacyGuardError | None = None action = "error" finding_count = 0 - limit_kind: str | None = None try: - response, limit_kind = await self._evaluate_http_request(request) + response = await self._evaluate_http_request(request) action = "allow" if response.decision == pb2.DECISION_ALLOW else "deny" finding_count = sum(finding.count for finding in response.findings) return response @@ -160,17 +155,15 @@ async def _evaluate_rpc( started=started, action=action, finding_count=finding_count, - limit_kind=limit_kind, failure=failure, ) _LOGGER.info( "privacy_guard_evaluation request_id=%s duration_ms=%.3f " - "action=%s finding_count=%d limit_kind=%s error_code=%s", + "action=%s finding_count=%d error_code=%s", log_extra["request_id"], log_extra["duration_ms"], log_extra["action"], log_extra["finding_count"], - log_extra["limit_kind"] or "none", log_extra["error_code"] or "none", extra=log_extra, ) @@ -184,7 +177,7 @@ async def _evaluate_rpc( async def _evaluate_http_request( self, request: pb2.HttpRequestEvaluation, - ) -> tuple[pb2.HttpRequestResult, str | None]: + ) -> pb2.HttpRequestResult: if request.phase != pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS: raise PrivacyGuardError(ErrorCode.REQUEST_PHASE_INVALID) if len(request.body) > MAX_BODY_BYTES: @@ -299,7 +292,6 @@ class _EvaluationLogExtra(TypedDict): duration_ms: float action: str finding_count: int - limit_kind: str | None error_code: str | None @@ -309,7 +301,6 @@ def _evaluation_log_extra( started: float, action: str, finding_count: int, - limit_kind: str | None, failure: PrivacyGuardError | None, ) -> _EvaluationLogExtra: return { @@ -317,7 +308,6 @@ def _evaluation_log_extra( "duration_ms": round((time.monotonic() - started) * 1000, 3), "action": action, "finding_count": finding_count, - "limit_kind": limit_kind, "error_code": failure.code.value if failure is not None else None, } @@ -329,9 +319,7 @@ def _mapping_from_proto(config: Message) -> dict[str, object]: raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None -def _result_to_proto( - result: RequestProcessingResult, -) -> tuple[pb2.HttpRequestResult, str | None]: +def _result_to_proto(result: RequestProcessingResult) -> pb2.HttpRequestResult: findings: list[pb2.Finding] = [] for detection in result.detection_summaries: finding = _detection_to_proto(detection) @@ -347,29 +335,21 @@ def _result_to_proto( ) if len(replacement_body) > MAX_BODY_BYTES: return _limit_deny() - return ( - pb2.HttpRequestResult( - decision=pb2.DECISION_ALLOW, - body=replacement_body, - has_body=replacement is not None, - findings=findings, - ), - None, + return pb2.HttpRequestResult( + decision=pb2.DECISION_ALLOW, + body=replacement_body, + has_body=replacement is not None, + findings=findings, ) if result.decision is RequestDecision.DENY: reason_code = result.reason_code or BLOCK_REASON_CODE if REASON_CODE_PATTERN.fullmatch(reason_code) is None: return _limit_deny() - return ( - pb2.HttpRequestResult( - decision=pb2.DECISION_DENY, - reason=LIMIT_REASON - if reason_code == LIMIT_REASON_CODE - else BLOCK_REASON, - reason_code=reason_code, - findings=findings, - ), - result.diagnostic_limit_kind, + return pb2.HttpRequestResult( + decision=pb2.DECISION_DENY, + reason=LIMIT_REASON if reason_code == LIMIT_REASON_CODE else BLOCK_REASON, + reason_code=reason_code, + findings=findings, ) raise PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) @@ -386,14 +366,11 @@ def _detection_to_proto(detection: EntityDetectionSummary) -> pb2.Finding: return result -def _limit_deny() -> tuple[pb2.HttpRequestResult, str]: - return ( - pb2.HttpRequestResult( - decision=pb2.DECISION_DENY, - reason=LIMIT_REASON, - reason_code=LIMIT_REASON_CODE, - ), - "resource", +def _limit_deny() -> pb2.HttpRequestResult: + return pb2.HttpRequestResult( + decision=pb2.DECISION_DENY, + reason=LIMIT_REASON, + reason_code=LIMIT_REASON_CODE, ) diff --git a/projects/privacy-guard/tests/examples/test_custom_engine.py b/projects/privacy-guard/tests/examples/test_custom_engine.py index 8738dcd..195e7ef 100644 --- a/projects/privacy-guard/tests/examples/test_custom_engine.py +++ b/projects/privacy-guard/tests/examples/test_custom_engine.py @@ -35,7 +35,7 @@ def test_custom_engine_runs_through_the_middleware_boundary() -> None: async def evaluate() -> None: middleware = PrivacyGuardMiddleware(create_registry()) try: - result, _ = await middleware._evaluate_http_request( + result = await middleware._evaluate_http_request( pb2.HttpRequestEvaluation( phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, config=config, @@ -86,7 +86,7 @@ def test_custom_registry_drives_cli_discovery_and_schema() -> None: env=environment, ) schema = subprocess.run( - [*command, "schema"], + [*command, "configuration-schema"], cwd=EXAMPLE_DIRECTORY, check=True, capture_output=True, diff --git a/projects/privacy-guard/tests/examples/test_regex_engine.py b/projects/privacy-guard/tests/examples/test_regex_engine.py index 9f2111b..7ef63ca 100644 --- a/projects/privacy-guard/tests/examples/test_regex_engine.py +++ b/projects/privacy-guard/tests/examples/test_regex_engine.py @@ -35,7 +35,7 @@ def test_regex_example_runs_through_the_middleware_boundary( async def evaluate() -> None: middleware = PrivacyGuardMiddleware(create_builtin_registry()) try: - result, _ = await middleware._evaluate_http_request( + result = await middleware._evaluate_http_request( pb2.HttpRequestEvaluation( phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, config=config, @@ -66,7 +66,7 @@ def test_builtin_registry_drives_documented_cli_discovery_and_schema() -> None: text=True, ) schema = subprocess.run( - [command, "schema"], + [command, "configuration-schema"], cwd=EXAMPLE_DIRECTORY, check=True, capture_output=True, diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 57eebfe..13a23c7 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -76,14 +76,14 @@ async def record_serve(self: PrivacyGuardServer, listen: str) -> None: served.append((self, listen)) await self._middleware.close() - monkeypatch.setattr(PrivacyGuardServer, "serve", record_serve) + monkeypatch.setattr(PrivacyGuardServer, "serve_async", record_serve) server = PrivacyGuardServer( registry=registry, timeout_seconds=4.5, log_request_content=True, ) - server.run() + server.serve_sync() assert served == [(server, "127.0.0.1:50051")] assert server._middleware._registry is registry @@ -119,9 +119,9 @@ async def interrupt(self: PrivacyGuardServer, listen: str) -> None: del self, listen raise KeyboardInterrupt - monkeypatch.setattr(PrivacyGuardServer, "serve", interrupt) + monkeypatch.setattr(PrivacyGuardServer, "serve_async", interrupt) - server.run() + server.serve_sync() asyncio.run(server._middleware.close()) @@ -204,7 +204,7 @@ def record_registration( ), ], ) -async def test_serve_sanitizes_bind_failures_and_closes_resources( +async def test_serve_async_sanitizes_bind_failures_and_closes_resources( monkeypatch: pytest.MonkeyPatch, fake_server: _LifecycleServerFake, sensitive_address: str, @@ -219,7 +219,7 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) with pytest.raises(PrivacyGuardError) as captured: - await server.serve(sensitive_address) + await server.serve_async(sensitive_address) assert captured.value.code is ErrorCode.SERVER_BIND_FAILED assert captured.value.__cause__ is None @@ -231,7 +231,7 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: @pytest.mark.asyncio -async def test_serve_starts_waits_and_closes_on_normal_termination( +async def test_serve_async_starts_waits_and_closes_on_normal_termination( monkeypatch: pytest.MonkeyPatch, ) -> None: fake_server = _LifecycleServerFake() @@ -244,7 +244,7 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - await server.serve("127.0.0.1:50053") + await server.serve_async("127.0.0.1:50053") assert fake_server.addresses == ["127.0.0.1:50053"] assert fake_server.started is True @@ -254,7 +254,7 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: @pytest.mark.asyncio -async def test_serve_propagates_cancellation_after_closing_resources( +async def test_serve_async_propagates_cancellation_after_closing_resources( monkeypatch: pytest.MonkeyPatch, ) -> None: fake_server = _LifecycleServerFake(wait_error=asyncio.CancelledError()) @@ -268,7 +268,7 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) with pytest.raises(asyncio.CancelledError): - await server.serve("127.0.0.1:50054") + await server.serve_async("127.0.0.1:50054") assert fake_server.started is True assert fake_server.stop_graces == [0] @@ -276,7 +276,7 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: @pytest.mark.asyncio -async def test_serve_preserves_cancellation_during_server_shutdown( +async def test_serve_async_preserves_cancellation_during_server_shutdown( monkeypatch: pytest.MonkeyPatch, ) -> None: fake_server = _LifecycleServerFake(block_stop=True) @@ -289,7 +289,7 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - serving = asyncio.create_task(server.serve("127.0.0.1:50055")) + serving = asyncio.create_task(server.serve_async("127.0.0.1:50055")) await fake_server.stop_started.wait() serving.cancel() await asyncio.sleep(0) @@ -305,7 +305,7 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: @pytest.mark.asyncio -async def test_serve_closes_resources_when_startup_fails( +async def test_serve_async_sanitizes_startup_failures_and_closes_resources( monkeypatch: pytest.MonkeyPatch, ) -> None: fake_server = _LifecycleServerFake(start_error=RuntimeError("startup failed")) @@ -318,9 +318,12 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - with pytest.raises(RuntimeError, match="startup failed"): - await server.serve("127.0.0.1:50056") + with pytest.raises(PrivacyGuardError) as captured: + await server.serve_async("127.0.0.1:50056") + assert captured.value.code is ErrorCode.SERVER_BIND_FAILED + assert captured.value.__cause__ is None + assert "startup failed" not in str(captured.value) assert fake_server.waited is False assert fake_server.stop_graces == [0] assert closed == [server._middleware] diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index 9ebd53b..248fb84 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -3,11 +3,8 @@ from __future__ import annotations import asyncio -import logging from threading import get_ident -from typing import Never -import grpc import pytest from google.protobuf import json_format from google.protobuf.message import Message @@ -73,11 +70,6 @@ def _request(body: bytes, *, action: str = "replace") -> pb2.HttpRequestEvaluati ) -class _UnusedAbortContext: - async def abort(self, code: grpc.StatusCode, details: str) -> Never: - raise AssertionError(f"unexpected abort: {code.name}: {details}") - - def test_copied_proto_remains_the_current_openshell_contract() -> None: evaluation = pb2.HttpRequestEvaluation() finding = pb2.Finding() @@ -103,19 +95,16 @@ def test_validate_config_is_pure_and_reports_invalid_config() -> None: def test_limit_deny_explains_recovery_options() -> None: - result, limit_kind = servicer_module._result_to_proto( + result = servicer_module._result_to_proto( RequestProcessingResult( decision=RequestDecision.DENY, reason_code=LIMIT_REASON_CODE, - diagnostic_limit_kind="timeout", ) ) - assert limit_kind == "timeout" assert result.reason == LIMIT_REASON assert "Reduce the request or replacement size" in result.reason assert "simplify the configured stages and patterns" in result.reason - assert "If Privacy Guard logs report a timeout" in result.reason assert "--timeout-seconds or PrivacyGuardServer(timeout_seconds=...)" in ( result.reason ) @@ -141,10 +130,7 @@ def test_evaluation_decodes_one_utf8_text_and_encodes_replacement() -> None: async def evaluate() -> pb2.HttpRequestResult: middleware = PrivacyGuardMiddleware(create_builtin_registry()) try: - result, _ = await middleware._evaluate_http_request( - _request(b"email a@b.com") - ) - return result + return await middleware._evaluate_http_request(_request(b"email a@b.com")) finally: await middleware.close() @@ -242,10 +228,9 @@ def test_detect_returns_no_body_mutation() -> None: async def evaluate() -> pb2.HttpRequestResult: middleware = PrivacyGuardMiddleware(create_builtin_registry()) try: - result, _ = await middleware._evaluate_http_request( + return await middleware._evaluate_http_request( _request(b"a@b.com", action="detect") ) - return result finally: await middleware.close() @@ -254,39 +239,3 @@ async def evaluate() -> pb2.HttpRequestResult: assert result.decision == pb2.DECISION_ALLOW assert result.has_body is False assert result.body == b"" - - -def test_service_resource_limit_log_is_content_safe_and_request_correlated( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - async def evaluate() -> pb2.HttpRequestResult: - middleware = PrivacyGuardMiddleware(create_builtin_registry()) - - async def return_resource_limit( - _: pb2.HttpRequestEvaluation, - ) -> tuple[pb2.HttpRequestResult, str]: - return servicer_module._limit_deny() - - monkeypatch.setattr( - middleware, - "_evaluate_http_request", - return_resource_limit, - ) - request = _request(b"sensitive request content") - request.context.request_id = "request-123" - try: - return await middleware._evaluate_rpc( - request, - _UnusedAbortContext(), - ) - finally: - await middleware.close() - - with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): - result = asyncio.run(evaluate()) - - assert result.reason_code == LIMIT_REASON_CODE - assert "request_id=request-123" in caplog.text - assert "limit_kind=resource" in caplog.text - assert "sensitive request content" not in caplog.text diff --git a/projects/privacy-guard/tests/test_cli.py b/projects/privacy-guard/tests/test_cli.py index a358a27..4ddcbbf 100644 --- a/projects/privacy-guard/tests/test_cli.py +++ b/projects/privacy-guard/tests/test_cli.py @@ -14,6 +14,7 @@ from privacy_guard import cli as cli_module from privacy_guard.cli import app from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry +from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.service.server import PrivacyGuardServer @@ -23,7 +24,7 @@ def test_cli_help_exposes_server_and_discovery_commands() -> None: assert result.exit_code == 0 output = _plain_output(result) assert "serve" in output - assert "schema" in output + assert "configuration-schema" in output assert "engines" in output assert "--debug" in output assert "--debug-log-content" in output @@ -64,8 +65,8 @@ def test_cli_engines_describes_the_installed_engine() -> None: assert description in result.output -def test_cli_schema_prints_the_finalized_discriminated_policy_schema() -> None: - result = CliRunner().invoke(app, ["schema"]) +def test_cli_configuration_schema_prints_finalized_policy_schema() -> None: + result = CliRunner().invoke(app, ["configuration-schema"]) assert result.exit_code == 0 schema = json.loads(result.output) @@ -110,7 +111,7 @@ def create_registry() -> EngineRegistry: ("factory_reference", "reason"), [ ("missing-separator", "my_engines:create_registry"), - ("operator_engines:missing", "does not export the named factory"), + ("operator_engines:missing", "Verify the module:factory reference"), ("operator_engines:not_callable", "Export a callable"), ("operator_engines:failed", "Run the factory directly"), ("operator_engines:wrong_type", "Return an EngineRegistry"), @@ -169,39 +170,13 @@ def fail_import(_: str) -> object: assert "sensitive import failure" not in _plain_output(result) -def test_cli_translates_dynamic_registry_factory_lookup_failures( - monkeypatch: pytest.MonkeyPatch, -) -> None: - class DynamicModule: - def __getattr__(self, _: str) -> object: - raise RuntimeError("sensitive dynamic lookup failure") - - monkeypatch.setattr( - cli_module.importlib, - "import_module", - lambda _: DynamicModule(), - ) - - result = CliRunner().invoke( - app, - ["--registry-factory", "operator_engines:create_registry", "engines"], - terminal_width=240, - ) - - assert result.exit_code == 2 - output = _normalized_output(result) - assert "Registry factory could not be resolved" in output - assert "fix its dynamic attribute lookup" in output - assert "sensitive dynamic lookup failure" not in _plain_output(result) - - def test_cli_serve_adapts_operational_options_to_the_programmatic_server( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: calls: list[tuple[str, float, bool]] = [] - def record_run(self: PrivacyGuardServer, listen: str) -> None: + def record_serve_sync(self: PrivacyGuardServer, listen: str) -> None: calls.append( ( listen, @@ -210,7 +185,7 @@ def record_run(self: PrivacyGuardServer, listen: str) -> None: ) ) - monkeypatch.setattr(PrivacyGuardServer, "run", record_run) + monkeypatch.setattr(PrivacyGuardServer, "serve_sync", record_serve_sync) with caplog.at_level(logging.WARNING, logger="privacy_guard.cli"): result = CliRunner().invoke( @@ -230,6 +205,27 @@ def record_run(self: PrivacyGuardServer, listen: str) -> None: assert "privacy_guard_request_content_logging_enabled" in caplog.text +def test_cli_serve_prints_cataloged_startup_errors_without_a_traceback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fail_safely(_: PrivacyGuardServer, listen: str) -> None: + del listen + raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) + + monkeypatch.setattr(PrivacyGuardServer, "serve_sync", fail_safely) + + result = CliRunner().invoke( + app, + ["serve", "--listen", "sensitive-listen-address"], + ) + + assert result.exit_code == 1 + assert "[server_bind_failed]" in result.output + assert "Choose an available listen address and port, then retry" in result.output + assert "sensitive-listen-address" not in result.output + assert "Traceback" not in result.output + + @pytest.mark.parametrize("timeout_seconds", ["0", "31", "nan"]) def test_cli_rejects_invalid_processing_timeout(timeout_seconds: str) -> None: result = CliRunner().invoke( diff --git a/projects/privacy-guard/tests/test_request_processor.py b/projects/privacy-guard/tests/test_request_processor.py index 7f8dd79..47f4538 100644 --- a/projects/privacy-guard/tests/test_request_processor.py +++ b/projects/privacy-guard/tests/test_request_processor.py @@ -2,7 +2,6 @@ from __future__ import annotations -import logging from time import monotonic import pytest @@ -116,9 +115,8 @@ def test_block_is_a_processor_disposition_not_an_engine_strategy() -> None: assert tuple(item.entity for item in result.detection_summaries) == ("person",) -def test_timeout_deny_logs_a_content_safe_diagnostic_kind( +def test_timeout_returns_the_bounded_limit_deny( monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, ) -> None: monkeypatch.setattr( Timeout, @@ -126,31 +124,21 @@ def test_timeout_deny_logs_a_content_safe_diagnostic_kind( classmethod(lambda cls, seconds: cls(deadline=monotonic() - 1)), ) - with caplog.at_level(logging.INFO, logger="privacy_guard.request_processor"): - result = _processor(PolicyAction.DETECT).process("Hello Alice") + result = _processor(PolicyAction.DETECT).process("Hello Alice") assert result.decision is RequestDecision.DENY assert result.reason_code == "privacy_guard_limit_exceeded" - assert result.diagnostic_limit_kind == "timeout" - assert "privacy_guard_processing_limit kind=timeout" in caplog.text - assert "Alice" not in caplog.text -def test_resource_limit_deny_logs_a_content_safe_diagnostic_kind( +def test_engine_resource_limit_returns_the_bounded_limit_deny( monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, ) -> None: def exceed_limit(*_: object, **__: object) -> object: raise EngineLimitExceededError("sensitive resource detail") monkeypatch.setattr(RegexEngine, "_run", exceed_limit) - with caplog.at_level(logging.INFO, logger="privacy_guard.request_processor"): - result = _processor(PolicyAction.DETECT).process("Hello Alice") + result = _processor(PolicyAction.DETECT).process("Hello Alice") assert result.decision is RequestDecision.DENY assert result.reason_code == "privacy_guard_limit_exceeded" - assert result.diagnostic_limit_kind == "resource" - assert "privacy_guard_processing_limit kind=resource" in caplog.text - assert "Alice" not in caplog.text - assert "sensitive resource detail" not in caplog.text From ce31a2cd12378802838fa6725c2cbf31f95e3cb3 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 15:30:18 +0000 Subject: [PATCH 54/82] Clarify Privacy Guard limit and startup diagnostics --- .../architecture/safety-and-limits.md | 7 ++-- .../architecture/service-boundary.md | 4 ++- .../src/privacy_guard/constants.py | 7 ++-- .../privacy-guard/src/privacy_guard/errors.py | 2 +- .../src/privacy_guard/request_processor.py | 2 ++ .../src/privacy_guard/service/servicer.py | 1 + .../tests/service/test_server.py | 1 + .../tests/service/test_servicer.py | 32 ++++++++++++++++++- .../tests/test_request_processor.py | 14 ++++++-- 9 files changed, 60 insertions(+), 10 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 95e7cf9..b2a7c9e 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -129,8 +129,11 @@ The service returns the same bounded deny when: - replacement text cannot be encoded within the body limit - a deny reason code is not safely representable -The deny reason directs users to reduce the request or replacement size, -simplify the configured stages and patterns, or increase `--timeout-seconds` or +The processor emits a content-safe `timeout` or `resource` limit kind, and the +service adapter emits `resource` for representation limits. Neither log includes +request content or collaborator error text. The deny reason directs users to +check that log, then reduce the request or replacement size, simplify the +configured stages and patterns, or increase `--timeout-seconds` or `PrivacyGuardServer(timeout_seconds=...)` up to 30 seconds. When increasing the processing timeout, users must give OpenShell's middleware timeout additional headroom for worker queueing and configuration or engine preparation. These diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index a5147e8..ddf4d97 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -232,7 +232,9 @@ The server: 4. stops gRPC 5. closes the middleware executor -A bind failure becomes the stable `server_bind_failed` error. +A bind or asynchronous gRPC startup `RuntimeError` becomes the stable +`server_bind_failed` error. Its catalog operation is the broader `server.start` +phase so the rendered failure does not mislabel a post-bind startup error. The server module has no command framework, module-import-string, discovery, schema-rendering, or command-logging responsibilities. diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index 8957d45..dd62524 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -19,9 +19,10 @@ BLOCK_REASON = "Privacy Guard blocked the request" BLOCK_REASON_CODE = "privacy_guard_blocked" LIMIT_REASON = ( - "Privacy Guard exceeded a processing safety limit. Reduce the request or " - "replacement size, simplify the configured stages and patterns, or increase " - "the processing timeout with --timeout-seconds or " + "Privacy Guard exceeded a processing safety limit. Check Privacy Guard logs " + "for the limit kind. Reduce the request or replacement size, simplify the " + "configured stages and patterns, or increase the processing timeout with " + "--timeout-seconds or " "PrivacyGuardServer(timeout_seconds=...) to at most " f"{MAX_TIMEOUT_SECONDS:g} seconds. If increasing it, give OpenShell's " "middleware timeout additional headroom for queueing and configuration " diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py index 0e43b22..313a8a7 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -168,7 +168,7 @@ class EngineRegistryError(Exception): ErrorCode.SERVER_BIND_FAILED: _ErrorSpec( ErrorKind.INTERNAL, ErrorComponent.SERVER, - "bind", + "start", "Server could not start on its listen address.", "Choose an available listen address and port, then retry.", ), diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index 8c11b45..fe54546 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -134,11 +134,13 @@ def process(self, text: str) -> RequestProcessingResult: current_text = result.text timeout.raise_if_expired() except TimeoutExpiredError: + _LOGGER.info("privacy_guard_processing_limit kind=timeout") return RequestProcessingResult( decision=RequestDecision.DENY, reason_code=LIMIT_REASON_CODE, ) except EngineLimitExceededError: + _LOGGER.info("privacy_guard_processing_limit kind=resource") return RequestProcessingResult( decision=RequestDecision.DENY, reason_code=LIMIT_REASON_CODE, diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index 1c87ae4..c7b9972 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -367,6 +367,7 @@ def _detection_to_proto(detection: EntityDetectionSummary) -> pb2.Finding: def _limit_deny() -> pb2.HttpRequestResult: + _LOGGER.info("privacy_guard_processing_limit kind=resource") return pb2.HttpRequestResult( decision=pb2.DECISION_DENY, reason=LIMIT_REASON, diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 13a23c7..eeca94b 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -323,6 +323,7 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: assert captured.value.code is ErrorCode.SERVER_BIND_FAILED assert captured.value.__cause__ is None + assert "server.start" in str(captured.value) assert "startup failed" not in str(captured.value) assert fake_server.waited is False assert fake_server.stop_graces == [0] diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index 248fb84..cd4877c 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import logging from threading import get_ident import pytest @@ -11,11 +12,16 @@ from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.config import PrivacyGuardConfig -from privacy_guard.constants import LIMIT_REASON, LIMIT_REASON_CODE +from privacy_guard.constants import ( + LIMIT_REASON, + LIMIT_REASON_CODE, + MAX_PROTO_FINDING_BYTES, +) from privacy_guard.engines import EngineConfig from privacy_guard.engines.registry import create_builtin_registry from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.request_processor import ( + EntityDetectionSummary, RequestDecision, RequestProcessingResult, RequestProcessor, @@ -103,6 +109,7 @@ def test_limit_deny_explains_recovery_options() -> None: ) assert result.reason == LIMIT_REASON + assert "Check Privacy Guard logs for the limit kind" in result.reason assert "Reduce the request or replacement size" in result.reason assert "simplify the configured stages and patterns" in result.reason assert "--timeout-seconds or PrivacyGuardServer(timeout_seconds=...)" in ( @@ -113,6 +120,29 @@ def test_limit_deny_explains_recovery_options() -> None: ) +def test_service_limit_deny_logs_a_content_safe_resource_kind( + caplog: pytest.LogCaptureFixture, +) -> None: + sentinel = "sensitive-finding-value" + with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): + result = servicer_module._result_to_proto( + RequestProcessingResult( + decision=RequestDecision.ALLOW, + detection_summaries=( + EntityDetectionSummary( + entity=sentinel + ("x" * MAX_PROTO_FINDING_BYTES), + source_stage="stage", + count=1, + ), + ), + ) + ) + + assert result.reason_code == LIMIT_REASON_CODE + assert "privacy_guard_processing_limit kind=resource" in caplog.text + assert sentinel not in caplog.text + + def test_middleware_applies_configured_timeout_to_cached_processors() -> None: middleware = PrivacyGuardMiddleware( create_builtin_registry(), diff --git a/projects/privacy-guard/tests/test_request_processor.py b/projects/privacy-guard/tests/test_request_processor.py index 47f4538..d75bb12 100644 --- a/projects/privacy-guard/tests/test_request_processor.py +++ b/projects/privacy-guard/tests/test_request_processor.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from time import monotonic import pytest @@ -117,6 +118,7 @@ def test_block_is_a_processor_disposition_not_an_engine_strategy() -> None: def test_timeout_returns_the_bounded_limit_deny( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: monkeypatch.setattr( Timeout, @@ -124,21 +126,29 @@ def test_timeout_returns_the_bounded_limit_deny( classmethod(lambda cls, seconds: cls(deadline=monotonic() - 1)), ) - result = _processor(PolicyAction.DETECT).process("Hello Alice") + with caplog.at_level(logging.INFO, logger="privacy_guard.request_processor"): + result = _processor(PolicyAction.DETECT).process("Hello Alice") assert result.decision is RequestDecision.DENY assert result.reason_code == "privacy_guard_limit_exceeded" + assert "privacy_guard_processing_limit kind=timeout" in caplog.text + assert "Alice" not in caplog.text def test_engine_resource_limit_returns_the_bounded_limit_deny( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: def exceed_limit(*_: object, **__: object) -> object: raise EngineLimitExceededError("sensitive resource detail") monkeypatch.setattr(RegexEngine, "_run", exceed_limit) - result = _processor(PolicyAction.DETECT).process("Hello Alice") + with caplog.at_level(logging.INFO, logger="privacy_guard.request_processor"): + result = _processor(PolicyAction.DETECT).process("Hello Alice") assert result.decision is RequestDecision.DENY assert result.reason_code == "privacy_guard_limit_exceeded" + assert "privacy_guard_processing_limit kind=resource" in caplog.text + assert "Alice" not in caplog.text + assert "sensitive resource detail" not in caplog.text From b4a4e46336ea9abbc0cb8e054f5270b0715e8825 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 15:53:17 +0000 Subject: [PATCH 55/82] Unify Privacy Guard logging configuration --- .../privacy-guard/architecture/index.md | 4 +- projects/privacy-guard/AGENTS.md | 1 + projects/privacy-guard/README.md | 19 ++++++ .../privacy-guard/src/privacy_guard/cli.py | 9 +-- .../src/privacy_guard/logging.py | 60 +++++++++++++++++ projects/privacy-guard/tests/test_cli.py | 35 +++++----- projects/privacy-guard/tests/test_logging.py | 67 +++++++++++++++++++ 7 files changed, 172 insertions(+), 23 deletions(-) create mode 100644 projects/privacy-guard/src/privacy_guard/logging.py create mode 100644 projects/privacy-guard/tests/test_logging.py diff --git a/docs/documentation/privacy-guard/architecture/index.md b/docs/documentation/privacy-guard/architecture/index.md index 0614b50..3015e8f 100644 --- a/docs/documentation/privacy-guard/architecture/index.md +++ b/docs/documentation/privacy-guard/architecture/index.md @@ -56,7 +56,9 @@ Source paths on these pages are relative to bindings. - `cli.py` owns command parsing, registry-factory loading, engine discovery, configuration-schema output, logging options, and the adapter that starts - the programmatic server. + the programmatic server. Top-level `logging.py` provides the shared, + standard-library logging configuration used by the CLI and available to + programmatic deployments. - `request_processor.py` runs configured stages over one text value, shares one timeout across them, aggregates detections, and applies the user-facing policy action. It does not import gRPC or implement an engine's algorithms. diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md index b742ebd..226cbe7 100644 --- a/projects/privacy-guard/AGENTS.md +++ b/projects/privacy-guard/AGENTS.md @@ -34,6 +34,7 @@ Run focused tests while working and `make check` before handoff. - `src/privacy_guard/config.py`: policy action and ordered stage configuration - `src/privacy_guard/request_processor.py`: stage execution and policy disposition - `src/privacy_guard/cli.py`: command parsing, discovery, configuration-schema output, and server adapter +- `src/privacy_guard/logging.py`: package-scoped standard-library logging configuration - `src/privacy_guard/base.py`: package-wide strict immutable domain-model base - `src/privacy_guard/string_validators.py`: shared string validators and field types - `src/privacy_guard/service/`: gRPC lifecycle and protobuf adapter diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 4682332..e90e466 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -189,6 +189,25 @@ Custom applications pass their own finalized registry to transport, worker executor, and shutdown lifecycle; it does not load Python modules, select engines, generate schemas, or parse command-line options. +Privacy Guard emits records through Python's standard `privacy_guard` logger +and does not configure logging when imported. Applications may use their own +standard-library logging setup or opt into Privacy Guard's concise console +format: + +```python +import logging + +from privacy_guard.logging import configure_logging + +configure_logging(level=logging.DEBUG) +``` + +Repeated calls replace only the handler installed by `configure_logging()`; +application-owned handlers are preserved. The CLI uses this same configuration +path. Its default level is `INFO`, and `--debug` enables content-safe `DEBUG` +diagnostics. `--debug-log-content` additionally logs complete request and +processed text and should be enabled only in a controlled environment. + ## CLI ```bash diff --git a/projects/privacy-guard/src/privacy_guard/cli.py b/projects/privacy-guard/src/privacy_guard/cli.py index 4a33451..b9f2879 100644 --- a/projects/privacy-guard/src/privacy_guard/cli.py +++ b/projects/privacy-guard/src/privacy_guard/cli.py @@ -14,6 +14,7 @@ from privacy_guard.engines import EntityProcessingStrategy from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry from privacy_guard.errors import PrivacyGuardError +from privacy_guard.logging import configure_logging from privacy_guard.service.server import DEFAULT_LISTEN_ADDRESS, PrivacyGuardServer from privacy_guard.timeout import validate_timeout_seconds @@ -49,13 +50,7 @@ def configure_cli( ] = False, ) -> None: """Configure the command application and its engine inventory.""" - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(name)s: %(message)s", - ) - logging.getLogger("privacy_guard").setLevel( - logging.DEBUG if debug or debug_log_content else logging.INFO - ) + configure_logging(logging.DEBUG if debug or debug_log_content else logging.INFO) context.obj = _CommandOptions( registry=_load_registry(registry_factory), log_request_content=debug_log_content, diff --git a/projects/privacy-guard/src/privacy_guard/logging.py b/projects/privacy-guard/src/privacy_guard/logging.py new file mode 100644 index 0000000..1422aad --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/logging.py @@ -0,0 +1,60 @@ +"""Native Python logging configuration for Privacy Guard.""" + +from __future__ import annotations + +import logging +from typing import TextIO + + +def configure_logging( + level: int | str = logging.INFO, + *, + stream: TextIO | None = None, +) -> None: + """Configure consistent console logging for the Privacy Guard package. + + Repeated calls replace the handler installed by this function. Handlers + installed by the containing application are left unchanged. + """ + package_logger = logging.getLogger("privacy_guard") + package_logger.setLevel(level) + + for handler in package_logger.handlers[:]: + if isinstance(handler, _PrivacyGuardStreamHandler): + package_logger.removeHandler(handler) + handler.close() + + handler = _PrivacyGuardStreamHandler(stream) + handler.setFormatter( + logging.Formatter( + "[%(asctime)s] [%(levelname)s] %(message)s", + datefmt="%H:%M:%S", + ) + ) + package_logger.addHandler(handler) + package_logger.propagate = False + + +def reset_logging() -> None: + """Remove logging configuration installed by :func:`configure_logging`.""" + package_logger = logging.getLogger("privacy_guard") + managed_handlers = [ + handler + for handler in package_logger.handlers + if isinstance(handler, _PrivacyGuardStreamHandler) + ] + if not managed_handlers: + return + + for handler in managed_handlers: + package_logger.removeHandler(handler) + handler.close() + package_logger.setLevel(logging.NOTSET) + package_logger.propagate = True + + +class _PrivacyGuardStreamHandler(logging.StreamHandler[TextIO]): + """Stream handler owned by Privacy Guard's logging configuration.""" + + +__all__ = ["configure_logging", "reset_logging"] diff --git a/projects/privacy-guard/tests/test_cli.py b/projects/privacy-guard/tests/test_cli.py index 4ddcbbf..eaa470b 100644 --- a/projects/privacy-guard/tests/test_cli.py +++ b/projects/privacy-guard/tests/test_cli.py @@ -3,8 +3,8 @@ from __future__ import annotations import json -import logging import re +from collections.abc import Iterator from importlib.metadata import entry_points from types import SimpleNamespace @@ -15,9 +15,16 @@ from privacy_guard.cli import app from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.logging import reset_logging from privacy_guard.service.server import PrivacyGuardServer +@pytest.fixture(autouse=True) +def _reset_cli_logging() -> Iterator[None]: + yield + reset_logging() + + def test_cli_help_exposes_server_and_discovery_commands() -> None: result = CliRunner().invoke(app, ["--help"]) @@ -172,7 +179,6 @@ def fail_import(_: str) -> object: def test_cli_serve_adapts_operational_options_to_the_programmatic_server( monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, ) -> None: calls: list[tuple[str, float, bool]] = [] @@ -187,22 +193,21 @@ def record_serve_sync(self: PrivacyGuardServer, listen: str) -> None: monkeypatch.setattr(PrivacyGuardServer, "serve_sync", record_serve_sync) - with caplog.at_level(logging.WARNING, logger="privacy_guard.cli"): - result = CliRunner().invoke( - app, - [ - "--debug-log-content", - "serve", - "--listen", - "127.0.0.1:50052", - "--timeout-seconds", - "4.5", - ], - ) + result = CliRunner().invoke( + app, + [ + "--debug-log-content", + "serve", + "--listen", + "127.0.0.1:50052", + "--timeout-seconds", + "4.5", + ], + ) assert result.exit_code == 0 assert calls == [("127.0.0.1:50052", 4.5, True)] - assert "privacy_guard_request_content_logging_enabled" in caplog.text + assert "privacy_guard_request_content_logging_enabled" in _plain_output(result) def test_cli_serve_prints_cataloged_startup_errors_without_a_traceback( diff --git a/projects/privacy-guard/tests/test_logging.py b/projects/privacy-guard/tests/test_logging.py new file mode 100644 index 0000000..35e348e --- /dev/null +++ b/projects/privacy-guard/tests/test_logging.py @@ -0,0 +1,67 @@ +"""Privacy Guard logging configuration tests.""" + +from __future__ import annotations + +import logging +from collections.abc import Iterator +from io import StringIO + +import pytest + +from privacy_guard.logging import configure_logging, reset_logging + + +@pytest.fixture(autouse=True) +def _restore_logging() -> Iterator[None]: + reset_logging() + yield + reset_logging() + + +def test_configure_logging_emits_consistent_package_logs() -> None: + stream = StringIO() + configure_logging(stream=stream) + + logging.getLogger("privacy_guard.service").info("server_started") + logging.getLogger("privacy_guard.service").debug("hidden_detail") + + output = stream.getvalue() + assert "[INFO] server_started" in output + assert "hidden_detail" not in output + + +def test_configure_logging_accepts_native_log_levels() -> None: + stream = StringIO() + configure_logging(logging.DEBUG, stream=stream) + + logging.getLogger("privacy_guard.request_processor").debug("processing_diagnostic") + + assert "[DEBUG] processing_diagnostic" in stream.getvalue() + + +def test_configure_logging_replaces_its_previous_handler() -> None: + first_stream = StringIO() + second_stream = StringIO() + configure_logging(stream=first_stream) + configure_logging(stream=second_stream) + + logging.getLogger("privacy_guard").info("configured_once") + + assert "configured_once" not in first_stream.getvalue() + assert second_stream.getvalue().count("configured_once") == 1 + + +def test_reset_logging_restores_application_logging() -> None: + package_logger = logging.getLogger("privacy_guard") + application_handler = logging.NullHandler() + package_logger.addHandler(application_handler) + configure_logging(stream=StringIO()) + + reset_logging() + + try: + assert package_logger.handlers == [application_handler] + assert package_logger.level == logging.NOTSET + assert package_logger.propagate is True + finally: + package_logger.removeHandler(application_handler) From 6cff82d2782e508a2d9de3e37b03fcdb636c4465 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 15:58:47 +0000 Subject: [PATCH 56/82] Route Privacy Guard loggers through shared module --- .../privacy-guard/src/privacy_guard/cli.py | 7 ++-- .../src/privacy_guard/logging.py | 11 +++++-- .../src/privacy_guard/request_processor.py | 4 +-- .../src/privacy_guard/service/server.py | 4 +-- .../src/privacy_guard/service/servicer.py | 4 +-- projects/privacy-guard/tests/test_logging.py | 33 ++++++++++++++++++- 6 files changed, 49 insertions(+), 14 deletions(-) diff --git a/projects/privacy-guard/src/privacy_guard/cli.py b/projects/privacy-guard/src/privacy_guard/cli.py index b9f2879..a73089b 100644 --- a/projects/privacy-guard/src/privacy_guard/cli.py +++ b/projects/privacy-guard/src/privacy_guard/cli.py @@ -4,7 +4,6 @@ import importlib import json -import logging from dataclasses import dataclass from typing import Annotated @@ -14,7 +13,7 @@ from privacy_guard.engines import EntityProcessingStrategy from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry from privacy_guard.errors import PrivacyGuardError -from privacy_guard.logging import configure_logging +from privacy_guard.logging import configure_logging, get_logger from privacy_guard.service.server import DEFAULT_LISTEN_ADDRESS, PrivacyGuardServer from privacy_guard.timeout import validate_timeout_seconds @@ -50,7 +49,7 @@ def configure_cli( ] = False, ) -> None: """Configure the command application and its engine inventory.""" - configure_logging(logging.DEBUG if debug or debug_log_content else logging.INFO) + configure_logging("DEBUG" if debug or debug_log_content else "INFO") context.obj = _CommandOptions( registry=_load_registry(registry_factory), log_request_content=debug_log_content, @@ -126,7 +125,7 @@ def engines(context: typer.Context) -> None: ) -_LOGGER = logging.getLogger(__name__) +_LOGGER = get_logger(__name__) @dataclass(frozen=True) diff --git a/projects/privacy-guard/src/privacy_guard/logging.py b/projects/privacy-guard/src/privacy_guard/logging.py index 1422aad..8673b48 100644 --- a/projects/privacy-guard/src/privacy_guard/logging.py +++ b/projects/privacy-guard/src/privacy_guard/logging.py @@ -6,6 +6,11 @@ from typing import TextIO +def get_logger(name: str) -> logging.Logger: + """Return a logger governed by Privacy Guard's shared configuration.""" + return logging.getLogger(name) + + def configure_logging( level: int | str = logging.INFO, *, @@ -16,7 +21,7 @@ def configure_logging( Repeated calls replace the handler installed by this function. Handlers installed by the containing application are left unchanged. """ - package_logger = logging.getLogger("privacy_guard") + package_logger = get_logger("privacy_guard") package_logger.setLevel(level) for handler in package_logger.handlers[:]: @@ -37,7 +42,7 @@ def configure_logging( def reset_logging() -> None: """Remove logging configuration installed by :func:`configure_logging`.""" - package_logger = logging.getLogger("privacy_guard") + package_logger = get_logger("privacy_guard") managed_handlers = [ handler for handler in package_logger.handlers @@ -57,4 +62,4 @@ class _PrivacyGuardStreamHandler(logging.StreamHandler[TextIO]): """Stream handler owned by Privacy Guard's logging configuration.""" -__all__ = ["configure_logging", "reset_logging"] +__all__ = ["configure_logging", "get_logger", "reset_logging"] diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index fe54546..23e1b41 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -2,7 +2,6 @@ from __future__ import annotations -import logging from collections import OrderedDict from collections.abc import Sequence from enum import StrEnum @@ -33,6 +32,7 @@ PrivacyGuardError, TimeoutExpiredError, ) +from privacy_guard.logging import get_logger from privacy_guard.string_validators import validate_scalar_string from privacy_guard.timeout import Timeout, validate_timeout_seconds @@ -205,7 +205,7 @@ def run( ) -> TextProcessingResult: ... -_LOGGER = logging.getLogger(__name__) +_LOGGER = get_logger(__name__) __all__ = [ diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index de09195..edcbd9f 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -import logging import grpc @@ -15,6 +14,7 @@ ) from privacy_guard.engines.registry import EngineRegistry from privacy_guard.errors import ErrorCode, PrivacyGuardError +from privacy_guard.logging import get_logger from privacy_guard.service.servicer import PrivacyGuardMiddleware DEFAULT_LISTEN_ADDRESS = "127.0.0.1:50051" @@ -63,7 +63,7 @@ async def serve_async(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: await self._middleware.close() -_LOGGER = logging.getLogger(__name__) +_LOGGER = get_logger(__name__) def _create_grpc_server( diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index c7b9972..4d149f5 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -import logging import time from collections import OrderedDict from collections.abc import Callable @@ -40,6 +39,7 @@ ErrorKind, PrivacyGuardError, ) +from privacy_guard.logging import get_logger from privacy_guard.request_processor import ( EntityDetectionSummary, RequestDecision, @@ -375,7 +375,7 @@ def _limit_deny() -> pb2.HttpRequestResult: ) -_LOGGER = logging.getLogger(__name__) +_LOGGER = get_logger(__name__) _MAX_CACHED_PROCESSORS = 128 diff --git a/projects/privacy-guard/tests/test_logging.py b/projects/privacy-guard/tests/test_logging.py index 35e348e..39ec6a8 100644 --- a/projects/privacy-guard/tests/test_logging.py +++ b/projects/privacy-guard/tests/test_logging.py @@ -2,13 +2,15 @@ from __future__ import annotations +import ast import logging from collections.abc import Iterator from io import StringIO +from pathlib import Path import pytest -from privacy_guard.logging import configure_logging, reset_logging +from privacy_guard.logging import configure_logging, get_logger, reset_logging @pytest.fixture(autouse=True) @@ -30,6 +32,12 @@ def test_configure_logging_emits_consistent_package_logs() -> None: assert "hidden_detail" not in output +def test_get_logger_returns_named_package_logger() -> None: + logger = get_logger("privacy_guard.custom_engine") + + assert logger is logging.getLogger("privacy_guard.custom_engine") + + def test_configure_logging_accepts_native_log_levels() -> None: stream = StringIO() configure_logging(logging.DEBUG, stream=stream) @@ -65,3 +73,26 @@ def test_reset_logging_restores_application_logging() -> None: assert package_logger.propagate is True finally: package_logger.removeHandler(application_handler) + + +def test_source_modules_use_the_shared_logging_module() -> None: + direct_imports: list[str] = [] + for path in _SOURCE_ROOT.rglob("*.py"): + if path == _LOGGING_MODULE or "bindings" in path.parts: + continue + tree = ast.parse(path.read_text(encoding="utf-8")) + if any( + ( + isinstance(node, ast.Import) + and any(imported.name == "logging" for imported in node.names) + ) + or (isinstance(node, ast.ImportFrom) and node.module == "logging") + for node in ast.walk(tree) + ): + direct_imports.append(str(path.relative_to(_SOURCE_ROOT))) + + assert direct_imports == [] + + +_SOURCE_ROOT = Path(__file__).parents[1] / "src" / "privacy_guard" +_LOGGING_MODULE = _SOURCE_ROOT / "logging.py" From 3a105516f3dc4ac9470a9046b3311de67b1203e1 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 16:04:41 +0000 Subject: [PATCH 57/82] Add clear colored logging defaults --- projects/privacy-guard/README.md | 16 ++-- .../privacy-guard/src/privacy_guard/cli.py | 6 +- .../src/privacy_guard/logging.py | 76 ++++++++++++++++--- projects/privacy-guard/tests/test_logging.py | 74 ++++++++++++++++-- 4 files changed, 144 insertions(+), 28 deletions(-) diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index e90e466..be947dd 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -195,18 +195,20 @@ standard-library logging setup or opt into Privacy Guard's concise console format: ```python -import logging +from privacy_guard.logging import LoggingConfig, configure_logging -from privacy_guard.logging import configure_logging - -configure_logging(level=logging.DEBUG) +configure_logging(LoggingConfig(level="DEBUG")) ``` Repeated calls replace only the handler installed by `configure_logging()`; application-owned handlers are preserved. The CLI uses this same configuration -path. Its default level is `INFO`, and `--debug` enables content-safe `DEBUG` -diagnostics. `--debug-log-content` additionally logs complete request and -processed text and should be enabled only in a controlled environment. +path. The default format includes the date, time, padded level, logger name, +and message. It adds level-aware ANSI colors when writing to an interactive +terminal and stays plain when redirected; set `use_colors=True` or +`use_colors=False` for an explicit choice. Its default level is `INFO`, and +`--debug` enables content-safe `DEBUG` diagnostics. `--debug-log-content` +additionally logs complete request and processed text and should be enabled +only in a controlled environment. ## CLI diff --git a/projects/privacy-guard/src/privacy_guard/cli.py b/projects/privacy-guard/src/privacy_guard/cli.py index a73089b..1122d98 100644 --- a/projects/privacy-guard/src/privacy_guard/cli.py +++ b/projects/privacy-guard/src/privacy_guard/cli.py @@ -13,7 +13,7 @@ from privacy_guard.engines import EntityProcessingStrategy from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry from privacy_guard.errors import PrivacyGuardError -from privacy_guard.logging import configure_logging, get_logger +from privacy_guard.logging import LoggingConfig, configure_logging, get_logger from privacy_guard.service.server import DEFAULT_LISTEN_ADDRESS, PrivacyGuardServer from privacy_guard.timeout import validate_timeout_seconds @@ -49,7 +49,9 @@ def configure_cli( ] = False, ) -> None: """Configure the command application and its engine inventory.""" - configure_logging("DEBUG" if debug or debug_log_content else "INFO") + configure_logging( + LoggingConfig(level="DEBUG" if debug or debug_log_content else "INFO") + ) context.obj = _CommandOptions( registry=_load_registry(registry_factory), log_request_content=debug_log_content, diff --git a/projects/privacy-guard/src/privacy_guard/logging.py b/projects/privacy-guard/src/privacy_guard/logging.py index 8673b48..8fc595d 100644 --- a/projects/privacy-guard/src/privacy_guard/logging.py +++ b/projects/privacy-guard/src/privacy_guard/logging.py @@ -2,19 +2,31 @@ from __future__ import annotations +import copy import logging +from dataclasses import dataclass from typing import TextIO +@dataclass(frozen=True) +class LoggingConfig: + """Privacy Guard console logging settings.""" + + level: int | str = logging.INFO + stream: TextIO | None = None + use_colors: bool | None = None + + +DEFAULT_LOGGING_CONFIG = LoggingConfig() + + def get_logger(name: str) -> logging.Logger: """Return a logger governed by Privacy Guard's shared configuration.""" return logging.getLogger(name) def configure_logging( - level: int | str = logging.INFO, - *, - stream: TextIO | None = None, + config: LoggingConfig = DEFAULT_LOGGING_CONFIG, ) -> None: """Configure consistent console logging for the Privacy Guard package. @@ -22,20 +34,18 @@ def configure_logging( installed by the containing application are left unchanged. """ package_logger = get_logger("privacy_guard") - package_logger.setLevel(level) + package_logger.setLevel(config.level) for handler in package_logger.handlers[:]: if isinstance(handler, _PrivacyGuardStreamHandler): package_logger.removeHandler(handler) handler.close() - handler = _PrivacyGuardStreamHandler(stream) - handler.setFormatter( - logging.Formatter( - "[%(asctime)s] [%(levelname)s] %(message)s", - datefmt="%H:%M:%S", - ) + handler = _PrivacyGuardStreamHandler(config.stream) + use_colors = ( + handler.stream.isatty() if config.use_colors is None else config.use_colors ) + handler.setFormatter(_PrivacyGuardFormatter(use_colors=use_colors)) package_logger.addHandler(handler) package_logger.propagate = False @@ -62,4 +72,48 @@ class _PrivacyGuardStreamHandler(logging.StreamHandler[TextIO]): """Stream handler owned by Privacy Guard's logging configuration.""" -__all__ = ["configure_logging", "get_logger", "reset_logging"] +class _PrivacyGuardFormatter(logging.Formatter): + """Readable console formatter with optional level-aware color.""" + + def __init__(self, *, use_colors: bool) -> None: + log_format = _PLAIN_LOG_FORMAT + if use_colors: + log_format = _COLOR_LOG_FORMAT + super().__init__(log_format, datefmt="%Y-%m-%d %H:%M:%S") + self._use_colors = use_colors + + def format(self, record: logging.LogRecord) -> str: + formatted_record = copy.copy(record) + level_name = f"{record.levelname:<8}" + if self._use_colors: + color = _LEVEL_COLORS.get(record.levelno, _ANSI_BRIGHT_BLACK) + level_name = f"{color}{level_name}{_ANSI_RESET}" + formatted_record.levelname = level_name + return super().format(formatted_record) + + +_PLAIN_LOG_FORMAT = "%(asctime)s | %(levelname)s | %(name)s | %(message)s" +_ANSI_RESET = "\033[0m" +_ANSI_DIM = "\033[2m" +_ANSI_BRIGHT_BLACK = "\033[90m" +_ANSI_CYAN = "\033[36m" +_COLOR_LOG_FORMAT = ( + f"{_ANSI_DIM}%(asctime)s{_ANSI_RESET} | " + f"%(levelname)s | {_ANSI_CYAN}%(name)s{_ANSI_RESET} | %(message)s" +) +_LEVEL_COLORS = { + logging.DEBUG: _ANSI_BRIGHT_BLACK, + logging.INFO: "\033[32m", + logging.WARNING: "\033[33m", + logging.ERROR: "\033[31m", + logging.CRITICAL: "\033[1;31m", +} + + +__all__ = [ + "DEFAULT_LOGGING_CONFIG", + "LoggingConfig", + "configure_logging", + "get_logger", + "reset_logging", +] diff --git a/projects/privacy-guard/tests/test_logging.py b/projects/privacy-guard/tests/test_logging.py index 39ec6a8..d43ed65 100644 --- a/projects/privacy-guard/tests/test_logging.py +++ b/projects/privacy-guard/tests/test_logging.py @@ -4,13 +4,20 @@ import ast import logging +import re from collections.abc import Iterator from io import StringIO from pathlib import Path import pytest -from privacy_guard.logging import configure_logging, get_logger, reset_logging +from privacy_guard.logging import ( + DEFAULT_LOGGING_CONFIG, + LoggingConfig, + configure_logging, + get_logger, + reset_logging, +) @pytest.fixture(autouse=True) @@ -22,14 +29,57 @@ def _restore_logging() -> Iterator[None]: def test_configure_logging_emits_consistent_package_logs() -> None: stream = StringIO() - configure_logging(stream=stream) + configure_logging(LoggingConfig(stream=stream)) logging.getLogger("privacy_guard.service").info("server_started") logging.getLogger("privacy_guard.service").debug("hidden_detail") output = stream.getvalue() - assert "[INFO] server_started" in output + assert re.fullmatch( + r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}" + r" \| INFO \| privacy_guard\.service \| server_started\n", + output, + ) assert "hidden_detail" not in output + assert "\033[" not in output + + +def test_default_logging_config_uses_info_and_terminal_aware_colors() -> None: + assert DEFAULT_LOGGING_CONFIG == LoggingConfig( + level=logging.INFO, + stream=None, + use_colors=None, + ) + + +def test_configure_logging_colors_interactive_output() -> None: + stream = _TerminalStream() + configure_logging(LoggingConfig(stream=stream)) + + logging.getLogger("privacy_guard.service").warning("resource_pressure") + + output = stream.getvalue() + assert "\033[33mWARNING \033[0m" in output + assert "\033[36mprivacy_guard.service\033[0m" in output + assert output.endswith(" | resource_pressure\n") + + +def test_configure_logging_can_disable_terminal_colors() -> None: + stream = _TerminalStream() + configure_logging(LoggingConfig(stream=stream, use_colors=False)) + + logging.getLogger("privacy_guard.service").error("startup_failed") + + assert "\033[" not in stream.getvalue() + + +def test_configure_logging_can_force_colors_for_redirected_output() -> None: + stream = StringIO() + configure_logging(LoggingConfig(stream=stream, use_colors=True)) + + logging.getLogger("privacy_guard.service").info("server_started") + + assert "\033[32mINFO \033[0m" in stream.getvalue() def test_get_logger_returns_named_package_logger() -> None: @@ -40,18 +90,21 @@ def test_get_logger_returns_named_package_logger() -> None: def test_configure_logging_accepts_native_log_levels() -> None: stream = StringIO() - configure_logging(logging.DEBUG, stream=stream) + configure_logging(LoggingConfig(level=logging.DEBUG, stream=stream)) logging.getLogger("privacy_guard.request_processor").debug("processing_diagnostic") - assert "[DEBUG] processing_diagnostic" in stream.getvalue() + assert ( + "DEBUG | privacy_guard.request_processor | processing_diagnostic" + in stream.getvalue() + ) def test_configure_logging_replaces_its_previous_handler() -> None: first_stream = StringIO() second_stream = StringIO() - configure_logging(stream=first_stream) - configure_logging(stream=second_stream) + configure_logging(LoggingConfig(stream=first_stream)) + configure_logging(LoggingConfig(stream=second_stream)) logging.getLogger("privacy_guard").info("configured_once") @@ -63,7 +116,7 @@ def test_reset_logging_restores_application_logging() -> None: package_logger = logging.getLogger("privacy_guard") application_handler = logging.NullHandler() package_logger.addHandler(application_handler) - configure_logging(stream=StringIO()) + configure_logging(LoggingConfig(stream=StringIO())) reset_logging() @@ -96,3 +149,8 @@ def test_source_modules_use_the_shared_logging_module() -> None: _SOURCE_ROOT = Path(__file__).parents[1] / "src" / "privacy_guard" _LOGGING_MODULE = _SOURCE_ROOT / "logging.py" + + +class _TerminalStream(StringIO): + def isatty(self) -> bool: + return True From 5f09d5dd6a91c8852788b060429d208ec02b0e5e Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sun, 26 Jul 2026 16:09:21 +0000 Subject: [PATCH 58/82] Make logging color policy explicit --- projects/privacy-guard/README.md | 15 ++++++++------- .../privacy-guard/src/privacy_guard/logging.py | 16 ++++++++++++++-- projects/privacy-guard/tests/test_logging.py | 7 ++++--- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index be947dd..a28c7d6 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -195,20 +195,21 @@ standard-library logging setup or opt into Privacy Guard's concise console format: ```python -from privacy_guard.logging import LoggingConfig, configure_logging +from privacy_guard.logging import ColorMode, LoggingConfig, configure_logging -configure_logging(LoggingConfig(level="DEBUG")) +configure_logging(LoggingConfig(level="DEBUG", color_mode=ColorMode.AUTO)) ``` Repeated calls replace only the handler installed by `configure_logging()`; application-owned handlers are preserved. The CLI uses this same configuration path. The default format includes the date, time, padded level, logger name, and message. It adds level-aware ANSI colors when writing to an interactive -terminal and stays plain when redirected; set `use_colors=True` or -`use_colors=False` for an explicit choice. Its default level is `INFO`, and -`--debug` enables content-safe `DEBUG` diagnostics. `--debug-log-content` -additionally logs complete request and processed text and should be enabled -only in a controlled environment. +terminal and stays plain when redirected. Set `color_mode=ColorMode.ALWAYS` or +`color_mode=ColorMode.NEVER` for an explicit choice instead of the default +`ColorMode.AUTO`. Its default level is `INFO`, and `--debug` enables +content-safe `DEBUG` diagnostics. `--debug-log-content` additionally logs +complete request and processed text and should be enabled only in a controlled +environment. ## CLI diff --git a/projects/privacy-guard/src/privacy_guard/logging.py b/projects/privacy-guard/src/privacy_guard/logging.py index 8fc595d..3489beb 100644 --- a/projects/privacy-guard/src/privacy_guard/logging.py +++ b/projects/privacy-guard/src/privacy_guard/logging.py @@ -5,16 +5,25 @@ import copy import logging from dataclasses import dataclass +from enum import StrEnum from typing import TextIO +class ColorMode(StrEnum): + """When Privacy Guard should add ANSI colors to console logs.""" + + AUTO = "auto" + ALWAYS = "always" + NEVER = "never" + + @dataclass(frozen=True) class LoggingConfig: """Privacy Guard console logging settings.""" level: int | str = logging.INFO stream: TextIO | None = None - use_colors: bool | None = None + color_mode: ColorMode = ColorMode.AUTO DEFAULT_LOGGING_CONFIG = LoggingConfig() @@ -43,7 +52,9 @@ def configure_logging( handler = _PrivacyGuardStreamHandler(config.stream) use_colors = ( - handler.stream.isatty() if config.use_colors is None else config.use_colors + handler.stream.isatty() + if config.color_mode is ColorMode.AUTO + else config.color_mode is ColorMode.ALWAYS ) handler.setFormatter(_PrivacyGuardFormatter(use_colors=use_colors)) package_logger.addHandler(handler) @@ -111,6 +122,7 @@ def format(self, record: logging.LogRecord) -> str: __all__ = [ + "ColorMode", "DEFAULT_LOGGING_CONFIG", "LoggingConfig", "configure_logging", diff --git a/projects/privacy-guard/tests/test_logging.py b/projects/privacy-guard/tests/test_logging.py index d43ed65..6521c4d 100644 --- a/projects/privacy-guard/tests/test_logging.py +++ b/projects/privacy-guard/tests/test_logging.py @@ -13,6 +13,7 @@ from privacy_guard.logging import ( DEFAULT_LOGGING_CONFIG, + ColorMode, LoggingConfig, configure_logging, get_logger, @@ -48,7 +49,7 @@ def test_default_logging_config_uses_info_and_terminal_aware_colors() -> None: assert DEFAULT_LOGGING_CONFIG == LoggingConfig( level=logging.INFO, stream=None, - use_colors=None, + color_mode=ColorMode.AUTO, ) @@ -66,7 +67,7 @@ def test_configure_logging_colors_interactive_output() -> None: def test_configure_logging_can_disable_terminal_colors() -> None: stream = _TerminalStream() - configure_logging(LoggingConfig(stream=stream, use_colors=False)) + configure_logging(LoggingConfig(stream=stream, color_mode=ColorMode.NEVER)) logging.getLogger("privacy_guard.service").error("startup_failed") @@ -75,7 +76,7 @@ def test_configure_logging_can_disable_terminal_colors() -> None: def test_configure_logging_can_force_colors_for_redirected_output() -> None: stream = StringIO() - configure_logging(LoggingConfig(stream=stream, use_colors=True)) + configure_logging(LoggingConfig(stream=stream, color_mode=ColorMode.ALWAYS)) logging.getLogger("privacy_guard.service").info("server_started") From 9eecfe658fb7c4224f27ca36c00c00edfcec01b4 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 02:56:22 +0000 Subject: [PATCH 59/82] test(privacy-guard): cover loopback gRPC flow (PG-014) --- .../tests/service/test_grpc_integration.py | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 projects/privacy-guard/tests/service/test_grpc_integration.py diff --git a/projects/privacy-guard/tests/service/test_grpc_integration.py b/projects/privacy-guard/tests/service/test_grpc_integration.py new file mode 100644 index 0000000..850eff9 --- /dev/null +++ b/projects/privacy-guard/tests/service/test_grpc_integration.py @@ -0,0 +1,151 @@ +"""Real loopback coverage for the generated OpenShell gRPC service.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +import grpc +import pytest +from google.protobuf import empty_pb2, json_format, message_factory +from google.protobuf.message import Message + +from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 +from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc +from privacy_guard.engines.registry import create_builtin_registry +from privacy_guard.service.servicer import PrivacyGuardMiddleware + + +def _config(*, action: str = "replace") -> pb2.ValidateConfigRequest: + request = pb2.ValidateConfigRequest() + json_format.ParseDict( + { + "entity_processing": { + "stages": [ + { + "name": "identifiers", + "config": { + "engine": "regex", + "pattern_catalog": { + "entities": [ + { + "name": "email", + "patterns": [ + { + "pattern": (r"[a-z]+@[a-z]+\.[a-z]+"), + "confidence": "high", + } + ], + } + ] + }, + "replacement": { + "strategy": "template", + "template": "[{entity}]", + }, + }, + } + ] + }, + "on_detection": {"action": action}, + }, + request.config, + ) + return request + + +def _evaluation( + body: bytes, + *, + action: str = "replace", + phase: pb2.SupervisorMiddlewarePhase = ( + pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS + ), +) -> pb2.HttpRequestEvaluation: + return pb2.HttpRequestEvaluation( + phase=phase, + context=pb2.RequestContext(request_id="grpc-integration"), + config=_config(action=action).config, + body=body, + ) + + +@asynccontextmanager +async def _running_stub( + middleware: PrivacyGuardMiddleware, +) -> AsyncIterator[pb2_grpc.SupervisorMiddlewareStub]: + server = grpc.aio.server() + pb2_grpc.add_SupervisorMiddlewareServicer_to_server(middleware, server) + port = server.add_insecure_port("127.0.0.1:0") + assert port > 0 + await server.start() + channel = grpc.aio.insecure_channel(f"127.0.0.1:{port}") + try: + yield pb2_grpc.SupervisorMiddlewareStub(channel) + finally: + await channel.close() + await server.stop(grace=0) + await middleware.close() + + +@pytest.mark.asyncio +async def test_generated_stub_round_trip_covers_manifest_config_and_actions() -> None: + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + async with _running_stub(middleware) as stub: + empty_message_type = message_factory.GetMessageClass( + empty_pb2.DESCRIPTOR.message_types_by_name["Empty"] + ) + empty_message: Message = empty_message_type() + manifest = await stub.Describe(empty_message) + valid = await stub.ValidateConfig(_config()) + invalid = await stub.ValidateConfig(pb2.ValidateConfigRequest()) + detected = await stub.EvaluateHttpRequest( + _evaluation(b"contact a@b.com", action="detect") + ) + replaced = await stub.EvaluateHttpRequest(_evaluation(b"contact a@b.com")) + blocked = await stub.EvaluateHttpRequest( + _evaluation(b"contact a@b.com", action="block") + ) + clean = await stub.EvaluateHttpRequest(_evaluation(b"no match", action="block")) + + assert manifest.name == "privacy-guard" + assert len(manifest.bindings) == 1 + assert valid.valid is True + assert invalid.valid is False + assert "config_invalid" in invalid.reason + assert detected.decision == pb2.DECISION_ALLOW + assert detected.has_body is False + assert len(detected.findings) == 1 + assert replaced.decision == pb2.DECISION_ALLOW + assert replaced.has_body is True + assert replaced.body == b"contact [email]" + assert blocked.decision == pb2.DECISION_DENY + assert blocked.reason_code == "privacy_guard_blocked" + assert clean.decision == pb2.DECISION_ALLOW + + +@pytest.mark.asyncio +async def test_generated_stub_maps_invalid_and_internal_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + async with _running_stub(middleware) as stub: + with pytest.raises(grpc.aio.AioRpcError) as invalid: + await stub.EvaluateHttpRequest( + _evaluation( + b"body", + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_UNSPECIFIED, + ) + ) + assert invalid.value.code() is grpc.StatusCode.INVALID_ARGUMENT + assert "request_phase_invalid" in (invalid.value.details() or "") + + def fail_unexpectedly(values: object, body: bytes) -> None: + del values, body + raise RuntimeError + + monkeypatch.setattr(middleware, "_prepare_and_process", fail_unexpectedly) + with pytest.raises(grpc.aio.AioRpcError) as internal: + await stub.EvaluateHttpRequest(_evaluation(b"body")) + assert internal.value.code() is grpc.StatusCode.INTERNAL + assert "unexpected_service_failure" in (internal.value.details() or "") From 4cc4d7ed325061046e9b8c876300b68ba7c57d55 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 02:57:24 +0000 Subject: [PATCH 60/82] fix(privacy-guard): accelerate scalar validation (PG-001) --- .../src/privacy_guard/string_validators.py | 4 +- .../tests/test_request_processor.py | 149 ++++++++++++------ 2 files changed, 103 insertions(+), 50 deletions(-) diff --git a/projects/privacy-guard/src/privacy_guard/string_validators.py b/projects/privacy-guard/src/privacy_guard/string_validators.py index 9c12506..44942f3 100644 --- a/projects/privacy-guard/src/privacy_guard/string_validators.py +++ b/projects/privacy-guard/src/privacy_guard/string_validators.py @@ -11,7 +11,9 @@ def validate_scalar_string(value: object) -> str: """Validate and return a string containing only Unicode scalar values.""" if not isinstance(value, str): raise ValueError("value must be a string") - if any("\ud800" <= character <= "\udfff" for character in value): + try: + value.encode("utf-8", errors="strict") + except UnicodeEncodeError: raise ValueError("string must contain valid Unicode scalar values") return value diff --git a/projects/privacy-guard/tests/test_request_processor.py b/projects/privacy-guard/tests/test_request_processor.py index d75bb12..ff5d823 100644 --- a/projects/privacy-guard/tests/test_request_processor.py +++ b/projects/privacy-guard/tests/test_request_processor.py @@ -3,78 +3,97 @@ from __future__ import annotations import logging +from concurrent.futures import ThreadPoolExecutor from time import monotonic import pytest from privacy_guard.config import PolicyAction +from privacy_guard.constants import MAX_BODY_BYTES from privacy_guard.engines import RegexEngine from privacy_guard.engines.registry import EngineRegistry -from privacy_guard.errors import EngineLimitExceededError +from privacy_guard.errors import ( + EngineLimitExceededError, + ErrorCode, + PrivacyGuardError, +) from privacy_guard.request_processor import RequestDecision, RequestProcessor +from privacy_guard.string_validators import validate_scalar_string from privacy_guard.timeout import Timeout -def _values(action: PolicyAction) -> dict[str, object]: - return { - "entity_processing": { - "stages": [ - { - "name": "people", - "config": { - "engine": "regex", - "pattern_catalog": { - "entities": [ +def _values( + action: PolicyAction, + *, + include_second_stage: bool = True, +) -> dict[str, object]: + stages: list[dict[str, object]] = [ + { + "name": "people", + "config": { + "engine": "regex", + "pattern_catalog": { + "entities": [ + { + "name": "person", + "patterns": [ { - "name": "person", - "patterns": [ - { - "pattern": "Alice", - "confidence": "high", - } - ], + "pattern": "Alice", + "confidence": "high", } - ] - }, - "replacement": { - "strategy": "template", - "template": "[{entity}]", - }, - }, + ], + } + ] }, - { - "config": { - "engine": "regex", - "pattern_catalog": { - "entities": [ - { - "name": "marker", - "patterns": [ - { - "pattern": "person", - "confidence": "medium", - } - ], - } - ] - }, - "replacement": { - "strategy": "template", - "template": "<{entity}>", - }, + "replacement": { + "strategy": "template", + "template": "[{entity}]", + }, + }, + } + ] + if include_second_stage: + stages.append( + { + "config": { + "engine": "regex", + "pattern_catalog": { + "entities": [ + { + "name": "marker", + "patterns": [ + { + "pattern": "person", + "confidence": "medium", + } + ], + } + ] + }, + "replacement": { + "strategy": "template", + "template": "<{entity}>", }, }, - ] - }, + } + ) + return { + "entity_processing": {"stages": stages}, "on_detection": {"action": action.value}, } -def _processor(action: PolicyAction) -> RequestProcessor: +def _processor( + action: PolicyAction, + *, + include_second_stage: bool = True, +) -> RequestProcessor: registry = EngineRegistry() registry.register(RegexEngine) registry.finalize() - config = registry.validate_config(_values(action)) + config = registry.validate_config( + _values(action, include_second_stage=include_second_stage) + ) stages = tuple( ( stage.diagnostic_name(index), @@ -116,6 +135,38 @@ def test_block_is_a_processor_disposition_not_an_engine_strategy() -> None: assert tuple(item.entity for item in result.detection_summaries) == ("person",) +def test_scalar_validation_rejects_lone_surrogates() -> None: + with pytest.raises(ValueError, match="Unicode scalar"): + validate_scalar_string("\ud800") + + +def test_processor_accepts_exact_body_limit_and_rejects_one_byte_more() -> None: + processor = _processor(PolicyAction.DETECT, include_second_stage=False) + + exact = processor.process("x" * MAX_BODY_BYTES) + + assert exact.decision is RequestDecision.ALLOW + with pytest.raises(PrivacyGuardError) as captured: + processor.process("x" * (MAX_BODY_BYTES + 1)) + assert captured.value.code is ErrorCode.REQUEST_BODY_TOO_LARGE + + +def test_exact_limit_requests_complete_concurrently() -> None: + processor = _processor(PolicyAction.DETECT, include_second_stage=False) + text = "x" * MAX_BODY_BYTES + + with ThreadPoolExecutor(max_workers=4) as executor: + results = tuple(executor.map(processor.process, (text,) * 4)) + + assert all(result.decision is RequestDecision.ALLOW for result in results) + + +def test_exact_limit_request_completes_with_multiple_stages() -> None: + result = _processor(PolicyAction.DETECT).process("x" * MAX_BODY_BYTES) + + assert result.decision is RequestDecision.ALLOW + + def test_timeout_returns_the_bounded_limit_deny( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, From aa3d455debb757f0b4ca569836ec4e30083d2429 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 03:01:38 +0000 Subject: [PATCH 61/82] fix(privacy-guard): bound and normalize transport config (PG-002 PG-003) --- .../architecture/configuration.md | 18 +- .../architecture/entity-processing-engines.md | 7 + .../architecture/service-boundary.md | 6 +- .../privacy-guard/src/privacy_guard/config.py | 3 + .../src/privacy_guard/constants.py | 7 + .../privacy-guard/src/privacy_guard/errors.py | 9 + .../src/privacy_guard/service/servicer.py | 71 ++++++- .../tests/service/test_grpc_integration.py | 146 +++++++++++++- .../tests/service/test_servicer.py | 188 +++++++++++++++--- projects/privacy-guard/tests/test_config.py | 16 ++ 10 files changed, 432 insertions(+), 39 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/configuration.md b/docs/documentation/privacy-guard/architecture/configuration.md index 3a1c5bd..576dc25 100644 --- a/docs/documentation/privacy-guard/architecture/configuration.md +++ b/docs/documentation/privacy-guard/architecture/configuration.md @@ -102,10 +102,20 @@ bounded by that transport. A file-backed configuration carries only its bounded relative path through the protocol and loads the deployment-mounted catalog in the middleware process. -The service validates the complete configuration, computes its canonical -fingerprint, and uses a bounded internal `RequestProcessor` cache. Caching -avoids repeated engine initialization but does not increase the transport -limit. +The service rejects an encoded configuration above that limit before protobuf +conversion or policy validation. A policy may contain at most ten ordered +entity-processing stages. The service validates the complete bounded +configuration, computes its canonical fingerprint, and uses a bounded internal +`RequestProcessor` cache. Caching avoids repeated engine initialization but +does not increase either limit. + +`Struct` carries every number as a double. At this transport boundary only, +Privacy Guard converts finite integral values in the safe integer range +`-(2^53 - 1)` through `2^53 - 1` to Python integers before strict policy +validation. Non-integral and out-of-range values remain floats, so integer +fields reject them. Direct Python registry validation remains strict and does +not perform this transport normalization. Custom-engine integer settings must +therefore remain within the safe range when supplied through OpenShell policy. A future self-contained transport for larger expanded catalogs requires an upstream OpenShell contract for preparing configuration and referring to it diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md index 4d57898..0203576 100644 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -47,6 +47,13 @@ The object under `EntityProcessingStage.config` is this exact concrete model. It is validated and serialized as a member of the registry-built Pydantic discriminated union, then passed unchanged to the engine constructor. +Custom integer fields remain strict for direct Python callers. OpenShell policy +uses protobuf `Struct`, whose numeric representation is a double, so the service +normalizes finite integral values only within the safe range `-(2^53 - 1)` +through `2^53 - 1`. Values outside that range and non-integral values remain +floats and fail strict integer fields. Nested models and lists follow the same +transport rule. + `EngineConfig` is a nominal, strict base model. It does not prescribe a `replacement` field or any other algorithm-specific setting. An engine that needs a replacement recipe declares that field on its concrete config. Engines diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index ddf4d97..16d8ada 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -78,9 +78,11 @@ RPC first. For each evaluation, the service: 1. validates the pre-credentials phase -2. validates the transport body byte limit +2. validates the encoded context (4 KiB), configuration (64 KiB), target + (32 KiB), headers (128 entries and 64 KiB), and body (4 MiB) limits 3. acquires one bounded worker slot -4. validates, normalizes, and resolves the supplied policy config in that worker +4. converts transport-safe integral numbers, validates the maximum ten stages, + and resolves the supplied policy config in that worker 5. allows an empty body without invoking an engine 6. decodes a non-empty body as strict UTF-8 7. calls `RequestProcessor.process(text)` in the same worker diff --git a/projects/privacy-guard/src/privacy_guard/config.py b/projects/privacy-guard/src/privacy_guard/config.py index e9d67ec..70a7df2 100644 --- a/projects/privacy-guard/src/privacy_guard/config.py +++ b/projects/privacy-guard/src/privacy_guard/config.py @@ -19,6 +19,7 @@ ) from privacy_guard.base import StrictDomainModel +from privacy_guard.constants import MAX_ENTITY_PROCESSING_STAGES from privacy_guard.engines import EngineConfig from privacy_guard.string_validators import ( BoundedMetadataString, @@ -87,6 +88,8 @@ class EntityProcessingStages( def _parse_stages(cls, value: object) -> object: if not isinstance(value, list | tuple) or not value: raise ValueError("stages must be a non-empty list") + if len(value) > MAX_ENTITY_PROCESSING_STAGES: + raise ValueError("policy has too many entity-processing stages") return tuple(value) @model_validator(mode="after") diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index dd62524..270bc9b 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -41,6 +41,7 @@ MAX_PROTO_FINDING_BYTES = 4 * 1024 # Engine configuration and regex execution limits. +MAX_ENTITY_PROCESSING_STAGES = 10 MAX_REGEX_NAME_BYTES = 128 MAX_REGEX_ENTITIES_PER_CATALOG = 2_000 MAX_REGEX_PATTERNS_PER_CATALOG = 10_000 @@ -51,6 +52,12 @@ # Service concurrency and transport limits. MAX_CONCURRENT_PROCESSING = 4 MAX_CONCURRENT_RPCS = 16 +# Mirrored from the encoded OpenShell v0.0.90 middleware contract. +MAX_PROTO_CONTEXT_BYTES = 4 * 1024 +MAX_PROTO_CONFIG_BYTES = 64 * 1024 +MAX_PROTO_TARGET_BYTES = 32 * 1024 +MAX_PROTO_HEADERS = 128 +MAX_PROTO_HEADERS_BYTES = 64 * 1024 PROTOBUF_ENVELOPE_ALLOWANCE_BYTES = 1024 * 1024 MAX_RECEIVE_MESSAGE_BYTES = MAX_BODY_BYTES + PROTOBUF_ENVELOPE_ALLOWANCE_BYTES diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py index 313a8a7..05bfe1c 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -30,6 +30,7 @@ class ErrorCode(StrEnum): CONFIG_INVALID = "config_invalid" REQUEST_PHASE_INVALID = "request_phase_invalid" + REQUEST_ENVELOPE_INVALID = "request_envelope_invalid" REQUEST_BODY_TOO_LARGE = "request_body_too_large" BODY_ENCODING_INVALID = "body_encoding_invalid" ENGINE_OUTPUT_INVALID = "engine_output_invalid" @@ -135,6 +136,14 @@ class EngineRegistryError(Exception): "Request evaluation phase is invalid.", "Use the advertised pre-credentials phase.", ), + ErrorCode.REQUEST_ENVELOPE_INVALID: _ErrorSpec( + ErrorKind.INVALID_INPUT, + ErrorComponent.SERVICE, + "validate_envelope", + "Request transport metadata exceeds an advertised limit.", + "Reduce the request context, target, or headers to the OpenShell " + "middleware contract limits.", + ), ErrorCode.REQUEST_BODY_TOO_LARGE: _ErrorSpec( ErrorKind.INVALID_INPUT, ErrorComponent.SERVICE, diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index 4d149f5..68db547 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -3,9 +3,10 @@ from __future__ import annotations import asyncio +import math import time from collections import OrderedDict -from collections.abc import Callable +from collections.abc import Callable, Iterable from concurrent.futures import Future, ThreadPoolExecutor from threading import RLock from typing import Never, Protocol, TypedDict, TypeVar @@ -25,8 +26,13 @@ LIMIT_REASON_CODE, MAX_BODY_BYTES, MAX_CONCURRENT_PROCESSING, + MAX_PROTO_CONFIG_BYTES, + MAX_PROTO_CONTEXT_BYTES, MAX_PROTO_FINDING_BYTES, MAX_PROTO_FINDING_GROUPS, + MAX_PROTO_HEADERS, + MAX_PROTO_HEADERS_BYTES, + MAX_PROTO_TARGET_BYTES, REASON_CODE_PATTERN, SERVICE_NAME, SERVICE_VERSION, @@ -122,6 +128,8 @@ def _validate_config( request: pb2.ValidateConfigRequest, ) -> pb2.ValidateConfigResponse: try: + if request.config.ByteSize() > MAX_PROTO_CONFIG_BYTES: + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) self._registry.validate_config(_mapping_from_proto(request.config)) except PrivacyGuardError as error: return pb2.ValidateConfigResponse(valid=False, reason=str(error)) @@ -182,17 +190,18 @@ async def _evaluate_http_request( raise PrivacyGuardError(ErrorCode.REQUEST_PHASE_INVALID) if len(request.body) > MAX_BODY_BYTES: raise PrivacyGuardError(ErrorCode.REQUEST_BODY_TOO_LARGE) - values = _mapping_from_proto(request.config) + _validate_evaluation_envelope(request) result = await self._run_in_worker( - lambda: self._prepare_and_process(values, request.body) + lambda: self._prepare_and_process(request.config, request.body) ) return _result_to_proto(result) def _prepare_and_process( self, - values: object, + config: Message, body: bytes, ) -> RequestProcessingResult: + values = _mapping_from_proto(config) processor = self._processors.resolve(values) if not body: return RequestProcessingResult(decision=RequestDecision.ALLOW) @@ -314,9 +323,60 @@ def _evaluation_log_extra( def _mapping_from_proto(config: Message) -> dict[str, object]: try: - return json_format.MessageToDict(config) + values: object = json_format.MessageToDict(config) except Exception: raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None + if not isinstance(values, dict) or any(not isinstance(key, str) for key in values): + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) + return { + key: _normalize_proto_numbers(item) + for key, item in values.items() + if isinstance(key, str) + } + + +def _normalize_proto_numbers(value: object) -> object: + if isinstance(value, float): + if ( + math.isfinite(value) + and value.is_integer() + and -_MAX_PROTO_SAFE_INTEGER <= value <= _MAX_PROTO_SAFE_INTEGER + ): + return int(value) + return value + if isinstance(value, list): + return [_normalize_proto_numbers(item) for item in value] + if isinstance(value, dict): + return {key: _normalize_proto_numbers(item) for key, item in value.items()} + return value + + +def _validate_evaluation_envelope(request: pb2.HttpRequestEvaluation) -> None: + if request.config.ByteSize() > MAX_PROTO_CONFIG_BYTES: + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) + if ( + request.context.ByteSize() > MAX_PROTO_CONTEXT_BYTES + or request.target.ByteSize() > MAX_PROTO_TARGET_BYTES + or len(request.headers) > MAX_PROTO_HEADERS + or _encoded_headers_size(request.headers) > MAX_PROTO_HEADERS_BYTES + ): + raise PrivacyGuardError(ErrorCode.REQUEST_ENVELOPE_INVALID) + + +def _encoded_headers_size(headers: Iterable[Message]) -> int: + total = 0 + for header in headers: + size = header.ByteSize() + total += 1 + _varint_size(size) + size + return total + + +def _varint_size(value: int) -> int: + size = 1 + while value >= 0x80: + value >>= 7 + size += 1 + return size def _result_to_proto(result: RequestProcessingResult) -> pb2.HttpRequestResult: @@ -377,6 +437,7 @@ def _limit_deny() -> pb2.HttpRequestResult: _LOGGER = get_logger(__name__) _MAX_CACHED_PROCESSORS = 128 +_MAX_PROTO_SAFE_INTEGER = (1 << 53) - 1 __all__ = ["PrivacyGuardMiddleware"] diff --git a/projects/privacy-guard/tests/service/test_grpc_integration.py b/projects/privacy-guard/tests/service/test_grpc_integration.py index 850eff9..0180dfd 100644 --- a/projects/privacy-guard/tests/service/test_grpc_integration.py +++ b/projects/privacy-guard/tests/service/test_grpc_integration.py @@ -4,16 +4,27 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager +from typing import Literal import grpc import pytest from google.protobuf import empty_pb2, json_format, message_factory from google.protobuf.message import Message +from pydantic import field_validator +from privacy_guard.base import StrictDomainModel from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc -from privacy_guard.engines.registry import create_builtin_registry +from privacy_guard.engines import ( + EngineConfig, + EntityProcessingEngine, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry +from privacy_guard.errors import PrivacyGuardError from privacy_guard.service.servicer import PrivacyGuardMiddleware +from privacy_guard.timeout import Timeout def _config(*, action: str = "replace") -> pb2.ValidateConfigRequest: @@ -54,6 +65,16 @@ def _config(*, action: str = "replace") -> pb2.ValidateConfigRequest: return request +def _config_with_stages(stage_count: int) -> pb2.ValidateConfigRequest: + values = json_format.MessageToDict(_config(action="detect").config) + stage = values["entity_processing"]["stages"][0] + stage.pop("name") + values["entity_processing"]["stages"] = [stage] * stage_count + request = pb2.ValidateConfigRequest() + json_format.ParseDict(values, request.config) + return request + + def _evaluation( body: bytes, *, @@ -149,3 +170,126 @@ def fail_unexpectedly(values: object, body: bytes) -> None: await stub.EvaluateHttpRequest(_evaluation(b"body")) assert internal.value.code() is grpc.StatusCode.INTERNAL assert "unexpected_service_failure" in (internal.value.details() or "") + + +@pytest.mark.asyncio +async def test_generated_stub_enforces_ten_stage_limit() -> None: + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + async with _running_stub(middleware) as stub: + exact_config = _config_with_stages(10) + oversized_config = _config_with_stages(11) + exact_validation = await stub.ValidateConfig(exact_config) + oversized_validation = await stub.ValidateConfig(oversized_config) + exact_evaluation = _evaluation(b"no match", action="detect") + exact_evaluation.config.CopyFrom(exact_config.config) + exact_result = await stub.EvaluateHttpRequest(exact_evaluation) + oversized_evaluation = _evaluation(b"no match", action="detect") + oversized_evaluation.config.CopyFrom(oversized_config.config) + with pytest.raises(grpc.aio.AioRpcError) as oversized_result: + await stub.EvaluateHttpRequest(oversized_evaluation) + + assert exact_validation.valid is True + assert oversized_validation.valid is False + assert exact_result.decision == pb2.DECISION_ALLOW + assert oversized_result.value.code() is grpc.StatusCode.INVALID_ARGUMENT + assert "config_invalid" in (oversized_result.value.details() or "") + + +class _NumericNestedConfig(StrictDomainModel): + count: int + + +class _NumericEngineConfig(EngineConfig): + engine: Literal["numeric"] = "numeric" + threshold: int + ratio: float + nested: _NumericNestedConfig + values: tuple[int, ...] + + @field_validator("values", mode="before") + @classmethod + def _values_are_a_tuple(cls, value: object) -> object: + if not isinstance(value, list | tuple): + raise ValueError("values must be a list") + return tuple(value) + + +class _NumericEngine(EntityProcessingEngine[_NumericEngineConfig]): + supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + del strategy, timeout + return TextProcessingResult(text=text, detections=()) + + +def _numeric_values( + threshold: int | float, + *, + ratio: float = 3.0, +) -> dict[str, object]: + return { + "entity_processing": { + "stages": [ + { + "config": { + "engine": "numeric", + "threshold": threshold, + "ratio": ratio, + "nested": {"count": 4}, + "values": [5, 6], + } + } + ] + }, + "on_detection": {"action": "detect"}, + } + + +def _numeric_request( + threshold: int | float, + *, + ratio: float = 3.0, +) -> pb2.ValidateConfigRequest: + request = pb2.ValidateConfigRequest() + json_format.ParseDict(_numeric_values(threshold, ratio=ratio), request.config) + return request + + +def _numeric_registry() -> EngineRegistry: + registry = EngineRegistry() + registry.register(_NumericEngine) + return registry.finalize() + + +@pytest.mark.asyncio +async def test_generated_stub_normalizes_transport_safe_integral_numbers() -> None: + registry = _numeric_registry() + with pytest.raises(PrivacyGuardError): + registry.validate_config(_numeric_values(3.0)) + + middleware = PrivacyGuardMiddleware(registry) + async with _running_stub(middleware) as stub: + ordinary = await stub.ValidateConfig(_numeric_request(3, ratio=3.5)) + safe_max = await stub.ValidateConfig(_numeric_request((1 << 53) - 1)) + safe_min = await stub.ValidateConfig(_numeric_request(-((1 << 53) - 1))) + non_integral = await stub.ValidateConfig(_numeric_request(3.5)) + beyond_safe = await stub.ValidateConfig(_numeric_request(1 << 53)) + evaluation = pb2.HttpRequestEvaluation( + phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, + config=_numeric_request(3).config, + body=b"body", + ) + result = await stub.EvaluateHttpRequest(evaluation) + + assert ordinary.valid is True + assert safe_max.valid is True + assert safe_min.valid is True + assert non_integral.valid is False + assert beyond_safe.valid is False + assert result.decision == pb2.DECISION_ALLOW diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index cd4877c..f5efd47 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -4,6 +4,7 @@ import asyncio import logging +from copy import deepcopy from threading import get_ident import pytest @@ -15,7 +16,12 @@ from privacy_guard.constants import ( LIMIT_REASON, LIMIT_REASON_CODE, + MAX_PROTO_CONFIG_BYTES, + MAX_PROTO_CONTEXT_BYTES, MAX_PROTO_FINDING_BYTES, + MAX_PROTO_HEADERS, + MAX_PROTO_HEADERS_BYTES, + MAX_PROTO_TARGET_BYTES, ) from privacy_guard.engines import EngineConfig from privacy_guard.engines.registry import create_builtin_registry @@ -30,34 +36,35 @@ from privacy_guard.service.servicer import PrivacyGuardMiddleware -def _values(action: str = "replace") -> dict[str, object]: - return { - "entity_processing": { - "stages": [ - { - "config": { - "engine": "regex", - "pattern_catalog": { - "entities": [ - { - "name": "email", - "patterns": [ - { - "pattern": r"[a-z]+@[a-z]+\.[a-z]+", - "confidence": "high", - } - ], - } - ] - }, - "replacement": { - "strategy": "template", - "template": "[{entity}]", - }, +def _values( + action: str = "replace", + *, + stage_count: int = 1, +) -> dict[str, object]: + stage: dict[str, object] = { + "config": { + "engine": "regex", + "pattern_catalog": { + "entities": [ + { + "name": "email", + "patterns": [ + { + "pattern": r"[a-z]+@[a-z]+\.[a-z]+", + "confidence": "high", + } + ], } - } - ] - }, + ] + }, + "replacement": { + "strategy": "template", + "template": "[{entity}]", + }, + } + } + return { + "entity_processing": {"stages": [deepcopy(stage) for _ in range(stage_count)]}, "on_detection": {"action": action}, } @@ -100,6 +107,104 @@ def test_validate_config_is_pure_and_reports_invalid_config() -> None: assert "config_invalid" in invalid.reason +def test_validate_config_rejects_oversized_proto_before_registry_validation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + validation_count = 0 + original_validate = servicer_module.EngineRegistry.validate_config + + def record_validation( + registry: servicer_module.EngineRegistry, + values: object, + ) -> PrivacyGuardConfig[EngineConfig]: + nonlocal validation_count + validation_count += 1 + return original_validate(registry, values) + + monkeypatch.setattr( + servicer_module.EngineRegistry, + "validate_config", + record_validation, + ) + exact_config = pb2.ValidateConfigRequest() + json_format.ParseDict({"padding": "x" * 65_515}, exact_config.config) + oversized_config = pb2.ValidateConfigRequest() + json_format.ParseDict({"padding": "x" * 65_516}, oversized_config.config) + assert exact_config.config.ByteSize() == MAX_PROTO_CONFIG_BYTES + assert oversized_config.config.ByteSize() == MAX_PROTO_CONFIG_BYTES + 1 + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + try: + exact = middleware._validate_config(exact_config) + oversized = middleware._validate_config(oversized_config) + finally: + asyncio.run(middleware.close()) + + assert exact.valid is False + assert oversized.valid is False + assert validation_count == 1 + + +def test_evaluation_enforces_exact_encoded_transport_boundaries() -> None: + request = _request(b"") + request.context.request_id = "x" * 4_093 + assert request.context.ByteSize() == MAX_PROTO_CONTEXT_BYTES + servicer_module._validate_evaluation_envelope(request) + request.context.request_id += "x" + assert request.context.ByteSize() == MAX_PROTO_CONTEXT_BYTES + 1 + with pytest.raises(PrivacyGuardError) as context_error: + servicer_module._validate_evaluation_envelope(request) + assert context_error.value.code is ErrorCode.REQUEST_ENVELOPE_INVALID + + request = _request(b"") + request.target.host = "x" * 32_764 + assert request.target.ByteSize() == MAX_PROTO_TARGET_BYTES + servicer_module._validate_evaluation_envelope(request) + request.target.host += "x" + assert request.target.ByteSize() == MAX_PROTO_TARGET_BYTES + 1 + with pytest.raises(PrivacyGuardError) as target_error: + servicer_module._validate_evaluation_envelope(request) + assert target_error.value.code is ErrorCode.REQUEST_ENVELOPE_INVALID + + request = _request(b"") + request.headers.add(name="x", value="x" * 65_525) + assert servicer_module._encoded_headers_size(request.headers) == ( + MAX_PROTO_HEADERS_BYTES + ) + servicer_module._validate_evaluation_envelope(request) + request.headers[0].value += "x" + assert servicer_module._encoded_headers_size(request.headers) == ( + MAX_PROTO_HEADERS_BYTES + 1 + ) + with pytest.raises(PrivacyGuardError) as header_size_error: + servicer_module._validate_evaluation_envelope(request) + assert header_size_error.value.code is ErrorCode.REQUEST_ENVELOPE_INVALID + + request = _request(b"") + for _ in range(MAX_PROTO_HEADERS): + request.headers.add() + servicer_module._validate_evaluation_envelope(request) + request.headers.add() + with pytest.raises(PrivacyGuardError) as header_count_error: + servicer_module._validate_evaluation_envelope(request) + assert header_count_error.value.code is ErrorCode.REQUEST_ENVELOPE_INVALID + + +def test_evaluation_enforces_exact_encoded_config_boundary() -> None: + request = _request(b"") + request.config.Clear() + json_format.ParseDict({"padding": "x" * 65_515}, request.config) + assert request.config.ByteSize() == MAX_PROTO_CONFIG_BYTES + servicer_module._validate_evaluation_envelope(request) + request.config.Clear() + json_format.ParseDict({"padding": "x" * 65_516}, request.config) + assert request.config.ByteSize() == MAX_PROTO_CONFIG_BYTES + 1 + + with pytest.raises(PrivacyGuardError) as captured: + servicer_module._validate_evaluation_envelope(request) + + assert captured.value.code is ErrorCode.CONFIG_INVALID + + def test_limit_deny_explains_recovery_options() -> None: result = servicer_module._result_to_proto( RequestProcessingResult( @@ -241,6 +346,35 @@ async def evaluate_twice() -> None: assert validation_count == 2 +def test_oversized_stage_list_fails_before_fingerprinting_or_construction( + monkeypatch: pytest.MonkeyPatch, +) -> None: + values = _values(action="detect", stage_count=10_000) + + def unexpected_call(*args: object, **kwargs: object) -> object: + del args, kwargs + raise AssertionError("oversized stage list reached preparation") + + monkeypatch.setattr( + servicer_module, + "configuration_fingerprint", + unexpected_call, + ) + monkeypatch.setattr( + servicer_module.EngineRegistry, + "create_engine", + unexpected_call, + ) + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + try: + with pytest.raises(PrivacyGuardError) as captured: + middleware._processors.resolve(values) + finally: + asyncio.run(middleware.close()) + + assert captured.value.code is ErrorCode.CONFIG_INVALID + + def test_invalid_utf8_fails_before_invoking_an_engine() -> None: async def evaluate() -> None: middleware = PrivacyGuardMiddleware(create_builtin_registry()) diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py index 578d37a..a488828 100644 --- a/projects/privacy-guard/tests/test_config.py +++ b/projects/privacy-guard/tests/test_config.py @@ -90,6 +90,22 @@ def test_known_discriminator_constructs_the_exact_engine_config() -> None: assert stage.diagnostic_name(1) == "regex[1]" +def test_policy_accepts_ten_stages_and_rejects_eleven() -> None: + registry = _registry() + exact = _config() + stage = deepcopy(exact["entity_processing"]["stages"][0]) + exact["entity_processing"]["stages"] = [deepcopy(stage) for _ in range(10)] + oversized = deepcopy(exact) + oversized["entity_processing"]["stages"].append(deepcopy(stage)) + + parsed = registry.validate_config(exact) + + assert len(parsed.entity_processing.stages) == 10 + with pytest.raises(PrivacyGuardError) as captured: + registry.validate_config(oversized) + assert captured.value.code is ErrorCode.CONFIG_INVALID + + def test_explicit_stage_name_is_the_diagnostic_source() -> None: config = _registry().validate_config(_config(stage_name="credentials")) From d69d30bfc9fb6a7159203a6b3a9c7016a787d02f Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 03:03:17 +0000 Subject: [PATCH 62/82] fix(privacy-guard): validate bound listen ports (PG-004) --- .../architecture/service-boundary.md | 5 ++ projects/privacy-guard/README.md | 3 + .../src/privacy_guard/service/server.py | 34 ++++++++- .../tests/service/test_server.py | 75 +++++++++++++++++-- 4 files changed, 110 insertions(+), 7 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index 16d8ada..23c6f9a 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -226,6 +226,11 @@ Async applications call `await server.serve_async(address)` instead. The explicit suffixes make the blocking behavior visible at each call site. Both entry points use the same server instance and lifecycle. +The listen address uses `host:port` or bracketed IPv6 `[address]:port` form, +with an explicit port from 1 through 65535. Privacy Guard rejects zero and +out-of-range numeric ports before calling gRPC, then verifies that gRPC bound +the requested port before starting the service. + The server: 1. creates an unstarted async gRPC server with receive and concurrency limits diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index a28c7d6..977fd88 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -184,6 +184,9 @@ explicitly named asynchronous counterpart: await server.serve_async("127.0.0.1:50051") ``` +Listen addresses use `host:port` or bracketed IPv6 `[address]:port` form. The +port must be an explicit integer from 1 through 65535. + Custom applications pass their own finalized registry to `PrivacyGuardServer`. The server owns the middleware adapter, bounded gRPC transport, worker executor, and shutdown lifecycle; it does not load Python diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index edcbd9f..b42b5d7 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -46,12 +46,13 @@ def serve_sync(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: async def serve_async(self, listen: str = DEFAULT_LISTEN_ADDRESS) -> None: """Serve asynchronously until termination, then close owned resources.""" server = _create_grpc_server(self._middleware) - _LOGGER.info("privacy_guard_server_starting listen=%s", listen) try: try: + requested_port = _validated_listen_port(listen) bound_port = server.add_insecure_port(listen) - if bound_port == 0: + if bound_port != requested_port: raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) + _LOGGER.info("privacy_guard_server_bound listen=%r", listen) await server.start() except RuntimeError: raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) from None @@ -87,6 +88,35 @@ async def _stop_grpc_server(server: grpc.aio.Server) -> None: raise +def _validated_listen_port(listen: str) -> int: + if not isinstance(listen, str): + raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) + if listen.startswith("["): + closing_bracket = listen.rfind("]") + if ( + closing_bracket < 2 + or listen[closing_bracket + 1 : closing_bracket + 2] != ":" + ): + raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) + host = listen[1:closing_bracket] + port_text = listen[closing_bracket + 2 :] + else: + host, separator, port_text = listen.rpartition(":") + if not separator or not host or ":" in host: + raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) + if ( + not host + or not port_text + or not port_text.isascii() + or not port_text.isdecimal() + ): + raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) + port = int(port_text) + if not 1 <= port <= 65_535: + raise PrivacyGuardError(ErrorCode.SERVER_BIND_FAILED) + return port + + __all__ = [ "DEFAULT_LISTEN_ADDRESS", "PrivacyGuardServer", diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index eeca94b..f587dd7 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import logging import subprocess import sys @@ -233,8 +234,9 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: @pytest.mark.asyncio async def test_serve_async_starts_waits_and_closes_on_normal_termination( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: - fake_server = _LifecycleServerFake() + fake_server = _LifecycleServerFake(bound_port=50053) closed: list[PrivacyGuardMiddleware] = [] async def record_close(middleware: PrivacyGuardMiddleware) -> None: @@ -244,20 +246,25 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) - await server.serve_async("127.0.0.1:50053") + with caplog.at_level(logging.INFO, logger="privacy_guard.service.server"): + await server.serve_async("127.0.0.1:50053") assert fake_server.addresses == ["127.0.0.1:50053"] assert fake_server.started is True assert fake_server.waited is True assert fake_server.stop_graces == [0] assert closed == [server._middleware] + assert "privacy_guard_server_bound listen='127.0.0.1:50053'" in caplog.text @pytest.mark.asyncio async def test_serve_async_propagates_cancellation_after_closing_resources( monkeypatch: pytest.MonkeyPatch, ) -> None: - fake_server = _LifecycleServerFake(wait_error=asyncio.CancelledError()) + fake_server = _LifecycleServerFake( + bound_port=50054, + wait_error=asyncio.CancelledError(), + ) closed: list[PrivacyGuardMiddleware] = [] async def record_close(middleware: PrivacyGuardMiddleware) -> None: @@ -279,7 +286,7 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: async def test_serve_async_preserves_cancellation_during_server_shutdown( monkeypatch: pytest.MonkeyPatch, ) -> None: - fake_server = _LifecycleServerFake(block_stop=True) + fake_server = _LifecycleServerFake(bound_port=50055, block_stop=True) closed: list[PrivacyGuardMiddleware] = [] async def record_close(middleware: PrivacyGuardMiddleware) -> None: @@ -308,7 +315,10 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: async def test_serve_async_sanitizes_startup_failures_and_closes_resources( monkeypatch: pytest.MonkeyPatch, ) -> None: - fake_server = _LifecycleServerFake(start_error=RuntimeError("startup failed")) + fake_server = _LifecycleServerFake( + bound_port=50056, + start_error=RuntimeError("startup failed"), + ) closed: list[PrivacyGuardMiddleware] = [] async def record_close(middleware: PrivacyGuardMiddleware) -> None: @@ -330,5 +340,60 @@ async def record_close(middleware: PrivacyGuardMiddleware) -> None: assert closed == [server._middleware] +@pytest.mark.parametrize( + ("listen", "port"), + [ + ("127.0.0.1:1", 1), + ("middleware.local:65535", 65_535), + ("[::1]:50051", 50_051), + ], +) +def test_listen_address_accepts_supported_tcp_forms(listen: str, port: int) -> None: + assert server_module._validated_listen_port(listen) == port + + +@pytest.mark.parametrize( + "listen", + [ + "127.0.0.1:0", + "127.0.0.1:65536", + "127.0.0.1:99999", + "127.0.0.1:-1", + "[::1]", + "::1:50051", + ], +) +def test_listen_address_rejects_invalid_numeric_ports_and_forms( + listen: str, +) -> None: + with pytest.raises(PrivacyGuardError) as captured: + server_module._validated_listen_port(listen) + + assert captured.value.code is ErrorCode.SERVER_BIND_FAILED + + +@pytest.mark.asyncio +async def test_serve_async_rejects_mismatched_bound_port( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_server = _LifecycleServerFake(bound_port=34_463) + closed: list[PrivacyGuardMiddleware] = [] + + async def record_close(middleware: PrivacyGuardMiddleware) -> None: + closed.append(middleware) + + server = PrivacyGuardServer(create_builtin_registry()) + monkeypatch.setattr(server_module, "_create_grpc_server", lambda _: fake_server) + monkeypatch.setattr(PrivacyGuardMiddleware, "close", record_close) + + with pytest.raises(PrivacyGuardError) as captured: + await server.serve_async("127.0.0.1:9999") + + assert captured.value.code is ErrorCode.SERVER_BIND_FAILED + assert fake_server.started is False + assert fake_server.stop_graces == [0] + assert closed == [server._middleware] + + def _middleware() -> PrivacyGuardMiddleware: return PrivacyGuardMiddleware(create_builtin_registry()) From e0ea80fb7365e4b33d7992b39b7078c215d01087 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 03:07:04 +0000 Subject: [PATCH 63/82] fix(privacy-guard): secure diagnostic identifiers (PG-005) --- .../architecture/safety-and-limits.md | 18 ++- .../src/privacy_guard/service/servicer.py | 11 +- .../src/privacy_guard/string_validators.py | 4 +- .../privacy-guard/tests/engines/test_base.py | 46 ++++++++ .../tests/service/test_servicer.py | 104 ++++++++++++++++++ 5 files changed, 179 insertions(+), 4 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index b2a7c9e..cd7423d 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -70,8 +70,12 @@ and configuration or engine preparation beyond this internal bound. The diagnostic-string bound applies to stage names, entity names, metadata keys and values, model-profile names, and other audit-safe identifiers built -from the shared domain field type. Regex entity and supplied pattern names have a -stricter ASCII grammar and limit described below. +from the shared domain field type. These values must be non-empty printable +Unicode without control or bidirectional formatting characters. Untrusted +request IDs use the same content and size rules for logging; an invalid request +ID is represented by a constant placeholder and does not change the evaluation +result. Regex entity and supplied pattern names have a stricter ASCII grammar +and limit described below. The processor aggregates occurrences before the service applies protobuf limits. If a safe result cannot be represented, the service returns a limit @@ -211,6 +215,8 @@ Custom engines should: - use the base constructor and public `run()` wrapper - return the exact `TextProcessingResult` contract +- return stable, declared entity identifiers rather than values derived from + request text - keep request data local to `_run()` - keep initialized state immutable and make resources concurrency-safe - pass the shared remaining timeout to delegated APIs when they support it @@ -222,6 +228,14 @@ Custom engines should: exception hierarchy - avoid logging input or caught exception text +Privacy Guard keeps framework-controlled fields and identifiers that pass its +shared validation content-safe for findings and operational logs. +Operator-installed Python engine code is trusted deployment code: it can access +request text and must honor the declared identifier and logging contract. The +framework cannot make arbitrary trusted Python code non-exfiltrating, so +operators must review and install custom engines with the same care as other +code in the middleware process. + Do not add retries, fallback providers, persistent request artifacts, or extra validation without a concrete failure mode and a clear owning layer. diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index 68db547..2690083 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -52,6 +52,7 @@ RequestProcessingResult, RequestProcessor, ) +from privacy_guard.string_validators import validate_bounded_metadata_string from privacy_guard.timeout import validate_timeout_seconds @@ -144,7 +145,7 @@ async def _evaluate_rpc( context: _AbortContext, ) -> pb2.HttpRequestResult: started = time.monotonic() - request_id = request.context.request_id + request_id = _request_id_for_logging(request.context.request_id) failure: PrivacyGuardError | None = None action = "error" finding_count = 0 @@ -321,6 +322,13 @@ def _evaluation_log_extra( } +def _request_id_for_logging(request_id: object) -> str: + try: + return validate_bounded_metadata_string(request_id) + except ValueError: + return _INVALID_REQUEST_ID + + def _mapping_from_proto(config: Message) -> dict[str, object]: try: values: object = json_format.MessageToDict(config) @@ -436,6 +444,7 @@ def _limit_deny() -> pb2.HttpRequestResult: _LOGGER = get_logger(__name__) +_INVALID_REQUEST_ID = "invalid" _MAX_CACHED_PROCESSORS = 128 _MAX_PROTO_SAFE_INTEGER = (1 << 53) - 1 diff --git a/projects/privacy-guard/src/privacy_guard/string_validators.py b/projects/privacy-guard/src/privacy_guard/string_validators.py index 44942f3..e45c220 100644 --- a/projects/privacy-guard/src/privacy_guard/string_validators.py +++ b/projects/privacy-guard/src/privacy_guard/string_validators.py @@ -19,10 +19,12 @@ def validate_scalar_string(value: object) -> str: def validate_bounded_metadata_string(value: object) -> str: - """Validate and return a non-empty, size-bounded metadata string.""" + """Validate and return a non-empty, printable, bounded metadata string.""" validated = validate_scalar_string(value) if not validated: raise ValueError("string must not be empty") + if not validated.isprintable(): + raise ValueError("metadata must contain only printable characters") if ( len(validated) > MAX_DIAGNOSTIC_TEXT_BYTES or len(validated.encode("utf-8")) > MAX_DIAGNOSTIC_TEXT_BYTES diff --git a/projects/privacy-guard/tests/engines/test_base.py b/projects/privacy-guard/tests/engines/test_base.py index e7262c6..9e12357 100644 --- a/projects/privacy-guard/tests/engines/test_base.py +++ b/projects/privacy-guard/tests/engines/test_base.py @@ -118,6 +118,52 @@ def test_detection_confidence_and_metadata_are_strict_bounded_values() -> None: ) +@pytest.mark.parametrize( + "unsafe_value", + [ + "line\nbreak", + "ansi\x1b[31m", + "nul\x00byte", + "right-to-left\u202eoverride", + ], +) +def test_detection_rejects_non_printable_identifiers_and_metadata( + unsafe_value: str, +) -> None: + with pytest.raises(ValidationError): + EntityDetection( + entity=unsafe_value, + start=0, + end=1, + ) + with pytest.raises(ValidationError): + EntityDetection( + entity="token", + start=0, + end=1, + metadata={unsafe_value: "value"}, + ) + with pytest.raises(ValidationError): + EntityDetection( + entity="token", + start=0, + end=1, + metadata={"key": unsafe_value}, + ) + + +def test_detection_accepts_printable_unicode_identifiers_and_metadata() -> None: + detection = EntityDetection( + entity="客户资料", + start=0, + end=1, + metadata={"提供者": "自定义 🛡️"}, + ) + + assert detection.entity == "客户资料" + assert detection.metadata == {"提供者": "自定义 🛡️"} + + def test_processing_result_bounds_a_lazy_detection_stream() -> None: produced = 0 diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index f5efd47..edfc672 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -6,7 +6,9 @@ import logging from copy import deepcopy from threading import get_ident +from typing import Never +import grpc import pytest from google.protobuf import json_format from google.protobuf.message import Message @@ -16,6 +18,7 @@ from privacy_guard.constants import ( LIMIT_REASON, LIMIT_REASON_CODE, + MAX_DIAGNOSTIC_TEXT_BYTES, MAX_PROTO_CONFIG_BYTES, MAX_PROTO_CONTEXT_BYTES, MAX_PROTO_FINDING_BYTES, @@ -40,6 +43,7 @@ def _values( action: str = "replace", *, stage_count: int = 1, + stage_name: str | None = None, ) -> dict[str, object]: stage: dict[str, object] = { "config": { @@ -63,6 +67,8 @@ def _values( }, } } + if stage_name is not None: + stage["name"] = stage_name return { "entity_processing": {"stages": [deepcopy(stage) for _ in range(stage_count)]}, "on_detection": {"action": action}, @@ -83,6 +89,12 @@ def _request(body: bytes, *, action: str = "replace") -> pb2.HttpRequestEvaluati ) +class _SuccessfulEvaluationContext: + async def abort(self, code: grpc.StatusCode, details: str) -> Never: + del code, details + raise AssertionError("successful evaluation unexpectedly aborted") + + def test_copied_proto_remains_the_current_openshell_contract() -> None: evaluation = pb2.HttpRequestEvaluation() finding = pb2.Finding() @@ -144,6 +156,34 @@ def record_validation( assert validation_count == 1 +@pytest.mark.parametrize( + "unsafe_value", + [ + "line\nbreak", + "ansi\x1b[31m", + "nul\x00byte", + "right-to-left\u202eoverride", + ], +) +def test_validate_config_rejects_non_printable_stage_names( + unsafe_value: str, +) -> None: + registry = create_builtin_registry() + + with pytest.raises(PrivacyGuardError) as captured: + registry.validate_config(_values(stage_name=unsafe_value)) + + assert captured.value.code is ErrorCode.CONFIG_INVALID + + +def test_validate_config_accepts_printable_unicode_stage_names() -> None: + config = create_builtin_registry().validate_config( + _values(stage_name="身份检查 🛡️") + ) + + assert config.entity_processing.stages[0].name == "身份检查 🛡️" + + def test_evaluation_enforces_exact_encoded_transport_boundaries() -> None: request = _request(b"") request.context.request_id = "x" * 4_093 @@ -248,6 +288,70 @@ def test_service_limit_deny_logs_a_content_safe_resource_kind( assert sentinel not in caplog.text +@pytest.mark.parametrize( + "invalid_request_id", + [ + "line\nbreak", + "ansi\x1b[31m", + "nul\x00byte", + "right-to-left\u202eoverride", + "x" * (MAX_DIAGNOSTIC_TEXT_BYTES + 1), + ], +) +def test_evaluation_logs_placeholder_for_invalid_request_id( + caplog: pytest.LogCaptureFixture, + invalid_request_id: str, +) -> None: + async def evaluate() -> pb2.HttpRequestResult: + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + request = _request(b"no match", action="detect") + request.context.request_id = invalid_request_id + try: + return await middleware._evaluate_rpc( + request, + _SuccessfulEvaluationContext(), + ) + finally: + await middleware.close() + + with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): + result = asyncio.run(evaluate()) + + records = [ + record + for record in caplog.records + if record.name == "privacy_guard.service.servicer" + and record.getMessage().startswith("privacy_guard_evaluation ") + ] + assert result.decision == pb2.DECISION_ALLOW + assert len(records) == 1 + assert "request_id=invalid " in records[0].getMessage() + assert records[0].getMessage().isprintable() + assert len(caplog.text.splitlines()) == 1 + + +def test_evaluation_logs_printable_unicode_request_id( + caplog: pytest.LogCaptureFixture, +) -> None: + async def evaluate() -> None: + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + request = _request(b"no match", action="detect") + request.context.request_id = "请求-42 🛡️" + try: + await middleware._evaluate_rpc( + request, + _SuccessfulEvaluationContext(), + ) + finally: + await middleware.close() + + with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): + asyncio.run(evaluate()) + + assert "request_id=请求-42 🛡️ " in caplog.text + assert len(caplog.text.splitlines()) == 1 + + def test_middleware_applies_configured_timeout_to_cached_processors() -> None: middleware = PrivacyGuardMiddleware( create_builtin_registry(), From a118097eeca0b47109a2900f6c54e76902373fd6 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 03:07:32 +0000 Subject: [PATCH 64/82] fix(privacy-guard): enforce engine lifecycle hooks (PG-007) --- .../architecture/entity-processing-engines.md | 4 +- .../src/privacy_guard/engines/base.py | 3 + .../src/privacy_guard/engines/registry.py | 10 ++ .../tests/engines/test_registry.py | 101 ++++++++++++++++++ 4 files changed, 117 insertions(+), 1 deletion(-) diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md index 0203576..8c8cee0 100644 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -30,7 +30,9 @@ An engine: The public `run()` method validates input, strategy support, the timeout, and the complete output contract. Custom engines implement `_run()` and do not override `run()` or define `__init__`. `_initialize()` is optional, and -`@override` is not required. +`@override` is not required. The base constructor and public wrapper are final +lifecycle methods. Registration rejects direct or inherited overrides and +directs custom engines to `_initialize()` and `_run()` instead. ## Configuration diff --git a/projects/privacy-guard/src/privacy_guard/engines/base.py b/projects/privacy-guard/src/privacy_guard/engines/base.py index 980d899..c6d60bf 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/base.py +++ b/projects/privacy-guard/src/privacy_guard/engines/base.py @@ -13,6 +13,7 @@ Generic, Self, TypeAlias, + final, get_args, get_origin, ) @@ -156,6 +157,7 @@ class EntityProcessingEngine(ABC, Generic[_ConfigT, _ResourcesT]): supported_strategies: ClassVar[frozenset[EntityProcessingStrategy]] + @final def __init__( self, config: _ConfigT, @@ -227,6 +229,7 @@ def resources(self) -> _ResourcesT: """Return the validated, injected runtime resources.""" return self.__resources + @final def run( self, text: str, diff --git a/projects/privacy-guard/src/privacy_guard/engines/registry.py b/projects/privacy-guard/src/privacy_guard/engines/registry.py index 9afc834..bee9deb 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/registry.py +++ b/projects/privacy-guard/src/privacy_guard/engines/registry.py @@ -72,6 +72,16 @@ def register( engine_type, EntityProcessingEngine ): raise EngineRegistryError("registered engine type is invalid") + if engine_type.__init__ is not EntityProcessingEngine.__init__: + raise EngineRegistryError( + "engine lifecycle contract requires EntityProcessingEngine.__init__; " + "use _initialize() instead" + ) + if engine_type.run is not EntityProcessingEngine.run: + raise EngineRegistryError( + "engine lifecycle contract requires EntityProcessingEngine.run; " + "implement _run() instead" + ) try: config_type = engine_type.get_config_type() diff --git a/projects/privacy-guard/tests/engines/test_registry.py b/projects/privacy-guard/tests/engines/test_registry.py index 086efd5..fc60047 100644 --- a/projects/privacy-guard/tests/engines/test_registry.py +++ b/projects/privacy-guard/tests/engines/test_registry.py @@ -261,6 +261,107 @@ def test_registry_rejects_duplicate_discriminators_and_resource_mismatch() -> No EngineRegistry().register(DetectEngine, resources=object()) +def _run_without_the_engine_wrapper( + self: DetectEngine, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, +) -> TextProcessingResult: + del self, strategy, timeout + return TextProcessingResult(text=text, detections=()) + + +def _initialize_without_the_engine_constructor( + self: DetectEngine, + config: DetectConfig, + resources: None, +) -> None: + del self, config, resources + + +@pytest.mark.parametrize( + ("method_name", "method", "expected_error"), + [ + ( + "run", + _run_without_the_engine_wrapper, + "engine lifecycle contract requires EntityProcessingEngine.run; " + "implement _run() instead", + ), + ( + "__init__", + _initialize_without_the_engine_constructor, + "engine lifecycle contract requires EntityProcessingEngine.__init__; " + "use _initialize() instead", + ), + ], +) +def test_registry_rejects_direct_and_inherited_lifecycle_overrides( + method_name: str, + method: object, + expected_error: str, +) -> None: + direct_override = type( + "LifecycleOverrideEngine", + (DetectEngine,), + {method_name: method}, + ) + inherited_override = type( + "InheritedOverrideEngine", + (direct_override,), + {}, + ) + + for engine_type in (direct_override, inherited_override): + with pytest.raises(EngineRegistryError) as error: + EngineRegistry().register(engine_type) + + assert str(error.value) == expected_error + + +def test_base_lifecycle_methods_are_final_for_static_feedback() -> None: + assert getattr(EntityProcessingEngine.__init__, "__final__", False) is True + assert getattr(EntityProcessingEngine.run, "__final__", False) is True + + +def test_registry_accepts_base_lifecycle_inherited_through_custom_base() -> None: + intermediate_base = type( + "ValidIntermediateEngineBase", + (DetectEngine,), + {}, + ) + inherited_lifecycle_engine = type( + "InheritedLifecycleEngine", + (intermediate_base,), + {}, + ) + + registry = EngineRegistry() + registry.register(inherited_lifecycle_engine) + + assert inherited_lifecycle_engine.__init__ is EntityProcessingEngine.__init__ + assert inherited_lifecycle_engine.run is EntityProcessingEngine.run + + +@pytest.mark.parametrize( + ("engine_type", "resources"), + [ + (DetectEngine, None), + (AcmeEngine, AcmeResources(prefix="token")), + ], +) +def test_registry_accepts_engines_using_the_base_lifecycle( + engine_type: type[object], + resources: object, +) -> None: + registry = EngineRegistry() + + registry.register(engine_type, resources=resources) + + assert registry.finalize().is_finalized is True + + def test_describe_does_not_construct_an_engine() -> None: class CountingEngine(EntityProcessingEngine[DetectConfig]): supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) From bb64fa36103256cdb241fd9bf9d163a4914546ba Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 03:07:47 +0000 Subject: [PATCH 65/82] fix(privacy-guard): classify zero-width regex config (PG-006) --- .../architecture/configuration.md | 8 +++ .../src/privacy_guard/engines/regex.py | 6 +- .../privacy-guard/tests/engines/test_regex.py | 57 +++++++++++++++++-- .../tests/service/test_grpc_integration.py | 29 +++++++++- 4 files changed, 92 insertions(+), 8 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/configuration.md b/docs/documentation/privacy-guard/architecture/configuration.md index 576dc25..68bede7 100644 --- a/docs/documentation/privacy-guard/architecture/configuration.md +++ b/docs/documentation/privacy-guard/architecture/configuration.md @@ -94,6 +94,14 @@ Privacy Guard maintains the catalog schema and safety limits but does not ship an authoritative pattern set. Repository catalogs are examples to copy and adapt, not presets or runtime defaults. +`ValidateConfig` proves that each Regex pattern has valid syntax and rejects +patterns that immediately match empty input. It cannot prove that every +context-dependent branch consumes text. If a boundary, lookaround, or +alternation produces a zero-width match only when triggering request text is +observed, evaluation rejects the policy as `config_invalid` rather than treating +the result as an internal engine failure. Lookarounds and boundaries remain +valid when the complete match consumes text. + ## Current transport constraint The copied OpenShell protocol carries policy configuration in a diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index 7491ac9..97430a9 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -45,6 +45,8 @@ EngineConfigurationError, EngineContractError, EngineLimitExceededError, + ErrorCode, + PrivacyGuardError, ) from privacy_guard.string_validators import ScalarString, validate_scalar_string from privacy_guard.timeout import Timeout @@ -248,7 +250,9 @@ def _run( if match is None: break start, end = match.span() - if start == end or match.span(rule.marker) != (end, end): + if start == end: + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) + if match.span(rule.marker) != (end, end): raise EngineConfigurationError( "regex engine configuration is invalid" ) diff --git a/projects/privacy-guard/tests/engines/test_regex.py b/projects/privacy-guard/tests/engines/test_regex.py index c8f35e3..007ba1c 100644 --- a/projects/privacy-guard/tests/engines/test_regex.py +++ b/projects/privacy-guard/tests/engines/test_regex.py @@ -13,7 +13,12 @@ RegexEngine, RegexEngineConfig, ) -from privacy_guard.errors import TimeoutExpiredError +from privacy_guard.errors import ( + ErrorCode, + ErrorKind, + PrivacyGuardError, + TimeoutExpiredError, +) from privacy_guard.timeout import Timeout @@ -139,17 +144,59 @@ def test_invalid_patterns_are_rejected_content_safely(pattern: str) -> None: assert pattern not in str(exception_info.value) -def test_contextual_zero_width_failure_is_atomic_at_runtime() -> None: - config = _config([{"pattern": "(?=a)", "confidence": "high"}]) +@pytest.mark.parametrize( + ("pattern", "text"), + [ + ("x|(?=SECRET-zero-width-493)", "SECRET-zero-width-493"), + ("(?=secret)", "secret"), + ("(?<=prefix)", "prefix"), + (r"\b", "secret"), + ("x|(?:y|(?=secret))", "secret"), + ], +) +def test_contextual_zero_width_match_is_invalid_configuration_at_runtime( + pattern: str, + text: str, +) -> None: + config = _config([{"pattern": pattern, "confidence": "high"}]) engine = RegexEngine(config, None) - with pytest.raises(EngineConfigurationError): + with pytest.raises(PrivacyGuardError) as exception_info: engine.run( - "a", + text, strategy=EntityProcessingStrategy.DETECT, timeout=Timeout.from_seconds(1), ) + assert exception_info.value.code is ErrorCode.CONFIG_INVALID + assert exception_info.value.kind is ErrorKind.INVALID_INPUT + assert pattern not in str(exception_info.value) + + +@pytest.mark.parametrize( + ("pattern", "text", "expected_span"), + [ + ("(?<=prefix)secret(?=suffix)", "prefixsecretsuffix", (6, 12)), + (r"\bsecret\b", "a secret value", (2, 8)), + ( + r"(? None: + config = _config([{"pattern": pattern, "confidence": "high"}]) + + _, detections = _run(config, text) + + assert [(item[1], item[2]) for item in detections] == [expected_span] + def test_duplicate_supplied_names_are_rejected_but_unnamed_patterns_are_not() -> None: with pytest.raises(ValidationError): diff --git a/projects/privacy-guard/tests/service/test_grpc_integration.py b/projects/privacy-guard/tests/service/test_grpc_integration.py index 0180dfd..2f8d684 100644 --- a/projects/privacy-guard/tests/service/test_grpc_integration.py +++ b/projects/privacy-guard/tests/service/test_grpc_integration.py @@ -27,7 +27,11 @@ from privacy_guard.timeout import Timeout -def _config(*, action: str = "replace") -> pb2.ValidateConfigRequest: +def _config( + *, + action: str = "replace", + pattern: str = r"[a-z]+@[a-z]+\.[a-z]+", +) -> pb2.ValidateConfigRequest: request = pb2.ValidateConfigRequest() json_format.ParseDict( { @@ -43,7 +47,7 @@ def _config(*, action: str = "replace") -> pb2.ValidateConfigRequest: "name": "email", "patterns": [ { - "pattern": (r"[a-z]+@[a-z]+\.[a-z]+"), + "pattern": pattern, "confidence": "high", } ], @@ -195,6 +199,27 @@ async def test_generated_stub_enforces_ten_stage_limit() -> None: assert "config_invalid" in (oversized_result.value.details() or "") +@pytest.mark.asyncio +async def test_generated_stub_maps_contextual_zero_width_to_invalid_config() -> None: + report_pattern = "x|(?=SECRET-zero-width-493)" + config = _config(action="detect", pattern=report_pattern) + evaluation = _evaluation(b"SECRET-zero-width-493", action="detect") + evaluation.config.CopyFrom(config.config) + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + + async with _running_stub(middleware) as stub: + validation = await stub.ValidateConfig(config) + with pytest.raises(grpc.aio.AioRpcError) as evaluation_error: + await stub.EvaluateHttpRequest(evaluation) + + details = evaluation_error.value.details() or "" + assert validation.valid is True + assert evaluation_error.value.code() is grpc.StatusCode.INVALID_ARGUMENT + assert "config_invalid" in details + assert "engine_execution_failed" not in details + assert report_pattern not in details + + class _NumericNestedConfig(StrictDomainModel): count: int From 30d8572998098e992907fec16b0ed8d6b88f478c Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 03:13:18 +0000 Subject: [PATCH 66/82] fix(privacy-guard): single-flight processor builds (PG-008) --- .../src/privacy_guard/service/servicer.py | 20 +- .../tests/service/test_servicer.py | 179 +++++++++++++++++- 2 files changed, 194 insertions(+), 5 deletions(-) diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index 2690083..ad5b246 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -242,6 +242,7 @@ def __init__( self._timeout_seconds = timeout_seconds self._log_request_content = log_request_content self._processors: OrderedDict[str, RequestProcessor] = OrderedDict() + self._in_flight: dict[str, Future[RequestProcessor]] = {} self._lock = RLock() def resolve(self, values: object) -> RequestProcessor: @@ -253,12 +254,29 @@ def resolve(self, values: object) -> RequestProcessor: if cached is not None: self._processors.move_to_end(fingerprint) return cached - processor = self._build_processor(config) + owner_future = self._in_flight.get(fingerprint) + owns_build = owner_future is None + if owner_future is None: + owner_future = Future() + self._in_flight[fingerprint] = owner_future + if not owns_build: + return owner_future.result() + try: + processor = self._build_processor(config) + except Exception as error: + with self._lock: + if self._in_flight.get(fingerprint) is owner_future: + del self._in_flight[fingerprint] + owner_future.set_exception(error) + raise with self._lock: self._processors[fingerprint] = processor self._processors.move_to_end(fingerprint) while len(self._processors) > _MAX_CACHED_PROCESSORS: self._processors.popitem(last=False) + if self._in_flight.get(fingerprint) is owner_future: + del self._in_flight[fingerprint] + owner_future.set_result(processor) return processor def _build_processor( diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index edfc672..afaaffd 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -4,8 +4,9 @@ import asyncio import logging +from concurrent.futures import Future, ThreadPoolExecutor from copy import deepcopy -from threading import get_ident +from threading import Barrier, Event, Lock, get_ident from typing import Never import grpc @@ -95,6 +96,26 @@ async def abort(self, code: grpc.StatusCode, details: str) -> Never: raise AssertionError("successful evaluation unexpectedly aborted") +def _waiter_observing_future_type( + *, + expected_waiters: int, + waiters_ready: Event, +) -> type[Future[RequestProcessor]]: + waiter_count = 0 + waiter_count_lock = Lock() + + class _WaiterObservingFuture(Future[RequestProcessor]): + def result(self, timeout: float | None = None) -> RequestProcessor: + nonlocal waiter_count + with waiter_count_lock: + waiter_count += 1 + if waiter_count == expected_waiters: + waiters_ready.set() + return super().result(timeout) + + return _WaiterObservingFuture + + def test_copied_proto_remains_the_current_openshell_contract() -> None: evaluation = pb2.HttpRequestEvaluation() finding = pb2.Finding() @@ -177,9 +198,7 @@ def test_validate_config_rejects_non_printable_stage_names( def test_validate_config_accepts_printable_unicode_stage_names() -> None: - config = create_builtin_registry().validate_config( - _values(stage_name="身份检查 🛡️") - ) + config = create_builtin_registry().validate_config(_values(stage_name="身份检查 🛡️")) assert config.entity_processing.stages[0].name == "身份检查 🛡️" @@ -450,6 +469,158 @@ async def evaluate_twice() -> None: assert validation_count == 2 +def test_same_fingerprint_misses_build_one_shared_processor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + worker_count = 4 + workers_ready = Barrier(worker_count) + waiters_ready = Event() + build_count = 0 + build_count_lock = Lock() + original_build = servicer_module._RequestProcessorCache._build_processor + + def synchronized_build( + cache: servicer_module._RequestProcessorCache, + config: PrivacyGuardConfig[EngineConfig], + ) -> RequestProcessor: + nonlocal build_count + with build_count_lock: + build_count += 1 + assert waiters_ready.wait(timeout=5) + return original_build(cache, config) + + monkeypatch.setattr( + servicer_module, + "Future", + _waiter_observing_future_type( + expected_waiters=worker_count - 1, + waiters_ready=waiters_ready, + ), + ) + monkeypatch.setattr( + servicer_module._RequestProcessorCache, + "_build_processor", + synchronized_build, + ) + cache = servicer_module._RequestProcessorCache( + create_builtin_registry(), + timeout_seconds=1, + log_request_content=False, + ) + + def resolve() -> RequestProcessor: + workers_ready.wait(timeout=5) + return cache.resolve(_values()) + + with ThreadPoolExecutor(max_workers=worker_count) as executor: + processors = tuple(executor.map(lambda _: resolve(), range(worker_count))) + + assert build_count == 1 + assert all(processor is processors[0] for processor in processors) + assert cache._in_flight == {} + + +def test_different_fingerprints_build_concurrently( + monkeypatch: pytest.MonkeyPatch, +) -> None: + builds_ready = Barrier(2) + original_build = servicer_module._RequestProcessorCache._build_processor + + def synchronized_build( + cache: servicer_module._RequestProcessorCache, + config: PrivacyGuardConfig[EngineConfig], + ) -> RequestProcessor: + builds_ready.wait(timeout=5) + return original_build(cache, config) + + monkeypatch.setattr( + servicer_module._RequestProcessorCache, + "_build_processor", + synchronized_build, + ) + cache = servicer_module._RequestProcessorCache( + create_builtin_registry(), + timeout_seconds=1, + log_request_content=False, + ) + with ThreadPoolExecutor(max_workers=2) as executor: + processors = tuple( + executor.map( + cache.resolve, + (_values(action="detect"), _values(action="block")), + ) + ) + + assert processors[0] is not processors[1] + assert len(cache._processors) == 2 + assert cache._in_flight == {} + + +def test_failed_processor_build_wakes_waiters_and_allows_retry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + worker_count = 4 + workers_ready = Barrier(worker_count) + waiters_ready = Event() + build_count = 0 + build_count_lock = Lock() + failure = PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) + original_build = servicer_module._RequestProcessorCache._build_processor + + def fail_first_build( + cache: servicer_module._RequestProcessorCache, + config: PrivacyGuardConfig[EngineConfig], + ) -> RequestProcessor: + nonlocal build_count + with build_count_lock: + build_count += 1 + current_build = build_count + if current_build == 1: + assert waiters_ready.wait(timeout=5) + raise failure + return original_build(cache, config) + + monkeypatch.setattr( + servicer_module, + "Future", + _waiter_observing_future_type( + expected_waiters=worker_count - 1, + waiters_ready=waiters_ready, + ), + ) + monkeypatch.setattr( + servicer_module._RequestProcessorCache, + "_build_processor", + fail_first_build, + ) + cache = servicer_module._RequestProcessorCache( + create_builtin_registry(), + timeout_seconds=1, + log_request_content=False, + ) + + def capture_failure() -> PrivacyGuardError: + workers_ready.wait(timeout=5) + with pytest.raises(PrivacyGuardError) as captured: + cache.resolve(_values()) + return captured.value + + with ThreadPoolExecutor(max_workers=worker_count) as executor: + failures = tuple(executor.map(lambda _: capture_failure(), range(worker_count))) + + assert build_count == 1 + assert all(error is failure for error in failures) + assert cache._processors == {} + assert cache._in_flight == {} + + processor = cache.resolve(_values()) + + assert isinstance(processor, RequestProcessor) + assert build_count == 2 + assert len(cache._processors) == 1 + assert cache._in_flight == {} + + def test_oversized_stage_list_fails_before_fingerprinting_or_construction( monkeypatch: pytest.MonkeyPatch, ) -> None: From f2bed7834babecb944b2e518091323cd5a253744 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 03:14:32 +0000 Subject: [PATCH 67/82] fix(privacy-guard): verify release artifacts (PG-010 PG-011 PG-013) --- projects/privacy-guard/LICENSE | 203 +++++++++++++++++ projects/privacy-guard/README.md | 12 +- projects/privacy-guard/pyproject.toml | 1 + projects/privacy-guard/scripts/check.sh | 23 ++ .../privacy-guard/scripts/check_artifacts.py | 204 ++++++++++++++++++ .../scripts/external_engine_typing.py | 66 ++++++ .../privacy-guard/src/privacy_guard/py.typed | 1 + 7 files changed, 505 insertions(+), 5 deletions(-) create mode 100644 projects/privacy-guard/LICENSE create mode 100644 projects/privacy-guard/scripts/check_artifacts.py create mode 100644 projects/privacy-guard/scripts/external_engine_typing.py create mode 100644 projects/privacy-guard/src/privacy_guard/py.typed diff --git a/projects/privacy-guard/LICENSE b/projects/privacy-guard/LICENSE new file mode 100644 index 0000000..10a6d3d --- /dev/null +++ b/projects/privacy-guard/LICENSE @@ -0,0 +1,203 @@ +Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 NVIDIA CORPORATION & AFFILIATES. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 977fd88..728f90b 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -44,8 +44,8 @@ Privacy Guard's working directory and normalized to the same `RegexPatternCatalog` model as inline input. Absolute paths, traversal, symlinks, unsafe YAML tags, aliases, duplicate keys, invalid UTF-8, and oversized files are rejected. The -[RegexEngine end-to-end example](examples/regex-engine/README.md) passes its -catalog as `patterns.yaml`. +[RegexEngine end-to-end example](https://github.com/NVIDIA/OpenShell-Research/blob/main/projects/privacy-guard/examples/regex-engine/README.md) +passes its catalog as `patterns.yaml`. ## Architecture @@ -141,7 +141,8 @@ uv run privacy-guard --registry-factory my_engines:create_registry configuration uv run privacy-guard --registry-factory my_engines:create_registry serve ``` -The [custom engine end-to-end example](examples/custom-engine/README.md) +The +[custom engine end-to-end example](https://github.com/NVIDIA/OpenShell-Research/blob/main/projects/privacy-guard/examples/custom-engine/README.md) contains one Python file with a typed detection config, engine implementation, and registry factory, plus its OpenShell policy and walkthrough. It also shows the explicit `PYTHONPATH` setup needed when the factory is a standalone local @@ -252,8 +253,9 @@ The `privacy_guard.cli` module owns the executable application and adapts its ## Updating the OpenShell protocol Privacy Guard uses -[`openshell-middleware-kit`](../openshell-middleware-kit/README.md) to keep its -copied protocol and generated Python bindings aligned with an OpenShell release. +[`openshell-middleware-kit`](https://github.com/NVIDIA/OpenShell-Research/blob/main/projects/openshell-middleware-kit/README.md) +to keep its copied protocol and generated Python bindings aligned with an +OpenShell release. Install the repository's local `omkit`, then update: ```bash diff --git a/projects/privacy-guard/pyproject.toml b/projects/privacy-guard/pyproject.toml index 36050c8..eb51964 100644 --- a/projects/privacy-guard/pyproject.toml +++ b/projects/privacy-guard/pyproject.toml @@ -5,6 +5,7 @@ description = "Inspect and enforce policy on provider-bound OpenShell requests." readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" +license-files = ["LICENSE"] authors = [ { name = "NVIDIA CORPORATION & AFFILIATES" }, ] diff --git a/projects/privacy-guard/scripts/check.sh b/projects/privacy-guard/scripts/check.sh index 93bd999..dd39849 100755 --- a/projects/privacy-guard/scripts/check.sh +++ b/projects/privacy-guard/scripts/check.sh @@ -4,12 +4,14 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." uv_run=(uv run --frozen) +artifact_python=() if [[ $# -gt 0 ]]; then if [[ $1 != "--python" || $# -ne 2 ]]; then echo "usage: scripts/check.sh [--python VERSION]" >&2 exit 2 fi uv_run+=(--python "$2") + artifact_python+=(--python "$2") fi "${uv_run[@]}" pytest -q @@ -18,3 +20,24 @@ fi "${uv_run[@]}" ty check "${uv_run[@]}" python -c "import privacy_guard" uv build +"${uv_run[@]}" python scripts/check_artifacts.py + +artifact_check_dir=$(mktemp -d) +trap 'rm -rf "$artifact_check_dir"' EXIT +uv venv --clear "${artifact_python[@]}" "$artifact_check_dir/venv" + +wheels=(dist/*.whl) +if [[ ${#wheels[@]} -ne 1 ]]; then + echo "expected exactly one built wheel, found ${#wheels[@]}" >&2 + exit 1 +fi + +uv pip install \ + --offline \ + --python "$artifact_check_dir/venv/bin/python" \ + "${wheels[0]}" +cp scripts/external_engine_typing.py "$artifact_check_dir/external_engine_typing.py" +"${uv_run[@]}" ty check \ + --project "$artifact_check_dir" \ + --python "$artifact_check_dir/venv" \ + "$artifact_check_dir/external_engine_typing.py" diff --git a/projects/privacy-guard/scripts/check_artifacts.py b/projects/privacy-guard/scripts/check_artifacts.py new file mode 100644 index 0000000..75d348b --- /dev/null +++ b/projects/privacy-guard/scripts/check_artifacts.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Verify release semantics that generic packaging checks do not cover.""" + +from __future__ import annotations + +import re +import tarfile +import tomllib +import zipfile +from collections.abc import Iterable +from email import policy +from email.message import Message +from email.parser import BytesParser +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +REPOSITORY_ROOT = PROJECT_ROOT.parent.parent +DIST_DIRECTORY = PROJECT_ROOT / "dist" + +CANONICAL_README_TARGETS = { + "https://github.com/NVIDIA/OpenShell-Research/blob/main/" + "projects/privacy-guard/examples/regex-engine/README.md": ( + REPOSITORY_ROOT / "projects/privacy-guard/examples/regex-engine/README.md" + ), + "https://github.com/NVIDIA/OpenShell-Research/blob/main/" + "projects/privacy-guard/examples/custom-engine/README.md": ( + REPOSITORY_ROOT / "projects/privacy-guard/examples/custom-engine/README.md" + ), + "https://github.com/NVIDIA/OpenShell-Research/blob/main/" + "projects/openshell-middleware-kit/README.md": ( + REPOSITORY_ROOT / "projects/openshell-middleware-kit/README.md" + ), +} +REPOSITORY_RELATIVE_README_TARGETS = ( + "(examples/regex-engine/README.md)", + "(examples/custom-engine/README.md)", + "(../openshell-middleware-kit/README.md)", +) + + +def main() -> None: + """Inspect the wheel and source distribution produced for this project.""" + project = _load_project_metadata() + distribution_name = re.sub(r"[-_.]+", "_", project["name"]) + version = project["version"] + archive_stem = f"{distribution_name}-{version}" + + wheel = _find_one(DIST_DIRECTORY.glob(f"{archive_stem}-*.whl"), "wheel") + source_distribution = _find_one( + DIST_DIRECTORY.glob(f"{archive_stem}.tar.gz"), + "source distribution", + ) + + _check_readme_targets() + _check_wheel(wheel, archive_stem) + _check_source_distribution(source_distribution, archive_stem) + + +def _load_project_metadata() -> dict[str, str]: + with (PROJECT_ROOT / "pyproject.toml").open("rb") as file: + project = tomllib.load(file)["project"] + return {"name": project["name"], "version": project["version"]} + + +def _find_one(paths: Iterable[Path], description: str) -> Path: + matches = sorted(paths) + if len(matches) != 1: + raise SystemExit(f"expected exactly one {description}, found {len(matches)}") + return matches[0] + + +def _check_readme_targets() -> None: + missing = [ + str(target.relative_to(REPOSITORY_ROOT)) + for target in CANONICAL_README_TARGETS.values() + if not target.is_file() + ] + if missing: + raise SystemExit( + "canonical README link targets are missing: " + ", ".join(missing) + ) + + +def _check_wheel(wheel: Path, archive_stem: str) -> None: + metadata_member = f"{archive_stem}.dist-info/METADATA" + license_member = f"{archive_stem}.dist-info/licenses/LICENSE" + typed_marker = "privacy_guard/py.typed" + + with zipfile.ZipFile(wheel) as archive: + members = set(archive.namelist()) + _require_members( + wheel.name, + members, + (metadata_member, license_member, typed_marker), + ) + _require_exact_license(wheel.name, archive.read(license_member)) + metadata = _parse_metadata(archive.read(metadata_member)) + _require_license_metadata(wheel.name, metadata) + _require_readme_links(wheel.name, _metadata_description(metadata)) + + print(f"{wheel.name}: {license_member}, {typed_marker}, License-File: LICENSE") + + +def _check_source_distribution( + source_distribution: Path, + archive_stem: str, +) -> None: + metadata_member = f"{archive_stem}/PKG-INFO" + license_member = f"{archive_stem}/LICENSE" + readme_member = f"{archive_stem}/README.md" + typed_marker = f"{archive_stem}/src/privacy_guard/py.typed" + + with tarfile.open(source_distribution, "r:gz") as archive: + members = set(archive.getnames()) + _require_members( + source_distribution.name, + members, + (metadata_member, license_member, readme_member, typed_marker), + ) + _require_exact_license( + source_distribution.name, + _read_tar_member(archive, license_member), + ) + metadata = _parse_metadata(_read_tar_member(archive, metadata_member)) + _require_license_metadata(source_distribution.name, metadata) + _require_readme_links( + f"{source_distribution.name}:{readme_member}", + _read_tar_member(archive, readme_member).decode("utf-8"), + ) + _require_readme_links( + f"{source_distribution.name}:{metadata_member}", + _metadata_description(metadata), + ) + + print( + f"{source_distribution.name}: {license_member}, {typed_marker}, " + "License-File: LICENSE" + ) + + +def _require_members( + artifact: str, + members: set[str], + required: tuple[str, ...], +) -> None: + missing = [member for member in required if member not in members] + if missing: + raise SystemExit(f"{artifact} is missing: {', '.join(missing)}") + + +def _require_exact_license(artifact: str, packaged_license: bytes) -> None: + expected_license = (PROJECT_ROOT / "LICENSE").read_bytes() + if packaged_license != expected_license: + raise SystemExit(f"{artifact} does not contain the exact project LICENSE") + + +def _parse_metadata(contents: bytes) -> Message: + return BytesParser(policy=policy.default).parsebytes(contents) + + +def _require_license_metadata(artifact: str, metadata: Message) -> None: + if metadata["License-Expression"] != "Apache-2.0": + raise SystemExit( + f"{artifact} metadata does not declare License-Expression: Apache-2.0" + ) + license_files = metadata.get_all("License-File") or [] + if "LICENSE" not in license_files: + raise SystemExit(f"{artifact} metadata is missing License-File: LICENSE") + + +def _metadata_description(metadata: Message) -> str: + description = metadata.get_payload() + if not isinstance(description, str): + raise SystemExit("package metadata description is not text") + return description + + +def _require_readme_links(artifact: str, readme: str) -> None: + missing = [url for url in CANONICAL_README_TARGETS if url not in readme] + if missing: + raise SystemExit( + f"{artifact} README is missing canonical links: {', '.join(missing)}" + ) + retained = [ + relative + for relative in REPOSITORY_RELATIVE_README_TARGETS + if relative in readme + ] + if retained: + raise SystemExit( + f"{artifact} README retains repository-relative links: " + + ", ".join(retained) + ) + + +def _read_tar_member(archive: tarfile.TarFile, member: str) -> bytes: + extracted = archive.extractfile(member) + if extracted is None: + raise SystemExit(f"could not read source distribution member {member}") + return extracted.read() + + +if __name__ == "__main__": + main() diff --git a/projects/privacy-guard/scripts/external_engine_typing.py b/projects/privacy-guard/scripts/external_engine_typing.py new file mode 100644 index 0000000..3089295 --- /dev/null +++ b/projects/privacy-guard/scripts/external_engine_typing.py @@ -0,0 +1,66 @@ +"""External consumer fixture checked only against an installed wheel.""" + +from __future__ import annotations + +from typing import Literal, assert_type + +from privacy_guard.engines import ( + EngineConfig, + EngineResources, + EntityProcessingEngine, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.timeout import Timeout + + +class ExternalConfig(EngineConfig): + engine: Literal["external"] = "external" + keyword: str + + +class ExternalClient: + def find(self, text: str, keyword: str) -> bool: + return keyword in text + + +class ExternalResources(EngineResources): + __slots__ = ("client",) + + def __init__(self, client: ExternalClient) -> None: + self.client = client + + +class ExternalEngine(EntityProcessingEngine[ExternalConfig, ExternalResources]): + supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + assert_type(self.config, ExternalConfig) + assert_type(self.config.keyword, str) + assert_type(self.resources, ExternalResources) + assert_type(self.resources.client, ExternalClient) + self.resources.client.find(text, self.config.keyword) + result = TextProcessingResult(text=text, detections=()) + assert_type(result, TextProcessingResult) + return result + + +engine = ExternalEngine( + ExternalConfig(keyword="secret"), + ExternalResources(ExternalClient()), +) +assert_type(engine.config, ExternalConfig) +assert_type(engine.resources, ExternalResources) +processed = engine.run( + "text", + strategy=EntityProcessingStrategy.DETECT, + timeout=Timeout.from_seconds(1), +) +assert_type(processed, TextProcessingResult) +assert_type(processed.text, str) diff --git a/projects/privacy-guard/src/privacy_guard/py.typed b/projects/privacy-guard/src/privacy_guard/py.typed new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/py.typed @@ -0,0 +1 @@ + From 12bfe1a843808007b7026876f0ec25c37207693f Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 03:22:11 +0000 Subject: [PATCH 68/82] fix(privacy-guard): bound prepared-state caches (PG-009) --- .../architecture/safety-and-limits.md | 15 ++ .../architecture/service-boundary.md | 34 +++- .../privacy-guard/src/privacy_guard/config.py | 10 +- .../src/privacy_guard/constants.py | 6 + .../src/privacy_guard/engines/regex.py | 125 ++++++++++++- .../src/privacy_guard/service/servicer.py | 55 +++++- .../privacy-guard/tests/engines/test_regex.py | 147 ++++++++++++++- .../tests/service/test_servicer.py | 173 +++++++++++++++++- projects/privacy-guard/tests/test_config.py | 118 +++++++++++- 9 files changed, 652 insertions(+), 31 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index cd7423d..3d80336 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -245,6 +245,21 @@ Cross-request entity memory is not implemented. Privacy Guard retains validated configuration and immutable engine state in its processor cache, but never retains request text, detections, or replacement mappings there. +Prepared state is bounded by weight and least-recently-used entry count: + +| Cache | Weight budget | Entry cap | Entry weight | +| --- | ---: | ---: | --- | +| Parsed Regex file catalogs | 8 MiB | 64 | source file bytes | +| Compiled Regex catalogs | 32 MiB | 128 | canonical catalog bytes plus 4 KiB per rule | +| Request processors | 1 MiB | 128 | canonical expanded configuration bytes | + +The compiled-rule allowance accounts for retained backend state that is much +larger than short pattern text. Processor weighting remains engine-neutral, so +custom engines do not need a cache-size hook. One entry that exceeds its cache +budget remains valid and usable for the current evaluation but is not retained. +Debug skip and eviction records contain only cache names and weights, plus +eviction counts. + ## Changing a limit A limit change may affect policy schema, processor behavior, engine contracts, diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index 23c6f9a..6636e22 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -53,17 +53,39 @@ the async gRPC event loop. During evaluation, the service validates and normalizes the same config, computes its canonical SHA-256 fingerprint, and resolves a configured processor -from a bounded 128-entry LRU cache. A cache miss constructs engines directly -from the exact stage configs and operator-injected resources, then constructs -the `RequestProcessor`. Validation, cache resolution, engine construction, -strict UTF-8 decoding, and processing all run in the bounded worker pool. -Validation still occurs for every evaluation, while equivalent normalized regex -catalogs reuse their bounded compiled-rule entry instead of recompiling. +from an LRU cache bounded by both 128 entries and 1 MiB of canonical expanded +configuration. A cache miss constructs engines directly from the exact stage +configs and operator-injected resources, then constructs the +`RequestProcessor`. Validation, cache resolution, engine construction, strict +UTF-8 decoding, and processing all run in the bounded worker pool. Validation +still occurs for every evaluation, while equivalent normalized regex catalogs +reuse their bounded compiled-rule entry instead of recompiling. + +Regex keeps two additional owner-local LRU caches. Parsed file catalogs retain +at most 64 entries and 8 MiB of source files. Compiled catalogs retain at most +128 entries and 32 MiB of weighted state, where each entry weighs its canonical +catalog bytes plus 4 KiB per compiled rule. The rule allowance conservatively +represents backend and Python state that the catalog encoding alone does not +capture. + +An otherwise valid entry larger than one cache's byte budget is built and +returned without being retained. Eviction or a skipped entry never makes a +policy invalid. Content-safe debug events report the cache name and weight, plus +the number of entries for an eviction, without logging configuration, paths, +patterns, or fingerprints. The cache is protected for concurrent access and is not correctness-relevant. Eviction or process restart simply causes reconstruction from a later evaluation's expanded config. +The entry cap remains useful for small policies. The 1 MiB processor budget +holds roughly sixteen near-ceiling inline configurations and up to 128 smaller +configurations. A representative 250-rule configuration weighs about 33 KiB in +that cache and about 1 MiB in the compiled cache, so the default budgets retain +roughly thirty such configurations while bounding their compiled state. +Operators with consistently larger catalogs should expect more frequent +preparation rather than unbounded retention. + Each cached processor receives the server's operational processing timeout. The default is 1 second shared across every configured stage. Operators may set up to 30 seconds with `privacy-guard serve --timeout-seconds`; programmatic diff --git a/projects/privacy-guard/src/privacy_guard/config.py b/projects/privacy-guard/src/privacy_guard/config.py index 70a7df2..40e5f82 100644 --- a/projects/privacy-guard/src/privacy_guard/config.py +++ b/projects/privacy-guard/src/privacy_guard/config.py @@ -117,6 +117,14 @@ def configuration_fingerprint( config: PrivacyGuardConfig[EngineConfig], ) -> str: """Return the canonical SHA-256 fingerprint of an expanded policy config.""" + fingerprint, _ = _configuration_fingerprint_and_size(config) + return fingerprint + + +def _configuration_fingerprint_and_size( + config: PrivacyGuardConfig[EngineConfig], +) -> tuple[str, int]: + """Return the canonical fingerprint and encoded size of an expanded config.""" serialized = json.dumps( config.model_dump(mode="json"), allow_nan=False, @@ -124,7 +132,7 @@ def configuration_fingerprint( separators=(",", ":"), sort_keys=True, ).encode("utf-8") - return sha256(serialized).hexdigest() + return sha256(serialized).hexdigest(), len(serialized) __all__ = [ diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index 270bc9b..0e7173a 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -49,6 +49,12 @@ MAX_REGEX_CATALOG_FILE_BYTES = 16 * 1024 * 1024 MAX_REGEX_CATALOG_PATH_BYTES = 1024 +# Prepared-state cache budgets. Entry-count caps remain secondary guards. +MAX_REGEX_PARSED_CATALOG_CACHE_BYTES = 8 * 1024 * 1024 +MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES = 32 * 1024 * 1024 +REGEX_COMPILED_RULE_WEIGHT_BYTES = 4 * 1024 +MAX_PROCESSOR_CACHE_CONFIG_BYTES = 1024 * 1024 + # Service concurrency and transport limits. MAX_CONCURRENT_PROCESSING = 4 MAX_CONCURRENT_RPCS = 16 diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index 97430a9..80cbb80 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -2,11 +2,11 @@ from __future__ import annotations +import json import os from collections import OrderedDict from collections.abc import Mapping from dataclasses import dataclass -from functools import lru_cache from pathlib import Path from stat import S_ISREG from string import Formatter @@ -28,10 +28,13 @@ MAX_DIAGNOSTIC_TEXT_BYTES, MAX_REGEX_CATALOG_FILE_BYTES, MAX_REGEX_CATALOG_PATH_BYTES, + MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES, MAX_REGEX_ENTITIES_PER_CATALOG, MAX_REGEX_NAME_BYTES, + MAX_REGEX_PARSED_CATALOG_CACHE_BYTES, MAX_REGEX_PATTERN_BYTES, MAX_REGEX_PATTERNS_PER_CATALOG, + REGEX_COMPILED_RULE_WEIGHT_BYTES, ) from privacy_guard.engines.base import ( ConfidenceLevel, @@ -48,6 +51,7 @@ ErrorCode, PrivacyGuardError, ) +from privacy_guard.logging import get_logger from privacy_guard.string_validators import ScalarString, validate_scalar_string from privacy_guard.timeout import Timeout @@ -461,7 +465,7 @@ def _read_pattern_catalog_file( cached = _PATTERN_CATALOG_CACHE.get(cache_key) if cached is not None: _PATTERN_CATALOG_CACHE.move_to_end(cache_key) - return cached + return cached[0] contents = _read_bounded_file(descriptor) if len(contents) != size or len(contents) > MAX_REGEX_CATALOG_FILE_BYTES: @@ -480,11 +484,40 @@ def _read_pattern_catalog_file( Loader=_StrictCatalogLoader, ) catalog = RegexPatternCatalog.model_validate(values) + if size > MAX_REGEX_PARSED_CATALOG_CACHE_BYTES: + _LOGGER.debug( + "privacy_guard_cache_skip cache=regex_parsed_catalog " + "weight_bytes=%d budget_bytes=%d", + size, + MAX_REGEX_PARSED_CATALOG_CACHE_BYTES, + ) + return catalog + evicted_entries = 0 + evicted_weight_bytes = 0 with _PATTERN_CATALOG_CACHE_LOCK: - _PATTERN_CATALOG_CACHE[cache_key] = catalog - _PATTERN_CATALOG_CACHE.move_to_end(cache_key) - while len(_PATTERN_CATALOG_CACHE) > _MAX_CACHED_PATTERN_CATALOGS: - _PATTERN_CATALOG_CACHE.popitem(last=False) + cached = _PATTERN_CATALOG_CACHE.get(cache_key) + if cached is not None: + _PATTERN_CATALOG_CACHE.move_to_end(cache_key) + return cached[0] + _PATTERN_CATALOG_CACHE[cache_key] = (catalog, size) + global _PATTERN_CATALOG_CACHE_WEIGHT_BYTES + _PATTERN_CATALOG_CACHE_WEIGHT_BYTES += size + while ( + len(_PATTERN_CATALOG_CACHE) > _MAX_CACHED_PATTERN_CATALOGS + or _PATTERN_CATALOG_CACHE_WEIGHT_BYTES + > MAX_REGEX_PARSED_CATALOG_CACHE_BYTES + ): + _, (_, evicted_weight) = _PATTERN_CATALOG_CACHE.popitem(last=False) + _PATTERN_CATALOG_CACHE_WEIGHT_BYTES -= evicted_weight + evicted_entries += 1 + evicted_weight_bytes += evicted_weight + if evicted_entries: + _LOGGER.debug( + "privacy_guard_cache_eviction cache=regex_parsed_catalog " + "entries=%d weight_bytes=%d", + evicted_entries, + evicted_weight_bytes, + ) return catalog @@ -511,11 +544,16 @@ def _validate_name(value: str) -> str: return value -@lru_cache(maxsize=128) def _compile_pattern_catalog( catalog: RegexPatternCatalog, ) -> tuple[_CompiledRule, ...]: - return tuple( + with _COMPILED_PATTERN_CACHE_LOCK: + cached = _COMPILED_PATTERN_CACHE.get(catalog) + if cached is not None: + _COMPILED_PATTERN_CACHE.move_to_end(catalog) + return cached[0] + + rules = tuple( _compile_rule( entity, pattern, @@ -526,6 +564,66 @@ def _compile_pattern_catalog( _iter_catalog_patterns(catalog) ) ) + catalog_bytes = len( + json.dumps( + catalog.model_dump(mode="json"), + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ) + weight_bytes = catalog_bytes + len(rules) * REGEX_COMPILED_RULE_WEIGHT_BYTES + if weight_bytes > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES: + _LOGGER.debug( + "privacy_guard_cache_skip cache=regex_compiled " + "weight_bytes=%d budget_bytes=%d", + weight_bytes, + MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES, + ) + return rules + + evicted_entries = 0 + evicted_weight_bytes = 0 + with _COMPILED_PATTERN_CACHE_LOCK: + cached = _COMPILED_PATTERN_CACHE.get(catalog) + if cached is not None: + _COMPILED_PATTERN_CACHE.move_to_end(catalog) + return cached[0] + _COMPILED_PATTERN_CACHE[catalog] = (rules, weight_bytes) + global _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES += weight_bytes + while ( + len(_COMPILED_PATTERN_CACHE) > _MAX_CACHED_COMPILED_CATALOGS + or _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES + ): + _, (_, evicted_weight) = _COMPILED_PATTERN_CACHE.popitem(last=False) + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES -= evicted_weight + evicted_entries += 1 + evicted_weight_bytes += evicted_weight + if evicted_entries: + _LOGGER.debug( + "privacy_guard_cache_eviction cache=regex_compiled " + "entries=%d weight_bytes=%d", + evicted_entries, + evicted_weight_bytes, + ) + return rules + + +def _clear_compiled_pattern_cache() -> None: + global _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + with _COMPILED_PATTERN_CACHE_LOCK: + _COMPILED_PATTERN_CACHE.clear() + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES = 0 + + +def _clear_parsed_pattern_catalog_cache() -> None: + global _PATTERN_CATALOG_CACHE_WEIGHT_BYTES + with _PATTERN_CATALOG_CACHE_LOCK: + _PATTERN_CATALOG_CACHE.clear() + _PATTERN_CATALOG_CACHE_WEIGHT_BYTES = 0 def _compile_rule( @@ -679,9 +777,18 @@ def _rendered_template_size(template: str, entity: str) -> int: _MAX_CACHED_PATTERN_CATALOGS = 64 _PATTERN_CATALOG_CACHE: OrderedDict[ tuple[str, int, int, int, int, int], - RegexPatternCatalog, + tuple[RegexPatternCatalog, int], ] = OrderedDict() +_PATTERN_CATALOG_CACHE_WEIGHT_BYTES = 0 _PATTERN_CATALOG_CACHE_LOCK = RLock() +_MAX_CACHED_COMPILED_CATALOGS = 128 +_COMPILED_PATTERN_CACHE: OrderedDict[ + RegexPatternCatalog, + tuple[tuple[_CompiledRule, ...], int], +] = OrderedDict() +_COMPILED_PATTERN_CACHE_WEIGHT_BYTES = 0 +_COMPILED_PATTERN_CACHE_LOCK = RLock() +_LOGGER = get_logger(__name__) __all__ = [ diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index ad5b246..1587f69 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -17,7 +17,10 @@ from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc -from privacy_guard.config import PrivacyGuardConfig, configuration_fingerprint +from privacy_guard.config import ( + PrivacyGuardConfig, + _configuration_fingerprint_and_size, +) from privacy_guard.constants import ( BLOCK_REASON, BLOCK_REASON_CODE, @@ -26,6 +29,7 @@ LIMIT_REASON_CODE, MAX_BODY_BYTES, MAX_CONCURRENT_PROCESSING, + MAX_PROCESSOR_CACHE_CONFIG_BYTES, MAX_PROTO_CONFIG_BYTES, MAX_PROTO_CONTEXT_BYTES, MAX_PROTO_FINDING_BYTES, @@ -241,19 +245,23 @@ def __init__( self._registry = registry self._timeout_seconds = timeout_seconds self._log_request_content = log_request_content - self._processors: OrderedDict[str, RequestProcessor] = OrderedDict() + self._processors: OrderedDict[ + str, + tuple[RequestProcessor, int], + ] = OrderedDict() + self._weight_bytes = 0 self._in_flight: dict[str, Future[RequestProcessor]] = {} self._lock = RLock() def resolve(self, values: object) -> RequestProcessor: """Return the cached or newly prepared processor for expanded config.""" config = self._registry.validate_config(values) - fingerprint = configuration_fingerprint(config) + fingerprint, weight_bytes = _configuration_fingerprint_and_size(config) with self._lock: cached = self._processors.get(fingerprint) if cached is not None: self._processors.move_to_end(fingerprint) - return cached + return cached[0] owner_future = self._in_flight.get(fingerprint) owns_build = owner_future is None if owner_future is None: @@ -264,19 +272,46 @@ def resolve(self, values: object) -> RequestProcessor: try: processor = self._build_processor(config) except Exception as error: + owner_future.set_exception(error) with self._lock: if self._in_flight.get(fingerprint) is owner_future: del self._in_flight[fingerprint] - owner_future.set_exception(error) raise + evicted_entries = 0 + evicted_weight_bytes = 0 + with self._lock: + if weight_bytes <= MAX_PROCESSOR_CACHE_CONFIG_BYTES: + replaced = self._processors.pop(fingerprint, None) + if replaced is not None: + self._weight_bytes -= replaced[1] + self._processors[fingerprint] = (processor, weight_bytes) + self._weight_bytes += weight_bytes + while ( + len(self._processors) > _MAX_CACHED_PROCESSORS + or self._weight_bytes > MAX_PROCESSOR_CACHE_CONFIG_BYTES + ): + _, (_, evicted_weight) = self._processors.popitem(last=False) + self._weight_bytes -= evicted_weight + evicted_entries += 1 + evicted_weight_bytes += evicted_weight + owner_future.set_result(processor) with self._lock: - self._processors[fingerprint] = processor - self._processors.move_to_end(fingerprint) - while len(self._processors) > _MAX_CACHED_PROCESSORS: - self._processors.popitem(last=False) if self._in_flight.get(fingerprint) is owner_future: del self._in_flight[fingerprint] - owner_future.set_result(processor) + if weight_bytes > MAX_PROCESSOR_CACHE_CONFIG_BYTES: + _LOGGER.debug( + "privacy_guard_cache_skip cache=processor " + "weight_bytes=%d budget_bytes=%d", + weight_bytes, + MAX_PROCESSOR_CACHE_CONFIG_BYTES, + ) + if evicted_entries: + _LOGGER.debug( + "privacy_guard_cache_eviction cache=processor " + "entries=%d weight_bytes=%d", + evicted_entries, + evicted_weight_bytes, + ) return processor def _build_processor( diff --git a/projects/privacy-guard/tests/engines/test_regex.py b/projects/privacy-guard/tests/engines/test_regex.py index 007ba1c..824aa41 100644 --- a/projects/privacy-guard/tests/engines/test_regex.py +++ b/projects/privacy-guard/tests/engines/test_regex.py @@ -1,6 +1,8 @@ from __future__ import annotations +import logging from concurrent.futures import ThreadPoolExecutor +from threading import Barrier import pytest from pydantic import ValidationError @@ -12,6 +14,7 @@ EntityProcessingStrategy, RegexEngine, RegexEngineConfig, + RegexPatternCatalog, ) from privacy_guard.errors import ( ErrorCode, @@ -64,6 +67,24 @@ def _run( ] +def _catalog(pattern: str) -> RegexPatternCatalog: + return RegexPatternCatalog.model_validate( + { + "entities": [ + { + "name": "token", + "patterns": [ + { + "pattern": pattern, + "confidence": "high", + } + ], + } + ] + } + ) + + def test_detects_overlaps_and_orders_matches_deterministically() -> None: config = _config( [ @@ -290,7 +311,7 @@ def test_pattern_search_has_an_enforceable_timeout() -> None: def test_patterns_compile_during_validation_and_preparation_not_run( monkeypatch: pytest.MonkeyPatch, ) -> None: - regex_module._compile_pattern_catalog.cache_clear() + regex_module._clear_compiled_pattern_cache() compile_count = 0 original_compile = regex_module.regex.compile @@ -314,6 +335,130 @@ def recording_compile(pattern: str, flags: int = 0) -> object: assert compile_count == prepared_count +def test_compiled_catalog_cache_evicts_least_recently_used_entry( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + regex_module._clear_compiled_pattern_cache() + catalogs = tuple(_catalog(f"sensitive-pattern-{suffix}") for suffix in "abc") + + try: + first_rules = regex_module._compile_pattern_catalog(catalogs[0]) + entry_weight = regex_module._COMPILED_PATTERN_CACHE[catalogs[0]][1] + monkeypatch.setattr( + regex_module, + "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", + entry_weight * 2, + ) + with caplog.at_level(logging.DEBUG, logger="privacy_guard.engines.regex"): + regex_module._compile_pattern_catalog(catalogs[1]) + assert regex_module._compile_pattern_catalog(catalogs[0]) is first_rules + regex_module._compile_pattern_catalog(catalogs[2]) + + assert tuple(regex_module._COMPILED_PATTERN_CACHE) == ( + catalogs[0], + catalogs[2], + ) + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == sum( + entry[1] for entry in regex_module._COMPILED_PATTERN_CACHE.values() + ) + assert ( + "privacy_guard_cache_eviction cache=regex_compiled entries=1" in caplog.text + ) + assert "sensitive-pattern" not in caplog.text + finally: + regex_module._clear_compiled_pattern_cache() + + +def test_compiled_catalog_cache_skips_oversized_valid_entry( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + regex_module._clear_compiled_pattern_cache() + monkeypatch.setattr(regex_module, "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", 1) + catalog = _catalog("sensitive-oversized-pattern") + + try: + with caplog.at_level(logging.DEBUG, logger="privacy_guard.engines.regex"): + first = regex_module._compile_pattern_catalog(catalog) + second = regex_module._compile_pattern_catalog(catalog) + + assert first is not second + assert regex_module._COMPILED_PATTERN_CACHE == {} + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == 0 + assert caplog.text.count("privacy_guard_cache_skip cache=regex_compiled") == 2 + assert "sensitive-oversized-pattern" not in caplog.text + finally: + regex_module._clear_compiled_pattern_cache() + + +def test_compiled_catalog_failure_preserves_existing_weight( + monkeypatch: pytest.MonkeyPatch, +) -> None: + regex_module._clear_compiled_pattern_cache() + retained_catalog = _catalog("retained") + regex_module._compile_pattern_catalog(retained_catalog) + retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES + retained_entries = tuple(regex_module._COMPILED_PATTERN_CACHE) + + def fail_compile(*args: object, **kwargs: object) -> object: + del args, kwargs + raise ValueError("expected test failure") + + monkeypatch.setattr(regex_module, "_compile_rule", fail_compile) + try: + with pytest.raises(ValueError, match="expected test failure"): + regex_module._compile_pattern_catalog(_catalog("failing")) + + assert tuple(regex_module._COMPILED_PATTERN_CACHE) == retained_entries + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == retained_weight + finally: + regex_module._clear_compiled_pattern_cache() + + +def test_compiled_catalog_same_key_race_accounts_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + regex_module._clear_compiled_pattern_cache() + worker_count = 4 + workers_ready = Barrier(worker_count) + catalog = _catalog("same-key") + original_compile_rule = regex_module._compile_rule + + def synchronized_compile( + entity: regex_module.RegexEntity, + pattern: regex_module.RegexPattern, + catalog_index: int, + entity_pattern_index: int, + ) -> regex_module._CompiledRule: + workers_ready.wait(timeout=5) + return original_compile_rule( + entity, + pattern, + catalog_index, + entity_pattern_index, + ) + + monkeypatch.setattr(regex_module, "_compile_rule", synchronized_compile) + try: + with ThreadPoolExecutor(max_workers=worker_count) as executor: + results = tuple( + executor.map( + lambda _: regex_module._compile_pattern_catalog(catalog), + range(worker_count), + ) + ) + + assert all(result is results[0] for result in results) + assert len(regex_module._COMPILED_PATTERN_CACHE) == 1 + assert ( + regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES + == (next(iter(regex_module._COMPILED_PATTERN_CACHE.values()))[1]) + ) + finally: + regex_module._clear_compiled_pattern_cache() + + def test_regex_engine_is_safe_for_concurrent_runs() -> None: engine = RegexEngine( _config([{"pattern": "x", "confidence": "high"}]), diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index afaaffd..5c1c351 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -15,7 +15,10 @@ from google.protobuf.message import Message from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 -from privacy_guard.config import PrivacyGuardConfig +from privacy_guard.config import ( + PrivacyGuardConfig, + _configuration_fingerprint_and_size, +) from privacy_guard.constants import ( LIMIT_REASON, LIMIT_REASON_CODE, @@ -517,6 +520,10 @@ def resolve() -> RequestProcessor: assert build_count == 1 assert all(processor is processors[0] for processor in processors) + _, expected_weight = _configuration_fingerprint_and_size( + create_builtin_registry().validate_config(_values()) + ) + assert cache._weight_bytes == expected_weight assert cache._in_flight == {} @@ -553,6 +560,13 @@ def synchronized_build( assert processors[0] is not processors[1] assert len(cache._processors) == 2 + expected_weight = sum( + _configuration_fingerprint_and_size( + create_builtin_registry().validate_config(values) + )[1] + for values in (_values(action="detect"), _values(action="block")) + ) + assert cache._weight_bytes == expected_weight assert cache._in_flight == {} @@ -611,6 +625,7 @@ def capture_failure() -> PrivacyGuardError: assert build_count == 1 assert all(error is failure for error in failures) assert cache._processors == {} + assert cache._weight_bytes == 0 assert cache._in_flight == {} processor = cache.resolve(_values()) @@ -618,6 +633,160 @@ def capture_failure() -> PrivacyGuardError: assert isinstance(processor, RequestProcessor) assert build_count == 2 assert len(cache._processors) == 1 + assert cache._weight_bytes > 0 + assert cache._in_flight == {} + + +def test_processor_cache_evicts_least_recently_used_config_by_weight( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + registry = create_builtin_registry() + values = tuple( + _values(action="detect", stage_name=f"sensitive-stage-{suffix}") + for suffix in ("a", "b", "c") + ) + identities = tuple( + _configuration_fingerprint_and_size(registry.validate_config(item)) + for item in values + ) + assert len({weight for _, weight in identities}) == 1 + entry_weight = identities[0][1] + monkeypatch.setattr( + servicer_module, + "MAX_PROCESSOR_CACHE_CONFIG_BYTES", + entry_weight * 2, + ) + cache = servicer_module._RequestProcessorCache( + registry, + timeout_seconds=1, + log_request_content=False, + ) + + with caplog.at_level(logging.DEBUG, logger="privacy_guard.service.servicer"): + first = cache.resolve(values[0]) + cache.resolve(values[1]) + assert cache.resolve(values[0]) is first + cache.resolve(values[2]) + + assert tuple(cache._processors) == (identities[0][0], identities[2][0]) + assert cache._weight_bytes == entry_weight * 2 + assert "privacy_guard_cache_eviction cache=processor entries=1" in caplog.text + assert "sensitive-stage" not in caplog.text + + +def test_processor_cache_builds_but_does_not_retain_oversized_config( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + registry = create_builtin_registry() + values = _values(action="detect", stage_name="sensitive-oversized-stage") + _, weight_bytes = _configuration_fingerprint_and_size( + registry.validate_config(values) + ) + monkeypatch.setattr( + servicer_module, + "MAX_PROCESSOR_CACHE_CONFIG_BYTES", + weight_bytes - 1, + ) + build_count = 0 + original_build = servicer_module._RequestProcessorCache._build_processor + + def record_build( + cache: servicer_module._RequestProcessorCache, + config: PrivacyGuardConfig[EngineConfig], + ) -> RequestProcessor: + nonlocal build_count + build_count += 1 + return original_build(cache, config) + + monkeypatch.setattr( + servicer_module._RequestProcessorCache, + "_build_processor", + record_build, + ) + cache = servicer_module._RequestProcessorCache( + registry, + timeout_seconds=1, + log_request_content=False, + ) + + with caplog.at_level(logging.DEBUG, logger="privacy_guard.service.servicer"): + first = cache.resolve(values) + second = cache.resolve(values) + + assert first is not second + assert build_count == 2 + assert cache._processors == {} + assert cache._weight_bytes == 0 + assert caplog.text.count("privacy_guard_cache_skip cache=processor") == 2 + assert "sensitive-oversized-stage" not in caplog.text + + +def test_oversized_same_fingerprint_misses_remain_single_flight( + monkeypatch: pytest.MonkeyPatch, +) -> None: + publication_started = Event() + waiter_ready = Event() + release_publication = Event() + + class _PublicationPausingFuture(Future[RequestProcessor]): + def result(self, timeout: float | None = None) -> RequestProcessor: + waiter_ready.set() + return super().result(timeout) + + def set_result(self, result: RequestProcessor) -> None: + publication_started.set() + assert release_publication.wait(timeout=5) + super().set_result(result) + + registry = create_builtin_registry() + values = _values(action="detect", stage_name="oversized-single-flight") + _, weight_bytes = _configuration_fingerprint_and_size( + registry.validate_config(values) + ) + monkeypatch.setattr( + servicer_module, + "MAX_PROCESSOR_CACHE_CONFIG_BYTES", + weight_bytes - 1, + ) + monkeypatch.setattr(servicer_module, "Future", _PublicationPausingFuture) + build_count = 0 + original_build = servicer_module._RequestProcessorCache._build_processor + + def record_build( + cache: servicer_module._RequestProcessorCache, + config: PrivacyGuardConfig[EngineConfig], + ) -> RequestProcessor: + nonlocal build_count + build_count += 1 + return original_build(cache, config) + + monkeypatch.setattr( + servicer_module._RequestProcessorCache, + "_build_processor", + record_build, + ) + cache = servicer_module._RequestProcessorCache( + registry, + timeout_seconds=1, + log_request_content=False, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + owner = executor.submit(cache.resolve, values) + assert publication_started.wait(timeout=5) + waiter = executor.submit(cache.resolve, values) + try: + assert waiter_ready.wait(timeout=5) + finally: + release_publication.set() + processors = (owner.result(timeout=5), waiter.result(timeout=5)) + + assert processors[0] is processors[1] + assert build_count == 1 + assert cache._processors == {} + assert cache._weight_bytes == 0 assert cache._in_flight == {} @@ -632,7 +801,7 @@ def unexpected_call(*args: object, **kwargs: object) -> object: monkeypatch.setattr( servicer_module, - "configuration_fingerprint", + "_configuration_fingerprint_and_size", unexpected_call, ) monkeypatch.setattr( diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py index a488828..878832f 100644 --- a/projects/privacy-guard/tests/test_config.py +++ b/projects/privacy-guard/tests/test_config.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os import subprocess import sys @@ -216,10 +217,123 @@ def test_catalog_file_change_produces_a_new_fingerprint( assert configuration_fingerprint(first) != configuration_fingerprint(second) +def test_parsed_catalog_cache_evicts_least_recently_used_file_by_weight( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + regex_module._clear_parsed_pattern_catalog_cache() + monkeypatch.chdir(tmp_path) + paths = tuple(Path(f"sensitive-{suffix}.yaml") for suffix in "abc") + sizes: list[int] = [] + for suffix, path in zip("abc", paths, strict=True): + catalog = _config()["entity_processing"]["stages"][0]["config"][ + "pattern_catalog" + ] + catalog["entities"][0]["patterns"][0]["pattern"] = f"pattern-{suffix}" + contents = yaml.safe_dump(catalog) + path.write_text(contents, encoding="utf-8") + sizes.append(len(contents.encode("utf-8"))) + assert len(set(sizes)) == 1 + monkeypatch.setattr( + regex_module, + "MAX_REGEX_PARSED_CATALOG_CACHE_BYTES", + sizes[0] * 2, + ) + + try: + with caplog.at_level(logging.DEBUG, logger="privacy_guard.engines.regex"): + first = regex_module._load_pattern_catalog_file(str(paths[0])) + regex_module._load_pattern_catalog_file(str(paths[1])) + assert regex_module._load_pattern_catalog_file(str(paths[0])) is first + regex_module._load_pattern_catalog_file(str(paths[2])) + + assert tuple( + cache_key[0] for cache_key in regex_module._PATTERN_CATALOG_CACHE + ) == (str(paths[0]), str(paths[2])) + assert regex_module._PATTERN_CATALOG_CACHE_WEIGHT_BYTES == sizes[0] * 2 + assert regex_module._PATTERN_CATALOG_CACHE_WEIGHT_BYTES == sum( + entry[1] for entry in regex_module._PATTERN_CATALOG_CACHE.values() + ) + assert ( + "privacy_guard_cache_eviction cache=regex_parsed_catalog entries=1" + in caplog.text + ) + assert "sensitive-" not in caplog.text + finally: + regex_module._clear_parsed_pattern_catalog_cache() + + +def test_parsed_catalog_cache_skips_oversized_valid_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + regex_module._clear_parsed_pattern_catalog_cache() + monkeypatch.chdir(tmp_path) + catalog = _config()["entity_processing"]["stages"][0]["config"]["pattern_catalog"] + catalog["entities"][0]["patterns"][0]["pattern"] = "sensitive-pattern" + contents = yaml.safe_dump(catalog) + path = Path("sensitive-oversized.yaml") + path.write_text(contents, encoding="utf-8") + size = len(contents.encode("utf-8")) + monkeypatch.setattr( + regex_module, + "MAX_REGEX_PARSED_CATALOG_CACHE_BYTES", + size - 1, + ) + + try: + with caplog.at_level(logging.DEBUG, logger="privacy_guard.engines.regex"): + first = regex_module._load_pattern_catalog_file(str(path)) + second = regex_module._load_pattern_catalog_file(str(path)) + + assert first is not second + assert regex_module._PATTERN_CATALOG_CACHE == {} + assert regex_module._PATTERN_CATALOG_CACHE_WEIGHT_BYTES == 0 + assert ( + caplog.text.count("privacy_guard_cache_skip cache=regex_parsed_catalog") + == 2 + ) + assert "sensitive-oversized" not in caplog.text + assert "sensitive-pattern" not in caplog.text + finally: + regex_module._clear_parsed_pattern_catalog_cache() + + +def test_parsed_catalog_failure_preserves_existing_weight( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + regex_module._clear_parsed_pattern_catalog_cache() + monkeypatch.chdir(tmp_path) + valid_catalog = _config()["entity_processing"]["stages"][0]["config"][ + "pattern_catalog" + ] + Path("valid.yaml").write_text( + yaml.safe_dump(valid_catalog), + encoding="utf-8", + ) + Path("invalid.yaml").write_text("entities: []\n", encoding="utf-8") + + try: + regex_module._load_pattern_catalog_file("valid.yaml") + retained_keys = tuple(regex_module._PATTERN_CATALOG_CACHE) + retained_weight = regex_module._PATTERN_CATALOG_CACHE_WEIGHT_BYTES + + with pytest.raises(ValueError, match="catalog file is invalid"): + regex_module._load_pattern_catalog_file("invalid.yaml") + + assert tuple(regex_module._PATTERN_CATALOG_CACHE) == retained_keys + assert regex_module._PATTERN_CATALOG_CACHE_WEIGHT_BYTES == retained_weight + finally: + regex_module._clear_parsed_pattern_catalog_cache() + + def test_equivalent_catalogs_reuse_compiled_regex_rules( monkeypatch: pytest.MonkeyPatch, ) -> None: - regex_module._compile_pattern_catalog.cache_clear() + regex_module._clear_compiled_pattern_cache() original_compile_rule = regex_module._compile_rule compile_calls = 0 @@ -252,7 +366,7 @@ def record_compile_rule( registry.create_engine(changed.entity_processing.stages[0].config) assert compile_calls == 2 - regex_module._compile_pattern_catalog.cache_clear() + regex_module._clear_compiled_pattern_cache() @pytest.mark.parametrize( From 4023108b855ecffd31b44d9467199aef6f51c062 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 03:22:15 +0000 Subject: [PATCH 69/82] fix(privacy-guard): audit fixed test toolchain (PG-012) --- projects/privacy-guard/pyproject.toml | 3 +- projects/privacy-guard/scripts/check.sh | 13 + projects/privacy-guard/uv.lock | 428 +++++++++++++++++++++++- 3 files changed, 439 insertions(+), 5 deletions(-) diff --git a/projects/privacy-guard/pyproject.toml b/projects/privacy-guard/pyproject.toml index eb51964..b1f3548 100644 --- a/projects/privacy-guard/pyproject.toml +++ b/projects/privacy-guard/pyproject.toml @@ -27,7 +27,8 @@ Repository = "https://github.com/NVIDIA/OpenShell-Research" [dependency-groups] dev = [ - "pytest>=8.3,<9", + "pip-audit==2.10.1", + "pytest>=9.0.3,<10", "pytest-asyncio>=0.25,<2", "ruff>=0.12,<0.13", "ty>=0.0.1a16,<0.1", diff --git a/projects/privacy-guard/scripts/check.sh b/projects/privacy-guard/scripts/check.sh index dd39849..0e844da 100755 --- a/projects/privacy-guard/scripts/check.sh +++ b/projects/privacy-guard/scripts/check.sh @@ -36,8 +36,21 @@ uv pip install \ --offline \ --python "$artifact_check_dir/venv/bin/python" \ "${wheels[0]}" +runtime_site_packages=$( + "$artifact_check_dir/venv/bin/python" \ + -c 'import sysconfig; print(sysconfig.get_path("purelib"))' +) +"${uv_run[@]}" pip-audit \ + --cache-dir "$artifact_check_dir/pip-audit-cache" \ + --progress-spinner off \ + --path "$runtime_site_packages" + cp scripts/external_engine_typing.py "$artifact_check_dir/external_engine_typing.py" "${uv_run[@]}" ty check \ --project "$artifact_check_dir" \ --python "$artifact_check_dir/venv" \ "$artifact_check_dir/external_engine_typing.py" +"${uv_run[@]}" pip-audit \ + --cache-dir "$artifact_check_dir/pip-audit-cache" \ + --progress-spinner off \ + --local diff --git a/projects/privacy-guard/uv.lock b/projects/privacy-guard/uv.lock index 2d17fe6..9899c41 100644 --- a/projects/privacy-guard/uv.lock +++ b/projects/privacy-guard/uv.lock @@ -20,6 +20,116 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "boolean-py" +version = "5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c4/cf/85379f13b76f3a69bca86b60237978af17d6aa0bc5998978c3b8cf05abb2/boolean_py-5.0.tar.gz", hash = "sha256:60cbc4bad079753721d32649545505362c754e121570ada4658b852a3a318d95", size = 37047, upload-time = "2025-04-03T10:39:49.734Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/ca/78d423b324b8d77900030fa59c4aa9054261ef0925631cd2501dd015b7b7/boolean_py-5.0-py3-none-any.whl", hash = "sha256:ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9", size = 26577, upload-time = "2025-04-03T10:39:48.449Z" }, +] + +[[package]] +name = "cachecontrol" +version = "0.14.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msgpack" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/f6/c972b32d80760fb79d6b9eeb0b3010a46b89c0b23cf6329417ff7886cd22/cachecontrol-0.14.4.tar.gz", hash = "sha256:e6220afafa4c22a47dd0badb319f84475d79108100d04e26e8542ef7d3ab05a1", size = 16150, upload-time = "2025-11-14T04:32:13.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/79/c45f2d53efe6ada1110cf6f9fca095e4ff47a0454444aefdde6ac4789179/cachecontrol-0.14.4-py3-none-any.whl", hash = "sha256:b7ac014ff72ee199b5f8af1de29d60239954f223e948196fa3d84adaffc71d2b", size = 22247, upload-time = "2025-11-14T04:32:11.733Z" }, +] + +[package.optional-dependencies] +filecache = [ + { name = "filelock" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/e3/85ec501f206fb049259288c1f3506e53876937fb00edb47009348e66756b/charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5", size = 317075, upload-time = "2026-07-07T14:32:56.021Z" }, + { url = "https://files.pythonhosted.org/packages/c3/69/2a5385192e67175f7d8bd5ce4f57c24bc956439adeae5c13a99aa28a53d1/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2", size = 213837, upload-time = "2026-07-07T14:32:57.78Z" }, + { url = "https://files.pythonhosted.org/packages/b3/46/03ddc7da576d814fe0a36dd1f0fd3258e95404b4b2e3c026b7923d7e133f/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a", size = 235503, upload-time = "2026-07-07T14:32:59.205Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6e/de0229a7ef40f6f9d28a837eebf4ec47bdca5dab4e900c84f22919af636a/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29", size = 229944, upload-time = "2026-07-07T14:33:00.803Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/49b9060e8418b14fb5cba9cf6bfb383111e2538a03a1fb18e66a95aeb3d5/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c", size = 221276, upload-time = "2026-07-07T14:33:02.199Z" }, + { url = "https://files.pythonhosted.org/packages/44/95/80282cce0fae9c3061203d723ee87da996aed79679e65d8935050ee7ca1f/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b", size = 205260, upload-time = "2026-07-07T14:33:03.698Z" }, + { url = "https://files.pythonhosted.org/packages/0c/74/2f62c8821b969ea3bd67cc2e6976834f48ca5d12664d2559ebcd9bcfbed7/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db", size = 217786, upload-time = "2026-07-07T14:33:05.12Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8d/feabb82cb49fcad14515b1d7d1ca4787b0da7fc723a212bf89bc9e0fac52/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993", size = 216798, upload-time = "2026-07-07T14:33:06.629Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ff/c946d63bc3786d5b84d960b0f7ab7e25b828486a946b5aa997625bcaf6a6/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da", size = 206429, upload-time = "2026-07-07T14:33:08.006Z" }, + { url = "https://files.pythonhosted.org/packages/af/ba/5e5007c370702f85d2ef75791fac7943ed41e080364a673b20142e430e3e/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3", size = 223066, upload-time = "2026-07-07T14:33:09.783Z" }, + { url = "https://files.pythonhosted.org/packages/83/d5/9096aa3cf532dfad237861544eb47a0f20d5adbf1039760fed8eaae935d9/charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d", size = 150456, upload-time = "2026-07-07T14:33:11.217Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a1/e29995109e455dc8eff8d0fac6ae509be39561318a7cfeac5d33ad029213/charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1", size = 161410, upload-time = "2026-07-07T14:33:12.743Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8d/1569f4d0032d6ba2a4fe4591c35bf87868c600c41a71eb5c2e1ffa8464c2/charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec", size = 152649, upload-time = "2026-07-07T14:33:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, + { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" }, + { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, + { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, + { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, + { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" }, + { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, + { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, + { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" }, + { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" }, + { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" }, + { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, + { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" }, + { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -29,6 +139,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "cyclonedx-python-lib" +version = "11.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "license-expression" }, + { name = "packageurl-python" }, + { name = "py-serializable" }, + { name = "sortedcontainers" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/c9/5d0ccdd19bc7d8ab803b90695c1706aa2ea8529685d18e682dc2524d2630/cyclonedx_python_lib-11.11.0.tar.gz", hash = "sha256:4b3194db72b613717f2912447e67ab618c75ff7dcac6c4af3c0e9e1ac617c102", size = 1442983, upload-time = "2026-06-17T11:57:49.055Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/f3/56ccb2884aaa3db5622368e5191a3384b15f35392aa93df8b2f508c660d2/cyclonedx_python_lib-11.11.0-py3-none-any.whl", hash = "sha256:3049fc83e06a059b5c5907a527625a8ed5073caab10607ed4c9e5503b590fd44", size = 528689, upload-time = "2026-06-17T11:57:47.358Z" }, +] + +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + +[[package]] +name = "filelock" +version = "3.32.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/80/8232b582c4b318b817cf1274ba74976b07b34d35ef439b3eb948f98645a1/filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402", size = 213757, upload-time = "2026-07-21T13:17:42.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, +] + [[package]] name = "grpcio" version = "1.82.1" @@ -80,6 +224,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/67/03329c847172c78ddeb1eb9be6b444fdbc12775a84c958b27e427e7b926d/grpcio-1.82.1-cp314-cp314-win_amd64.whl", hash = "sha256:e20f1edbb15f99e3128ec86433f9785fd5a451d8f115e74fe0056134f092a9d5", size = 5141114, upload-time = "2026-07-08T12:36:13.595Z" }, ] +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -89,6 +242,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "license-expression" +version = "30.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "boolean-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/71/d89bb0e71b1415453980fd32315f2a037aad9f7f70f695c7cec7035feb13/license_expression-30.4.4.tar.gz", hash = "sha256:73448f0aacd8d0808895bdc4b2c8e01a8d67646e4188f887375398c761f340fd", size = 186402, upload-time = "2025-07-22T11:13:32.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/40/791891d4c0c4dab4c5e187c17261cedc26285fd41541577f900470a45a4d/license_expression-30.4.4-py3-none-any.whl", hash = "sha256:421788fdcadb41f049d2dc934ce666626265aeccefddd25e162a26f23bcbf8a4", size = 120615, upload-time = "2025-07-22T11:13:31.217Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -110,6 +275,78 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "msgpack" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/f9/c0a1c127f9049db9155afc316952ea571720dd01833ff5e4d7e8e6352dbb/msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647", size = 183960, upload-time = "2026-06-18T16:13:52.594Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/6b/e9b1cdc042c4458801d2545ed782a95f3d6ba8e270cce8745b8603c7f748/msgpack-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:29a3f6e9667868429d8240dfd063ea5ffdc1321c13d783aa23827a38de0dcb22", size = 82812, upload-time = "2026-06-18T16:12:45.022Z" }, + { url = "https://files.pythonhosted.org/packages/0c/3a/dd518a1bf78ed1e9ad8afe57307c079a00eafe4b3068932a27ca1ea56b4f/msgpack-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aded5bdf32609dc7987a49bbbd15a8ef096193f96dd8bbeb791de729e650acf5", size = 82739, upload-time = "2026-06-18T16:12:46.025Z" }, + { url = "https://files.pythonhosted.org/packages/70/e0/7ba9e1542bf0771a27b8b37c1316e3f95ae9d748fd765284655c476ad4ef/msgpack-1.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:146ee4e9ce80b365c6d4c47073da9da7bcec473e58194ceee5dd7620ace77e06", size = 414233, upload-time = "2026-06-18T16:12:47.029Z" }, + { url = "https://files.pythonhosted.org/packages/03/8d/671d81534ea0e2b0e8a121be100020da09eb78861fe3aa8f3ef7dcd3bed1/msgpack-1.2.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a28d076ca7c82b9c8728ad90b7147489449557038bed50e4241eb832395169b4", size = 423843, upload-time = "2026-06-18T16:12:48.19Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b6/e5c737515ed1f166664b87601b532f58cbb73d8aa6a90b99f7c2c5037e8e/msgpack-1.2.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7d31c0ac0c640f877804c67cb2bc9f4e23dc2db97e96c2e67fa27d38283b41f8", size = 390772, upload-time = "2026-06-18T16:12:49.624Z" }, + { url = "https://files.pythonhosted.org/packages/a8/46/62ed8c2e87d7021eab19921594d961ef3aa3794eec76c716dc30f3bfd433/msgpack-1.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ff92d7feeaf5bc26c51495b69e2f99ed97ab79346fb6555f44be7dd2ac6503b", size = 409559, upload-time = "2026-06-18T16:12:50.936Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/59aa3887b860bbf43532835e192b1c388a17590d6068ae4f8b2bc74c906e/msgpack-1.2.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:779197a6513bab3c3632265e3d0f7cb3227e62510841a6f34f1eaa37efbb345e", size = 387838, upload-time = "2026-06-18T16:12:52.161Z" }, + { url = "https://files.pythonhosted.org/packages/09/11/f8563e471093420cf6478cb3271a0175d8402b82d879783d4035d2d03360/msgpack-1.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:67f6dd22fa72a93752643f07889796d62739a13415ee630169a8ce764f86cf9f", size = 421732, upload-time = "2026-06-18T16:12:53.556Z" }, + { url = "https://files.pythonhosted.org/packages/57/cf/e673683c4c6c90c1022b24c65af4b03eda72b182a1176ef6449069d66acc/msgpack-1.2.1-cp311-cp311-win32.whl", hash = "sha256:91054a783328e0ea7954b8771095705c8d2243b814743fbaadf14552c9c52c5d", size = 64091, upload-time = "2026-06-18T16:12:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/3f/07/ca212739d179f9083bff2c7c08c24101c3555a334fadc2b876b18768a3ae/msgpack-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2eda0b7ebb1283a98d3e4492ac933c8af6aff59fd3df1c3ed024f536af4b1dc8", size = 70462, upload-time = "2026-06-18T16:12:55.898Z" }, + { url = "https://files.pythonhosted.org/packages/6d/be/6798347b425e26f35db82e69dd83c09716c856a3714e7bffc4c0860fd830/msgpack-1.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:6ee967f7c7e1df2890c671ff2ee51a28ded0efc95da3e507176dee881ce36c66", size = 65059, upload-time = "2026-06-18T16:12:57.053Z" }, + { url = "https://files.pythonhosted.org/packages/bc/dd/9e8cbd8f5582ca4b590336f2b91ee5662f6a6ca562b565abaf696a0f81ff/msgpack-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ef59c659f289eddf8aa6623823f19fa2f40a4029266889eac7a2505dd210c35", size = 83531, upload-time = "2026-06-18T16:12:58.249Z" }, + { url = "https://files.pythonhosted.org/packages/50/2e/ebdb85a8da151397a2790363676b7ed7c125924fe618e4c6d8befb0cc62c/msgpack-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3567748a5107cb40cdf66a275430c2f87c07777698f4bfd25c35f44d533258c", size = 82657, upload-time = "2026-06-18T16:12:59.396Z" }, + { url = "https://files.pythonhosted.org/packages/26/aa/753ad8b007b464e1d8aa0c8e650b9c5f4f725e658fc5ac8a7635c55b7f6e/msgpack-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60926b75d00c8e816ef98f3034f484a8bc64242d66839cef4cf7e503142316a0", size = 410634, upload-time = "2026-06-18T16:13:00.383Z" }, + { url = "https://files.pythonhosted.org/packages/6a/fd/6adabd4f6d5e686f97dd02ce7fce3fe4cf672cbac36b8f67ff4040e8ad8b/msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:020e881a764b20d8d7ca1a54fc01b8175519d108e3c3f194fddc200bda95951a", size = 419989, upload-time = "2026-06-18T16:13:01.776Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cc/85039b7b0eb168aaad7383a23c97e291a11f08351cb45a606ce865e4e3f1/msgpack-1.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4202c74688ca06591f78cb18988228bd4cca2cc75d57b60008372892d2f1e6e6", size = 377544, upload-time = "2026-06-18T16:13:03.637Z" }, + { url = "https://files.pythonhosted.org/packages/ed/bf/35963899493b32030c85fc513b723ae66144ac70c11ebc52e889e16e3d99/msgpack-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b267ce94efb76fbd1b3373511420074ee3187f0f7811bf394531de13294735a", size = 400842, upload-time = "2026-06-18T16:13:05.012Z" }, + { url = "https://files.pythonhosted.org/packages/a6/df/8e2ac970c8f99264cd9997d1c73df5466bc19da3301d7dc5500862a9b089/msgpack-1.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f1d0f8f98ade9634e01fb704a408f9336c0a8f1117b369f5db83dc7551d8b1", size = 374108, upload-time = "2026-06-18T16:13:06.232Z" }, + { url = "https://files.pythonhosted.org/packages/17/dd/fa8bd265110dfa51c20cb529f9e6d240a16fafe7e645004c6af2d01353ba/msgpack-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f02cf17a6ca1abe29b5f980644f7551f94d71f2011509b26d8625ce038f0df64", size = 414939, upload-time = "2026-06-18T16:13:07.478Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b9/8377a5ad8953fc0437c70cc98d9ae29f27fe5ac5109fbec0812085865735/msgpack-1.2.1-cp312-cp312-win32.whl", hash = "sha256:0c0d9802354507bcba62af19c17918e3eb437cc25e6f50657d511b5856a77aac", size = 64504, upload-time = "2026-06-18T16:13:08.822Z" }, + { url = "https://files.pythonhosted.org/packages/57/7f/ce1e377df7e62461fefd9eb23bfb93a4a523f40a517b377b8f844d836828/msgpack-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:5c24aa15d5963051e1a5c62b12c50cd705992502b5ec1f3bece6046f33c9fc24", size = 71421, upload-time = "2026-06-18T16:13:09.828Z" }, + { url = "https://files.pythonhosted.org/packages/8f/32/ebfe84c9929f08f188d56c7a2fd913406a9ddad76a634697c1c43b8112e6/msgpack-1.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:4227224aaec8f7fbcbfbd4272319347b2bb4030366502600f8c45588c5187b07", size = 64775, upload-time = "2026-06-18T16:13:11.056Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ac/dcddcab6f6c20ecb387ca5e980371cdb3f87ff69aeca388be97eebc4c074/msgpack-1.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0a70e3cf2804a300d921bb0940426e35f4e489a23adfb77a808892241db0a064", size = 83151, upload-time = "2026-06-18T16:13:12.173Z" }, + { url = "https://files.pythonhosted.org/packages/64/71/fbcfa83a1d6a9c6091942d1cfd070962244664b87427a9a49a6897b1b219/msgpack-1.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:491cc39455ca765fad51fb451bf2915eb2cf41192ab5801ce8d67c1d614fe056", size = 82351, upload-time = "2026-06-18T16:13:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/e3/10/ddf7b06db879e8792d13934ddda09ff20bd2a583fd84c9b59aae9b0e650b/msgpack-1.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f310233ef7fb9c14e201c93639fe5f5260b005f56f0b29048e999c30935596cc", size = 407518, upload-time = "2026-06-18T16:13:14.233Z" }, + { url = "https://files.pythonhosted.org/packages/79/d3/36a46a8ed992b781acbc05928bd5bee3c810cb0c3563bf81a7b0c04a1a76/msgpack-1.2.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:787c9bebb5833e8f6fc8abca3c0597683d8d87f56a8842b6b89c75a5f3176e2d", size = 416405, upload-time = "2026-06-18T16:13:15.435Z" }, + { url = "https://files.pythonhosted.org/packages/f9/84/e8e9598b557c0ba6ddae901a73780a4c75ac667dddf59414b1e56a42fb34/msgpack-1.2.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dc871b997a9370d855b7394465f2f350e847a5b806dd38dcc9c989e7d87da155", size = 376257, upload-time = "2026-06-18T16:13:17.022Z" }, + { url = "https://files.pythonhosted.org/packages/40/16/738fe6d875ad7e2a9429c165322a4ec088f4f273cdfae63d96a89c467961/msgpack-1.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:85f57e960d877f2977f6430896191b04a21f8901b3b4baf2e4604329f4db5402", size = 397469, upload-time = "2026-06-18T16:13:18.287Z" }, + { url = "https://files.pythonhosted.org/packages/ca/be/6d5952df75a7f24f35833af764c3a6860780364cb3a0030beb8099e1b2b4/msgpack-1.2.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1233ee2dd0cefba127583de50ea654677277047d238303521db35def3d7b2e7c", size = 372802, upload-time = "2026-06-18T16:13:19.685Z" }, + { url = "https://files.pythonhosted.org/packages/e1/39/e2ef7dbf0473bcb8dc7c50bf782a892d67414877b63e47fc88eb189ef5e6/msgpack-1.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3dc2feb0876209d9c38aa56cb1de169bd6c4348f1aa48271f241226590993e6", size = 411273, upload-time = "2026-06-18T16:13:21.028Z" }, + { url = "https://files.pythonhosted.org/packages/ef/c5/133f4512a56e983a93445c836c9d94d88f3bc2e0980ff4b9e577bd8416ce/msgpack-1.2.1-cp313-cp313-win32.whl", hash = "sha256:6d09badf350af2be9d189184e04e64cf54ad93569ab3d96fca58bd3e84aad707", size = 64471, upload-time = "2026-06-18T16:13:22.293Z" }, + { url = "https://files.pythonhosted.org/packages/e2/98/577e10b055096a7dd40732358cabaf7180a20c79ed1dcdbb618e4b9deac7/msgpack-1.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:33f14fba63278b714efe6ad07e50ea5f03d91537aa6a1c5f1ceca4cf44013ca9", size = 71274, upload-time = "2026-06-18T16:13:23.455Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ee/0c0048e7cfbef23c6a94791b8959ab28155232e7956de8a305b5ff588f05/msgpack-1.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:afc5febcd4c99effbc02b528e49d6fd0760b2b7d48c05239e345a5fa6e743d9a", size = 64795, upload-time = "2026-06-18T16:13:24.687Z" }, + { url = "https://files.pythonhosted.org/packages/77/58/cce442852c6b9e1639c7c8ac8fd9143121cb32dab0f308df4d1426a8eb9c/msgpack-1.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:05f340e47e7e47d2da8db9b53e1bb1d294369e9ef45a747441309f6650b8351d", size = 83610, upload-time = "2026-06-18T16:13:25.724Z" }, + { url = "https://files.pythonhosted.org/packages/60/5c/15b4c7a0182f75ffa90751958ba36a9c01cafee367d49a3edc10ed140b01/msgpack-1.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:810b916696c86ef0deb3b74588480224df4c1b071136c34183e4a2a4284d7ac7", size = 83138, upload-time = "2026-06-18T16:13:26.781Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a6/99e58722feaffc5f2fbcc0c8c0d1451ab9f84097f7af87291b46af2390f4/msgpack-1.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0dacff965c47afdc3749a8469d7302a8f801d6a28758d55120d75e66ce6889", size = 406090, upload-time = "2026-06-18T16:13:28.072Z" }, + { url = "https://files.pythonhosted.org/packages/19/03/8c63e8cf52958534ef688625965ab04c269a6cadd8caef16758b380a821a/msgpack-1.2.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e2bf9280bceb5efca998435904b5d3e9fdbcc11d90dc9df30aec7973252b720", size = 412106, upload-time = "2026-06-18T16:13:29.427Z" }, + { url = "https://files.pythonhosted.org/packages/63/d2/155d9e71b40e41fd934bc0c48b9b2770f22263e1ac20aad8e29fdca7be3f/msgpack-1.2.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6c4be5d1c02a42b066ca6ddb71adf36432868fdcdb6ee87e634e86e0674190", size = 374851, upload-time = "2026-06-18T16:13:30.631Z" }, + { url = "https://files.pythonhosted.org/packages/98/48/deaf2326262a8d5ea3295ce9649912ecd3f551ba7ec8e33c665d2ba583f3/msgpack-1.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec0e675d59150a6269ddc9139087c722292664a37d071a849c05c473350f1f2d", size = 396168, upload-time = "2026-06-18T16:13:31.977Z" }, + { url = "https://files.pythonhosted.org/packages/10/2a/b4410f906c2ec0008f1608d3ab5143afc3ad3f4e6da0fed3ea2231d0bef4/msgpack-1.2.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:dd3bfe82d53edfe4b7fc9a7ec9761e23a7a5b1dac22264505af428253c29ed24", size = 371959, upload-time = "2026-06-18T16:13:33.282Z" }, + { url = "https://files.pythonhosted.org/packages/59/86/1edc67270099a528fa2093ea60fe191233cd238e4bd30cfacf7db79fc959/msgpack-1.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5ad5467fc3f68b5468e06c5f788d712e9f8ffc8b0cd1bcb160c105c1ee92dae7", size = 408457, upload-time = "2026-06-18T16:13:34.567Z" }, + { url = "https://files.pythonhosted.org/packages/82/90/8b630fef07d8c5ab457b71ff2c217910c83d333c7a68472c186e87cc504a/msgpack-1.2.1-cp314-cp314-win32.whl", hash = "sha256:98b58bdb89c46190e4609bb36abe17c6d4105ad13f9c5f8f6f64d320f8ced3fb", size = 65942, upload-time = "2026-06-18T16:13:36.056Z" }, + { url = "https://files.pythonhosted.org/packages/16/f1/467b81e98b24dd3885d7b1857728797b4ffc76a7a7483af4fb321a07de3c/msgpack-1.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:74847557e28ce71bd3c438a447ca90e4b507e997ddbdef8a12a7b283b86c156b", size = 72627, upload-time = "2026-06-18T16:13:37.079Z" }, + { url = "https://files.pythonhosted.org/packages/a7/1d/5d8c4c89985feb6acefb82a09e501c60392261856d2408d20bfe4f0360b1/msgpack-1.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:b50b727bd652bdc37d950336c848ef20ec54a4cafc38dce19b1cd86ad625d0f7", size = 66908, upload-time = "2026-06-18T16:13:38.23Z" }, + { url = "https://files.pythonhosted.org/packages/1b/02/ad2afb678b4de94496cd432b581759b756a92c1192d8c767edd6b132efdc/msgpack-1.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:8d00f177ca88a77c1cf848d204a38f249751650b601cb6532acc68805d8a8273", size = 86000, upload-time = "2026-06-18T16:13:39.44Z" }, + { url = "https://files.pythonhosted.org/packages/54/74/0b797484013128837f3b1cbb6cea019277c4de4e377dc512b4d9a0f92940/msgpack-1.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5bb9c386f0a329c035ddbab4b72d1028bf9627add8dda41070288563d57ed1b1", size = 86544, upload-time = "2026-06-18T16:13:40.447Z" }, + { url = "https://files.pythonhosted.org/packages/a9/b4/b774d7eb95561739907fec675582f83203cf41c597a418c2589b4bfb8e9d/msgpack-1.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20466cca18c49c7292a8984bc15d65857b171e7264bdcb5f96baf8be238791fc", size = 427661, upload-time = "2026-06-18T16:13:41.574Z" }, + { url = "https://files.pythonhosted.org/packages/b2/f9/3243191dc9937e00756c8bc1b0272fed8f23758e43df2a3b46f533e5090f/msgpack-1.2.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:196300e7e5d6e74d50f1607ab9c06c4a1484c383cd22defd727902591f7e8dde", size = 426375, upload-time = "2026-06-18T16:13:42.936Z" }, + { url = "https://files.pythonhosted.org/packages/23/c7/1693111db9944ba4ad4b67a1e788400d78a0b6af7a6523dc7e4e58f8274b/msgpack-1.2.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575957e79cd51903a4e8495a242442949641e08f1efd5197b43bebd3ea7682b4", size = 380495, upload-time = "2026-06-18T16:13:44.306Z" }, + { url = "https://files.pythonhosted.org/packages/3e/2b/92f86956a0c13e8662f7e2ad630c4eb4db07497b967589bd5245e018b2c1/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8c2ed1e48cc0f460bf3c7780e7137ff21a4e18433451916f2442c1b21036cd7d", size = 410897, upload-time = "2026-06-18T16:13:45.629Z" }, + { url = "https://files.pythonhosted.org/packages/da/ea/1479f72d200313a76fc2f823a79d1e07ed052ab7b8a0280640aa7b95de42/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5f6277e5f783c36786a145e0247fc189a03f35f84b251646e53592d2bc12b355", size = 378519, upload-time = "2026-06-18T16:13:46.998Z" }, + { url = "https://files.pythonhosted.org/packages/f5/4d/fa006060ffa1011d32bfae826fe766fe73e02982183601633b7121058ab3/msgpack-1.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9389552ecf4784886345ead0647e4edc96bee37cbab05b75540f542f766c48c", size = 419815, upload-time = "2026-06-18T16:13:48.205Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/aab6c946570496b78e67804721f3d5e2d62a93081b9b37df77764ef56347/msgpack-1.2.1-cp314-cp314t-win32.whl", hash = "sha256:c1c79a604a2969a868a78b6ebd27a887e00c624f14f66b3038e0590cb23332d1", size = 70914, upload-time = "2026-06-18T16:13:49.385Z" }, + { url = "https://files.pythonhosted.org/packages/13/0a/e608956488a2af014cfe6e3d665e090b8ee42aa14b07f8f95b8880d66b09/msgpack-1.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f12038a35fabd52e56a3547bab42401af49a45caa6dd00b34c44de235bc93ee2", size = 77999, upload-time = "2026-06-18T16:13:50.467Z" }, + { url = "https://files.pythonhosted.org/packages/d2/8a/27e2e57055176e366a46b85d02d68e7a5bcfbdd8474c9706375d965f24d3/msgpack-1.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0adcf06ffde0777c0e1a9b771a2b1c4226ba1bbf748c8efcc02fcdeca3299107", size = 71160, upload-time = "2026-06-18T16:13:51.498Z" }, +] + +[[package]] +name = "packageurl-python" +version = "0.17.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/d6/3b5a4e3cfaef7a53869a26ceb034d1ff5e5c27c814ce77260a96d50ab7bb/packageurl_python-0.17.6.tar.gz", hash = "sha256:1252ce3a102372ca6f86eb968e16f9014c4ba511c5c37d95a7f023e2ca6e5c25", size = 50618, upload-time = "2025-11-24T15:20:17.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/2f/c7277b7615a93f51b5fbc1eacfc1b75e8103370e786fd8ce2abf6e5c04ab/packageurl_python-0.17.6-py3-none-any.whl", hash = "sha256:31a85c2717bc41dd818f3c62908685ff9eebcb68588213745b14a6ee9e7df7c9", size = 36776, upload-time = "2025-11-24T15:20:16.962Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -119,6 +356,70 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pip" +version = "26.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/91/47e7d486260f618783899587af63ccf7980fb60245c3e63dd4571c6b57ad/pip-26.1.2.tar.gz", hash = "sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605", size = 1840799, upload-time = "2026-05-31T17:33:58.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/95/6b5cb3461ea5673ba0995989746db58eb18b91b54dbf331e72f569540946/pip-26.1.2-py3-none-any.whl", hash = "sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab", size = 1813144, upload-time = "2026-05-31T17:33:56.772Z" }, +] + +[[package]] +name = "pip-api" +version = "0.0.34" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pip" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/f1/ee85f8c7e82bccf90a3c7aad22863cc6e20057860a1361083cd2adacb92e/pip_api-0.0.34.tar.gz", hash = "sha256:9b75e958f14c5a2614bae415f2adf7eeb54d50a2cfbe7e24fd4826471bac3625", size = 123017, upload-time = "2024-07-09T20:32:30.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/f7/ebf5003e1065fd00b4cbef53bf0a65c3d3e1b599b676d5383ccb7a8b88ba/pip_api-0.0.34-py3-none-any.whl", hash = "sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb", size = 120369, upload-time = "2024-07-09T20:32:29.099Z" }, +] + +[[package]] +name = "pip-audit" +version = "2.10.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachecontrol", extra = ["filecache"] }, + { name = "cyclonedx-python-lib" }, + { name = "packaging" }, + { name = "pip-api" }, + { name = "pip-requirements-parser" }, + { name = "platformdirs" }, + { name = "requests" }, + { name = "rich" }, + { name = "tomli" }, + { name = "tomli-w" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/a4/f21d5f0a0edabcbce31560b73c7c5a6f72ae87af4236fd1069c8f59a353d/pip_audit-2.10.1.tar.gz", hash = "sha256:1eb4565d19ebe5d48996f4b770b4d2b32887e12cb12cfa637f1a064011b55ffc", size = 54275, upload-time = "2026-06-10T22:17:01.744Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/a7/b0c504148114047bd1bc9d97447453c6850ca176bb2f3c0038835994e8b7/pip_audit-2.10.1-py3-none-any.whl", hash = "sha256:99ef3f600a317c1945f1e89e227ef26e1c2d618429b8bd3fa6f4f7c440c4611a", size = 62023, upload-time = "2026-06-10T22:17:00.309Z" }, +] + +[[package]] +name = "pip-requirements-parser" +version = "32.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "pyparsing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/2a/63b574101850e7f7b306ddbdb02cb294380d37948140eecd468fae392b54/pip-requirements-parser-32.0.1.tar.gz", hash = "sha256:b4fa3a7a0be38243123cf9d1f3518da10c51bdb165a2b2985566247f9155a7d3", size = 209359, upload-time = "2022-12-21T15:25:22.732Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/d0/d04f1d1e064ac901439699ee097f58688caadea42498ec9c4b4ad2ef84ab/pip_requirements_parser-32.0.1-py3-none-any.whl", hash = "sha256:4659bc2a667783e7a15d190f6fccf8b2486685b6dba4c19c3876314769c57526", size = 35648, upload-time = "2022-12-21T15:25:21.046Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/78/9b/560e4be8e26f6fd133a03630a8df0c663b9e8d61b4ade152b72005aec83b/platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0", size = 31953, upload-time = "2026-07-21T13:09:36.565Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/68/d8d58938dfb1370b266a1a729e6d77a985be23689a0496498ee17b2cbf90/platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74", size = 23247, upload-time = "2026-07-21T13:09:35.422Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -144,6 +445,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "pip-audit" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "ruff" }, @@ -163,7 +465,8 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ - { name = "pytest", specifier = ">=8.3,<9" }, + { name = "pip-audit", specifier = "==2.10.1" }, + { name = "pytest", specifier = ">=9.0.3,<10" }, { name = "pytest-asyncio", specifier = ">=0.25,<2" }, { name = "ruff", specifier = ">=0.12,<0.13" }, { name = "ty", specifier = ">=0.0.1a16,<0.1" }, @@ -184,6 +487,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] +[[package]] +name = "py-serializable" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "defusedxml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/21/d250cfca8ff30c2e5a7447bc13861541126ce9bd4426cd5d0c9f08b5547d/py_serializable-2.1.0.tar.gz", hash = "sha256:9d5db56154a867a9b897c0163b33a793c804c80cee984116d02d49e4578fc103", size = 52368, upload-time = "2025-07-21T09:56:48.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/bf/7595e817906a29453ba4d99394e781b6fabe55d21f3c15d240f85dd06bb1/py_serializable-2.1.0-py3-none-any.whl", hash = "sha256:b56d5d686b5a03ba4f4db5e769dc32336e142fc3bd4d68a8c25579ebb0a67304", size = 23045, upload-time = "2025-07-21T09:56:46.848Z" }, +] + [[package]] name = "pydantic" version = "2.13.4" @@ -310,9 +625,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyparsing" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, +] + [[package]] name = "pytest" -version = "8.4.2" +version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -321,9 +645,9 @@ dependencies = [ { name = "pluggy" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] [[package]] @@ -498,6 +822,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/b6/26e41975febae63b7a6e3e02f32cff6cff2e4f10d19c929082f56aebf7c6/regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547", size = 283451, upload-time = "2026-07-19T00:19:46.639Z" }, ] +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + [[package]] name = "rich" version = "15.0.0" @@ -546,6 +885,78 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "tomli-w" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, +] + [[package]] name = "ty" version = "0.0.61" @@ -606,3 +1017,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] From a59b06d96130669841bdd977e5fd0da42aa705db Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 03:38:42 +0000 Subject: [PATCH 70/82] fix(privacy-guard): close panel error gaps (PG-002 PG-004 PG-005) --- projects/privacy-guard/README.md | 6 ++++-- projects/privacy-guard/src/privacy_guard/errors.py | 8 +++++--- .../privacy-guard/src/privacy_guard/service/server.py | 1 + projects/privacy-guard/tests/service/test_server.py | 7 +++++++ projects/privacy-guard/tests/test_errors.py | 6 ++++++ 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 728f90b..8be2264 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -245,8 +245,10 @@ The `privacy_guard.cli` module owns the executable application and adapts its - Regex searches receive the remaining timeout and fail atomically. - Intermediate text and detection cardinality are bounded. - Detect and block never return a body mutation; replace returns final text. -- Findings expose entity, bounded confidence, count, and stage provenance, but - never matched text, surrounding text, offsets, patterns, or raw tool metadata. +- Findings expose entity, bounded confidence, count, and stage provenance. + Framework-controlled fields and compliant custom engines never expose matched + or surrounding text, offsets, patterns, or raw tool metadata. Custom engines + must return stable, declared identifiers rather than request-derived values. - Engine instances and injected resources must be safe for concurrent requests. - Cross-request entity memory is intentionally out of scope. diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py index 05bfe1c..9bfec62 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from enum import StrEnum -from privacy_guard.constants import MAX_TIMEOUT_SECONDS +from privacy_guard.constants import MAX_PROTO_CONFIG_BYTES, MAX_TIMEOUT_SECONDS class ErrorKind(StrEnum): @@ -126,8 +126,10 @@ class EngineRegistryError(Exception): ErrorComponent.CONFIG, "parse", "Policy configuration is invalid.", - "Compare it with `privacy-guard configuration-schema`, then check the " - "stages, engine settings, pattern catalogs, replacements, and action.", + "Keep the encoded configuration at or below " + f"{MAX_PROTO_CONFIG_BYTES // 1024} KiB, compare it with " + "`privacy-guard configuration-schema`, then check the stages, engine " + "settings, pattern catalogs, replacements, and action.", ), ErrorCode.REQUEST_PHASE_INVALID: _ErrorSpec( ErrorKind.INVALID_INPUT, diff --git a/projects/privacy-guard/src/privacy_guard/service/server.py b/projects/privacy-guard/src/privacy_guard/service/server.py index b42b5d7..4465c04 100644 --- a/projects/privacy-guard/src/privacy_guard/service/server.py +++ b/projects/privacy-guard/src/privacy_guard/service/server.py @@ -107,6 +107,7 @@ def _validated_listen_port(listen: str) -> int: if ( not host or not port_text + or len(port_text) > 5 or not port_text.isascii() or not port_text.isdecimal() ): diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index f587dd7..2539740 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -372,6 +372,13 @@ def test_listen_address_rejects_invalid_numeric_ports_and_forms( assert captured.value.code is ErrorCode.SERVER_BIND_FAILED +def test_listen_address_rejects_arbitrarily_long_decimal_port() -> None: + with pytest.raises(PrivacyGuardError) as captured: + server_module._validated_listen_port(f"127.0.0.1:{'9' * 5_000}") + + assert captured.value.code is ErrorCode.SERVER_BIND_FAILED + + @pytest.mark.asyncio async def test_serve_async_rejects_mismatched_bound_port( monkeypatch: pytest.MonkeyPatch, diff --git a/projects/privacy-guard/tests/test_errors.py b/projects/privacy-guard/tests/test_errors.py index eee5509..e38e0ed 100644 --- a/projects/privacy-guard/tests/test_errors.py +++ b/projects/privacy-guard/tests/test_errors.py @@ -35,5 +35,11 @@ def test_error_kinds_distinguish_invalid_input_from_internal_failures() -> None: ) +def test_config_error_explains_the_transport_size_limit() -> None: + error = PrivacyGuardError(ErrorCode.CONFIG_INVALID) + + assert "encoded configuration at or below 64 KiB" in error.hint + + def test_privacy_guard_error_exposes_only_a_catalog_code_parameter() -> None: assert list(inspect.signature(PrivacyGuardError).parameters) == ["code"] From 24cca6d5985fc445cb7199684507cab711bb1cd1 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 03:51:27 +0000 Subject: [PATCH 71/82] fix(privacy-guard): resolve panel findings (PG-005 PG-006 PG-009) --- .../architecture/safety-and-limits.md | 13 +- .../architecture/service-boundary.md | 32 +- .../src/privacy_guard/engines/regex.py | 245 +++++++++-- .../src/privacy_guard/request_processor.py | 3 + .../src/privacy_guard/service/servicer.py | 134 +++++- .../privacy-guard/tests/engines/test_regex.py | 79 +++- .../tests/service/test_servicer.py | 384 +++++++++++++++++- .../tests/test_request_processor.py | 16 + 8 files changed, 818 insertions(+), 88 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 3d80336..2976e7c 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -74,8 +74,10 @@ from the shared domain field type. These values must be non-empty printable Unicode without control or bidirectional formatting characters. Untrusted request IDs use the same content and size rules for logging; an invalid request ID is represented by a constant placeholder and does not change the evaluation -result. Regex entity and supplied pattern names have a stricter ASCII grammar -and limit described below. +result. The human-readable log message quotes request IDs and escapes ASCII +spaces so their text cannot introduce another key/value token; structured log +records retain the exact validated ID. Regex entity and supplied pattern names +have a stricter ASCII grammar and limit described below. The processor aggregates occurrences before the service applies protobuf limits. If a safe result cannot be represented, the service returns a limit @@ -250,11 +252,14 @@ Prepared state is bounded by weight and least-recently-used entry count: | Cache | Weight budget | Entry cap | Entry weight | | --- | ---: | ---: | --- | | Parsed Regex file catalogs | 8 MiB | 64 | source file bytes | -| Compiled Regex catalogs | 32 MiB | 128 | canonical catalog bytes plus 4 KiB per rule | +| Compiled Regex state retained by its LRU or cached processors | 32 MiB | 128 | canonical catalog bytes plus 4 KiB per rule | | Request processors | 1 MiB | 128 | canonical expanded configuration bytes | The compiled-rule allowance accounts for retained backend state that is much -larger than short pattern text. Processor weighting remains engine-neutral, so +larger than short pattern text. Equal catalogs share one compiled tuple and one +weight charge across the compiled LRU and any cached Regex processors that use +it. Processor configuration weighting remains engine-neutral, and the +Regex-specific ownership accounting is private to the built-in engine, so custom engines do not need a cache-size hook. One entry that exceeds its cache budget remains valid and usable for the current evaluation but is not retained. Debug skip and eviction records contain only cache names and weights, plus diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index 6636e22..78262b5 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -40,11 +40,12 @@ validation and every request evaluation. `ValidateConfig`: -1. converts the `Struct` to a mapping -2. validates the registry-built discriminated policy model -3. validates each exact engine config against registered resources -4. validates replacement support for a replace action -5. returns `valid=true`, or a content-safe reason +1. rejects an encoded `Struct` larger than 64 KiB before conversion +2. converts the `Struct` to a mapping +3. validates the registry-built discriminated policy model +4. validates each exact engine config against registered resources +5. validates replacement support for a replace action +6. returns `valid=true`, or a content-safe reason It does not construct engines, populate the processor cache, contact model providers, download resources, or write artifacts. Validation runs in the @@ -62,17 +63,20 @@ still occurs for every evaluation, while equivalent normalized regex catalogs reuse their bounded compiled-rule entry instead of recompiling. Regex keeps two additional owner-local LRU caches. Parsed file catalogs retain -at most 64 entries and 8 MiB of source files. Compiled catalogs retain at most -128 entries and 32 MiB of weighted state, where each entry weighs its canonical -catalog bytes plus 4 KiB per compiled rule. The rule allowance conservatively -represents backend and Python state that the catalog encoding alone does not -capture. +at most 64 entries and 8 MiB of source files. Compiled Regex state retained by +either the compiled-catalog LRU or cached processors shares one 128-catalog, +32 MiB union budget. Each canonical catalog weighs its encoded bytes plus 4 KiB +per compiled rule and is counted once when both owners refer to it. The rule +allowance conservatively represents backend and Python state that the catalog +encoding alone does not capture. An otherwise valid entry larger than one cache's byte budget is built and -returned without being retained. Eviction or a skipped entry never makes a -policy invalid. Content-safe debug events report the cache name and weight, plus -the number of entries for an eviction, without logging configuration, paths, -patterns, or fingerprints. +returned without being retained. A processor whose unique compiled Regex state +does not fit the union budget is likewise returned for the current evaluation +without processor-cache retention. Eviction or a skipped entry never makes a +policy invalid. Content-safe debug events report the cache name and weight, +plus the number of entries for an eviction, without logging configuration, +paths, patterns, or fingerprints. The cache is protected for concurrent access and is not correctness-relevant. Eviction or process restart simply causes reconstruction from a later diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index 80cbb80..2582099 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -48,8 +48,6 @@ EngineConfigurationError, EngineContractError, EngineLimitExceededError, - ErrorCode, - PrivacyGuardError, ) from privacy_guard.logging import get_logger from privacy_guard.string_validators import ScalarString, validate_scalar_string @@ -255,7 +253,9 @@ def _run( break start, end = match.span() if start == end: - raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) + raise EngineConfigurationError( + "regex engine configuration is invalid" + ) if match.span(rule.marker) != (end, end): raise EngineConfigurationError( "regex engine configuration is invalid" @@ -308,6 +308,30 @@ class _CompiledRule: compiled: _CompiledPattern +@dataclass +class _LeasedCompiledRules: + rules: tuple[_CompiledRule, ...] + weight_bytes: int + lease_count: int + + +class _CompiledProcessorLease: + """Private ownership token for Regex state retained by one processor.""" + + __slots__ = ("_catalogs", "_released") + + def __init__(self, catalogs: tuple[RegexPatternCatalog, ...]) -> None: + self._catalogs = catalogs + self._released = False + + def release(self) -> None: + """Release this processor's retained compiled-state ownership.""" + if self._released: + return + self._released = True + _release_compiled_processor_lease(self._catalogs) + + class _RegexMatch(Protocol): def span(self, group: int | str = 0) -> tuple[int, int]: """Return the matched span for a numbered or named group.""" @@ -552,6 +576,9 @@ def _compile_pattern_catalog( if cached is not None: _COMPILED_PATTERN_CACHE.move_to_end(catalog) return cached[0] + leased = _COMPILED_PATTERN_LEASES.get(catalog) + if leased is not None: + return leased.rules rules = tuple( _compile_rule( @@ -564,16 +591,7 @@ def _compile_pattern_catalog( _iter_catalog_patterns(catalog) ) ) - catalog_bytes = len( - json.dumps( - catalog.model_dump(mode="json"), - allow_nan=False, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - ) - weight_bytes = catalog_bytes + len(rules) * REGEX_COMPILED_RULE_WEIGHT_BYTES + weight_bytes = _compiled_pattern_weight(catalog, len(rules)) if weight_bytes > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES: _LOGGER.debug( "privacy_guard_cache_skip cache=regex_compiled " @@ -585,23 +603,47 @@ def _compile_pattern_catalog( evicted_entries = 0 evicted_weight_bytes = 0 + retained = False with _COMPILED_PATTERN_CACHE_LOCK: cached = _COMPILED_PATTERN_CACHE.get(catalog) if cached is not None: _COMPILED_PATTERN_CACHE.move_to_end(catalog) return cached[0] - _COMPILED_PATTERN_CACHE[catalog] = (rules, weight_bytes) + leased = _COMPILED_PATTERN_LEASES.get(catalog) + if leased is not None: + return leased.rules + global _COMPILED_PATTERN_CACHE_WEIGHT_BYTES - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES += weight_bytes - while ( - len(_COMPILED_PATTERN_CACHE) > _MAX_CACHED_COMPILED_CATALOGS - or _COMPILED_PATTERN_CACHE_WEIGHT_BYTES - > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES + retained_count = _compiled_pattern_retained_count() + removable_weight = sum( + cached_weight + for cached_catalog, (_, cached_weight) in _COMPILED_PATTERN_CACHE.items() + if cached_catalog not in _COMPILED_PATTERN_LEASES + ) + removable_count = sum( + cached_catalog not in _COMPILED_PATTERN_LEASES + for cached_catalog in _COMPILED_PATTERN_CACHE + ) + if ( + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + weight_bytes - removable_weight + <= MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES + and retained_count + 1 - removable_count <= _MAX_CACHED_COMPILED_CATALOGS ): - _, (_, evicted_weight) = _COMPILED_PATTERN_CACHE.popitem(last=False) - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES -= evicted_weight - evicted_entries += 1 - evicted_weight_bytes += evicted_weight + while _COMPILED_PATTERN_CACHE and ( + _compiled_pattern_retained_count() >= _MAX_CACHED_COMPILED_CATALOGS + or _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + weight_bytes + > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES + ): + evicted_catalog, (_, evicted_weight) = _COMPILED_PATTERN_CACHE.popitem( + last=False + ) + if evicted_catalog not in _COMPILED_PATTERN_LEASES: + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES -= evicted_weight + evicted_weight_bytes += evicted_weight + evicted_entries += 1 + _COMPILED_PATTERN_CACHE[catalog] = (rules, weight_bytes) + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES += weight_bytes + retained = True if evicted_entries: _LOGGER.debug( "privacy_guard_cache_eviction cache=regex_compiled " @@ -609,14 +651,167 @@ def _compile_pattern_catalog( evicted_entries, evicted_weight_bytes, ) + if not retained: + _LOGGER.debug( + "privacy_guard_cache_skip cache=regex_compiled " + "weight_bytes=%d budget_bytes=%d", + weight_bytes, + MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES, + ) return rules +def _compiled_processor_weight(engines: tuple[RegexEngine, ...]) -> int: + catalogs = {engine.config.pattern_catalog: len(engine._rules) for engine in engines} + return sum( + _compiled_pattern_weight(catalog, rule_count) + for catalog, rule_count in catalogs.items() + ) + + +def _try_acquire_compiled_processor_lease( + engines: tuple[RegexEngine, ...], +) -> _CompiledProcessorLease | None: + """Try to retain one processor's unique Regex state under the union budget.""" + engines_by_catalog: dict[RegexPatternCatalog, list[RegexEngine]] = {} + for engine in engines: + engines_by_catalog.setdefault(engine.config.pattern_catalog, []).append(engine) + compiled = { + catalog: catalog_engines[0]._rules + for catalog, catalog_engines in engines_by_catalog.items() + } + if ( + sum( + _compiled_pattern_weight(catalog, len(rules)) + for catalog, rules in compiled.items() + ) + > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES + ): + return None + + evicted_entries = 0 + evicted_weight_bytes = 0 + lease: _CompiledProcessorLease | None = None + with _COMPILED_PATTERN_CACHE_LOCK: + additional_weight = sum( + _compiled_pattern_weight(catalog, len(rules)) + for catalog, rules in compiled.items() + if catalog not in _COMPILED_PATTERN_CACHE + and catalog not in _COMPILED_PATTERN_LEASES + ) + additional_count = sum( + catalog not in _COMPILED_PATTERN_CACHE + and catalog not in _COMPILED_PATTERN_LEASES + for catalog in compiled + ) + global _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + retained_count = _compiled_pattern_retained_count() + removable_weight = sum( + cached_weight + for cached_catalog, (_, cached_weight) in _COMPILED_PATTERN_CACHE.items() + if cached_catalog not in _COMPILED_PATTERN_LEASES + ) + removable_count = sum( + cached_catalog not in _COMPILED_PATTERN_LEASES + for cached_catalog in _COMPILED_PATTERN_CACHE + ) + if ( + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + additional_weight - removable_weight + <= MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES + and retained_count + additional_count - removable_count + <= _MAX_CACHED_COMPILED_CATALOGS + ): + while _COMPILED_PATTERN_CACHE and ( + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + additional_weight + > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES + or _compiled_pattern_retained_count() + additional_count + > _MAX_CACHED_COMPILED_CATALOGS + ): + evicted_catalog, (_, evicted_weight) = _COMPILED_PATTERN_CACHE.popitem( + last=False + ) + if evicted_catalog not in _COMPILED_PATTERN_LEASES: + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES -= evicted_weight + evicted_weight_bytes += evicted_weight + evicted_entries += 1 + + for catalog, rules in compiled.items(): + leased = _COMPILED_PATTERN_LEASES.get(catalog) + if leased is not None: + retained_rules = leased.rules + leased.lease_count += 1 + else: + cached = _COMPILED_PATTERN_CACHE.get(catalog) + if cached is not None: + retained_rules, weight_bytes = cached + else: + retained_rules = rules + weight_bytes = _compiled_pattern_weight(catalog, len(rules)) + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES += weight_bytes + _COMPILED_PATTERN_LEASES[catalog] = _LeasedCompiledRules( + rules=retained_rules, + weight_bytes=weight_bytes, + lease_count=1, + ) + for engine in engines_by_catalog[catalog]: + engine._rules = retained_rules + lease = _CompiledProcessorLease(tuple(compiled)) + + if evicted_entries: + _LOGGER.debug( + "privacy_guard_cache_eviction cache=regex_compiled " + "entries=%d weight_bytes=%d", + evicted_entries, + evicted_weight_bytes, + ) + return lease + + +def _compiled_pattern_retained_count() -> int: + return len(_COMPILED_PATTERN_CACHE) + sum( + catalog not in _COMPILED_PATTERN_CACHE for catalog in _COMPILED_PATTERN_LEASES + ) + + +def _release_compiled_processor_lease( + catalogs: tuple[RegexPatternCatalog, ...], +) -> None: + global _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + with _COMPILED_PATTERN_CACHE_LOCK: + for catalog in catalogs: + leased = _COMPILED_PATTERN_LEASES[catalog] + leased.lease_count -= 1 + if leased.lease_count: + continue + del _COMPILED_PATTERN_LEASES[catalog] + if catalog not in _COMPILED_PATTERN_CACHE: + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES -= leased.weight_bytes + + +def _compiled_pattern_weight( + catalog: RegexPatternCatalog, + rule_count: int, +) -> int: + catalog_bytes = len( + json.dumps( + catalog.model_dump(mode="json"), + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ) + return catalog_bytes + rule_count * REGEX_COMPILED_RULE_WEIGHT_BYTES + + def _clear_compiled_pattern_cache() -> None: global _COMPILED_PATTERN_CACHE_WEIGHT_BYTES with _COMPILED_PATTERN_CACHE_LOCK: + retained_lease_weight = sum( + leased.weight_bytes for leased in _COMPILED_PATTERN_LEASES.values() + ) _COMPILED_PATTERN_CACHE.clear() - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES = 0 + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES = retained_lease_weight def _clear_parsed_pattern_catalog_cache() -> None: @@ -786,7 +981,9 @@ def _rendered_template_size(template: str, entity: str) -> int: RegexPatternCatalog, tuple[tuple[_CompiledRule, ...], int], ] = OrderedDict() +# This is the union retained by the LRU and processor leases, not their sum. _COMPILED_PATTERN_CACHE_WEIGHT_BYTES = 0 +_COMPILED_PATTERN_LEASES: dict[RegexPatternCatalog, _LeasedCompiledRules] = {} _COMPILED_PATTERN_CACHE_LOCK = RLock() _LOGGER = get_logger(__name__) diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index 23e1b41..b12ad91 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -25,6 +25,7 @@ TextProcessingResult, ) from privacy_guard.errors import ( + EngineConfigurationError, EngineContractError, EngineLimitExceededError, EntityProcessingError, @@ -145,6 +146,8 @@ def process(self, text: str) -> RequestProcessingResult: decision=RequestDecision.DENY, reason_code=LIMIT_REASON_CODE, ) + except EngineConfigurationError: + raise PrivacyGuardError(ErrorCode.CONFIG_INVALID) from None except EngineContractError: raise PrivacyGuardError(ErrorCode.ENGINE_OUTPUT_INVALID) from None except EntityProcessingError: diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index 1587f69..36bbffc 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -3,11 +3,13 @@ from __future__ import annotations import asyncio +import json import math import time from collections import OrderedDict from collections.abc import Callable, Iterable from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass from threading import RLock from typing import Never, Protocol, TypedDict, TypeVar @@ -42,6 +44,7 @@ SERVICE_VERSION, ) from privacy_guard.engines import EngineConfig +from privacy_guard.engines import regex as regex_engine from privacy_guard.engines.registry import EngineRegistry from privacy_guard.errors import ( EngineRegistryError, @@ -87,6 +90,7 @@ def __init__( async def close(self) -> None: """Wait for in-flight synchronous engines during shutdown.""" self._processing_executor.shutdown(wait=True, cancel_futures=True) + self._processors.clear() async def Describe( self, @@ -173,7 +177,7 @@ async def _evaluate_rpc( _LOGGER.info( "privacy_guard_evaluation request_id=%s duration_ms=%.3f " "action=%s finding_count=%d error_code=%s", - log_extra["request_id"], + _request_id_for_log_message(log_extra["request_id"]), log_extra["duration_ms"], log_extra["action"], log_extra["finding_count"], @@ -232,6 +236,12 @@ async def _run_in_worker( return await asyncio.shield(future) +@dataclass(frozen=True) +class _PreparedProcessor: + processor: RequestProcessor + regex_engines: tuple[regex_engine.RegexEngine, ...] + + class _RequestProcessorCache: """Bounded, recoverable cache keyed by canonical expanded configuration.""" @@ -247,7 +257,11 @@ def __init__( self._log_request_content = log_request_content self._processors: OrderedDict[ str, - tuple[RequestProcessor, int], + tuple[ + RequestProcessor, + int, + regex_engine._CompiledProcessorLease | None, + ], ] = OrderedDict() self._weight_bytes = 0 self._in_flight: dict[str, Future[RequestProcessor]] = {} @@ -270,40 +284,98 @@ def resolve(self, values: object) -> RequestProcessor: if not owns_build: return owner_future.result() try: - processor = self._build_processor(config) + prepared = self._build_processor(config) except Exception as error: owner_future.set_exception(error) with self._lock: if self._in_flight.get(fingerprint) is owner_future: del self._in_flight[fingerprint] raise + processor = prepared.processor + regex_weight_bytes = regex_engine._compiled_processor_weight( + prepared.regex_engines + ) evicted_entries = 0 evicted_weight_bytes = 0 + cacheable = ( + weight_bytes <= MAX_PROCESSOR_CACHE_CONFIG_BYTES + and regex_weight_bytes <= regex_engine.MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES + ) with self._lock: - if weight_bytes <= MAX_PROCESSOR_CACHE_CONFIG_BYTES: - replaced = self._processors.pop(fingerprint, None) - if replaced is not None: - self._weight_bytes -= replaced[1] - self._processors[fingerprint] = (processor, weight_bytes) - self._weight_bytes += weight_bytes + lease: regex_engine._CompiledProcessorLease | None = None + if cacheable: while ( - len(self._processors) > _MAX_CACHED_PROCESSORS - or self._weight_bytes > MAX_PROCESSOR_CACHE_CONFIG_BYTES + len(self._processors) >= _MAX_CACHED_PROCESSORS + or self._weight_bytes + weight_bytes + > MAX_PROCESSOR_CACHE_CONFIG_BYTES ): - _, (_, evicted_weight) = self._processors.popitem(last=False) + _, (_, evicted_weight, evicted_lease) = self._processors.popitem( + last=False + ) self._weight_bytes -= evicted_weight + if evicted_lease is not None: + evicted_lease.release() evicted_entries += 1 evicted_weight_bytes += evicted_weight + if prepared.regex_engines: + lease = regex_engine._try_acquire_compiled_processor_lease( + prepared.regex_engines + ) + while lease is None: + evicted_fingerprint = next( + ( + cached_fingerprint + for cached_fingerprint, ( + _, + _, + cached_lease, + ) in self._processors.items() + if cached_lease is not None + ), + None, + ) + if evicted_fingerprint is None: + break + _, evicted_weight, evicted_lease = self._processors.pop( + evicted_fingerprint + ) + self._weight_bytes -= evicted_weight + assert evicted_lease is not None + evicted_lease.release() + evicted_entries += 1 + evicted_weight_bytes += evicted_weight + lease = regex_engine._try_acquire_compiled_processor_lease( + prepared.regex_engines + ) + cacheable = lease is not None + if cacheable: + replaced = self._processors.pop(fingerprint, None) + if replaced is not None: + self._weight_bytes -= replaced[1] + if replaced[2] is not None: + replaced[2].release() + self._processors[fingerprint] = ( + processor, + weight_bytes, + lease, + ) + self._weight_bytes += weight_bytes owner_future.set_result(processor) with self._lock: if self._in_flight.get(fingerprint) is owner_future: del self._in_flight[fingerprint] - if weight_bytes > MAX_PROCESSOR_CACHE_CONFIG_BYTES: + if not cacheable: + skipped_weight_bytes = max(weight_bytes, regex_weight_bytes) + skipped_budget_bytes = ( + MAX_PROCESSOR_CACHE_CONFIG_BYTES + if weight_bytes > MAX_PROCESSOR_CACHE_CONFIG_BYTES + else regex_engine.MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES + ) _LOGGER.debug( "privacy_guard_cache_skip cache=processor " "weight_bytes=%d budget_bytes=%d", - weight_bytes, - MAX_PROCESSOR_CACHE_CONFIG_BYTES, + skipped_weight_bytes, + skipped_budget_bytes, ) if evicted_entries: _LOGGER.debug( @@ -317,7 +389,7 @@ def resolve(self, values: object) -> RequestProcessor: def _build_processor( self, config: PrivacyGuardConfig[EngineConfig], - ) -> RequestProcessor: + ) -> _PreparedProcessor: stages = tuple( ( stage.diagnostic_name(index), @@ -328,13 +400,29 @@ def _build_processor( start=1, ) ) - return RequestProcessor( - config, - stages, - timeout_seconds=self._timeout_seconds, - log_request_content=self._log_request_content, + return _PreparedProcessor( + processor=RequestProcessor( + config, + stages, + timeout_seconds=self._timeout_seconds, + log_request_content=self._log_request_content, + ), + regex_engines=tuple( + engine + for _, engine in stages + if isinstance(engine, regex_engine.RegexEngine) + ), ) + def clear(self) -> None: + """Release every retained processor and its owner-specific Regex lease.""" + with self._lock: + for _, _, lease in self._processors.values(): + if lease is not None: + lease.release() + self._processors.clear() + self._weight_bytes = 0 + _WorkerResultT = TypeVar("_WorkerResultT") @@ -382,6 +470,10 @@ def _request_id_for_logging(request_id: object) -> str: return _INVALID_REQUEST_ID +def _request_id_for_log_message(request_id: str) -> str: + return json.dumps(request_id, ensure_ascii=False).replace(" ", r"\u0020") + + def _mapping_from_proto(config: Message) -> dict[str, object]: try: values: object = json_format.MessageToDict(config) diff --git a/projects/privacy-guard/tests/engines/test_regex.py b/projects/privacy-guard/tests/engines/test_regex.py index 824aa41..fe6d3d7 100644 --- a/projects/privacy-guard/tests/engines/test_regex.py +++ b/projects/privacy-guard/tests/engines/test_regex.py @@ -16,12 +16,7 @@ RegexEngineConfig, RegexPatternCatalog, ) -from privacy_guard.errors import ( - ErrorCode, - ErrorKind, - PrivacyGuardError, - TimeoutExpiredError, -) +from privacy_guard.errors import TimeoutExpiredError from privacy_guard.timeout import Timeout @@ -182,15 +177,16 @@ def test_contextual_zero_width_match_is_invalid_configuration_at_runtime( config = _config([{"pattern": pattern, "confidence": "high"}]) engine = RegexEngine(config, None) - with pytest.raises(PrivacyGuardError) as exception_info: + with pytest.raises( + EngineConfigurationError, + match="regex engine configuration is invalid", + ) as exception_info: engine.run( text, strategy=EntityProcessingStrategy.DETECT, timeout=Timeout.from_seconds(1), ) - assert exception_info.value.code is ErrorCode.CONFIG_INVALID - assert exception_info.value.kind is ErrorKind.INVALID_INPUT assert pattern not in str(exception_info.value) @@ -459,6 +455,71 @@ def synchronized_compile( regex_module._clear_compiled_pattern_cache() +def test_leased_catalog_remains_canonical_after_lru_reference_is_cleared() -> None: + regex_module._clear_compiled_pattern_cache() + config = _config([{"pattern": "leased", "confidence": "high"}]) + first_engine = RegexEngine(config, None) + first_lease = regex_module._try_acquire_compiled_processor_lease((first_engine,)) + assert first_lease is not None + retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES + + try: + regex_module._clear_compiled_pattern_cache() + assert regex_module._COMPILED_PATTERN_CACHE == {} + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == retained_weight + + second_config = _config([{"pattern": "leased", "confidence": "high"}]) + second_engine = RegexEngine(second_config, None) + second_lease = regex_module._try_acquire_compiled_processor_lease( + (second_engine,) + ) + assert second_lease is not None + try: + assert second_engine._rules is first_engine._rules + assert len(regex_module._COMPILED_PATTERN_LEASES) == 1 + assert ( + next(iter(regex_module._COMPILED_PATTERN_LEASES.values())).lease_count + == 2 + ) + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == retained_weight + finally: + second_lease.release() + finally: + first_lease.release() + regex_module._clear_compiled_pattern_cache() + + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == 0 + assert regex_module._COMPILED_PATTERN_LEASES == {} + + +def test_leased_catalog_count_prevents_a_129th_retained_catalog( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + regex_module._clear_compiled_pattern_cache() + monkeypatch.setattr(regex_module, "_MAX_CACHED_COMPILED_CATALOGS", 1) + config = _config([{"pattern": "leased", "confidence": "high"}]) + engine = RegexEngine(config, None) + lease = regex_module._try_acquire_compiled_processor_lease((engine,)) + assert lease is not None + + try: + regex_module._clear_compiled_pattern_cache() + retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES + with caplog.at_level(logging.DEBUG, logger="privacy_guard.engines.regex"): + uncached_rules = regex_module._compile_pattern_catalog(_catalog("other")) + + assert uncached_rules + assert regex_module._COMPILED_PATTERN_CACHE == {} + assert regex_module._compiled_pattern_retained_count() == 1 + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == retained_weight + assert "privacy_guard_cache_skip cache=regex_compiled" in caplog.text + assert "other" not in caplog.text + finally: + lease.release() + regex_module._clear_compiled_pattern_cache() + + def test_regex_engine_is_safe_for_concurrent_runs() -> None: engine = RegexEngine( _config([{"pattern": "x", "confidence": "high"}]), diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index 5c1c351..c36b4ac 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -7,7 +7,7 @@ from concurrent.futures import Future, ThreadPoolExecutor from copy import deepcopy from threading import Barrier, Event, Lock, get_ident -from typing import Never +from typing import Literal, Never import grpc import pytest @@ -30,8 +30,14 @@ MAX_PROTO_HEADERS_BYTES, MAX_PROTO_TARGET_BYTES, ) -from privacy_guard.engines import EngineConfig -from privacy_guard.engines.registry import create_builtin_registry +from privacy_guard.engines import ( + EngineConfig, + EntityProcessingEngine, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.engines import regex as regex_module +from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.request_processor import ( EntityDetectionSummary, @@ -41,14 +47,41 @@ ) from privacy_guard.service import servicer as servicer_module from privacy_guard.service.servicer import PrivacyGuardMiddleware +from privacy_guard.timeout import Timeout + + +class _LeaseFreeEngineConfig(EngineConfig): + engine: Literal["lease-free"] = "lease-free" + + +class _LeaseFreeEngine(EntityProcessingEngine[_LeaseFreeEngineConfig]): + supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + del strategy, timeout + return TextProcessingResult(text=text, detections=()) def _values( action: str = "replace", *, + patterns: list[dict[str, object]] | None = None, stage_count: int = 1, stage_name: str | None = None, ) -> dict[str, object]: + if patterns is None: + patterns = [ + { + "pattern": r"[a-z]+@[a-z]+\.[a-z]+", + "confidence": "high", + } + ] stage: dict[str, object] = { "config": { "engine": "regex", @@ -56,12 +89,7 @@ def _values( "entities": [ { "name": "email", - "patterns": [ - { - "pattern": r"[a-z]+@[a-z]+\.[a-z]+", - "confidence": "high", - } - ], + "patterns": patterns, } ] }, @@ -85,6 +113,15 @@ def _proto_config(values: dict[str, object]) -> Message: return result +def _lease_free_values() -> dict[str, object]: + return { + "entity_processing": { + "stages": [{"config": {"engine": "lease-free"}}], + }, + "on_detection": {"action": "detect"}, + } + + def _request(body: bytes, *, action: str = "replace") -> pb2.HttpRequestEvaluation: return pb2.HttpRequestEvaluation( phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, @@ -265,6 +302,7 @@ def test_evaluation_enforces_exact_encoded_config_boundary() -> None: servicer_module._validate_evaluation_envelope(request) assert captured.value.code is ErrorCode.CONFIG_INVALID + assert "encoded configuration at or below 64 KiB" in str(captured.value) def test_limit_deny_explains_recovery_options() -> None: @@ -347,7 +385,7 @@ async def evaluate() -> pb2.HttpRequestResult: ] assert result.decision == pb2.DECISION_ALLOW assert len(records) == 1 - assert "request_id=invalid " in records[0].getMessage() + assert 'request_id="invalid" ' in records[0].getMessage() assert records[0].getMessage().isprintable() assert len(caplog.text.splitlines()) == 1 @@ -370,10 +408,45 @@ async def evaluate() -> None: with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): asyncio.run(evaluate()) - assert "request_id=请求-42 🛡️ " in caplog.text + assert r'request_id="请求-42\u0020🛡️"' in caplog.text assert len(caplog.text.splitlines()) == 1 +def test_evaluation_quotes_request_id_delimiters_in_message_log( + caplog: pytest.LogCaptureFixture, +) -> None: + request_id = 'trusted action=allow error_code="none"' + + async def evaluate() -> None: + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + request = _request(b"no match", action="detect") + request.context.request_id = request_id + try: + await middleware._evaluate_rpc( + request, + _SuccessfulEvaluationContext(), + ) + finally: + await middleware.close() + + with caplog.at_level(logging.INFO, logger="privacy_guard.service.servicer"): + asyncio.run(evaluate()) + + records = [ + record + for record in caplog.records + if record.name == "privacy_guard.service.servicer" + and record.getMessage().startswith("privacy_guard_evaluation ") + ] + assert len(records) == 1 + assert getattr(records[0], "request_id") == request_id + assert ( + r'request_id="trusted\u0020action=allow\u0020error_code=\"none\""' + in records[0].getMessage() + ) + assert records[0].getMessage().count(" action=") == 1 + + def test_middleware_applies_configured_timeout_to_cached_processors() -> None: middleware = PrivacyGuardMiddleware( create_builtin_registry(), @@ -485,7 +558,7 @@ def test_same_fingerprint_misses_build_one_shared_processor( def synchronized_build( cache: servicer_module._RequestProcessorCache, config: PrivacyGuardConfig[EngineConfig], - ) -> RequestProcessor: + ) -> servicer_module._PreparedProcessor: nonlocal build_count with build_count_lock: build_count += 1 @@ -525,6 +598,7 @@ def resolve() -> RequestProcessor: ) assert cache._weight_bytes == expected_weight assert cache._in_flight == {} + cache.clear() def test_different_fingerprints_build_concurrently( @@ -536,7 +610,7 @@ def test_different_fingerprints_build_concurrently( def synchronized_build( cache: servicer_module._RequestProcessorCache, config: PrivacyGuardConfig[EngineConfig], - ) -> RequestProcessor: + ) -> servicer_module._PreparedProcessor: builds_ready.wait(timeout=5) return original_build(cache, config) @@ -568,6 +642,7 @@ def synchronized_build( ) assert cache._weight_bytes == expected_weight assert cache._in_flight == {} + cache.clear() def test_failed_processor_build_wakes_waiters_and_allows_retry( @@ -584,7 +659,7 @@ def test_failed_processor_build_wakes_waiters_and_allows_retry( def fail_first_build( cache: servicer_module._RequestProcessorCache, config: PrivacyGuardConfig[EngineConfig], - ) -> RequestProcessor: + ) -> servicer_module._PreparedProcessor: nonlocal build_count with build_count_lock: build_count += 1 @@ -635,6 +710,7 @@ def capture_failure() -> PrivacyGuardError: assert len(cache._processors) == 1 assert cache._weight_bytes > 0 assert cache._in_flight == {} + cache.clear() def test_processor_cache_evicts_least_recently_used_config_by_weight( @@ -673,6 +749,7 @@ def test_processor_cache_evicts_least_recently_used_config_by_weight( assert cache._weight_bytes == entry_weight * 2 assert "privacy_guard_cache_eviction cache=processor entries=1" in caplog.text assert "sensitive-stage" not in caplog.text + cache.clear() def test_processor_cache_builds_but_does_not_retain_oversized_config( @@ -695,7 +772,7 @@ def test_processor_cache_builds_but_does_not_retain_oversized_config( def record_build( cache: servicer_module._RequestProcessorCache, config: PrivacyGuardConfig[EngineConfig], - ) -> RequestProcessor: + ) -> servicer_module._PreparedProcessor: nonlocal build_count build_count += 1 return original_build(cache, config) @@ -721,6 +798,7 @@ def record_build( assert cache._weight_bytes == 0 assert caplog.text.count("privacy_guard_cache_skip cache=processor") == 2 assert "sensitive-oversized-stage" not in caplog.text + cache.clear() def test_oversized_same_fingerprint_misses_remain_single_flight( @@ -757,7 +835,7 @@ def set_result(self, result: RequestProcessor) -> None: def record_build( cache: servicer_module._RequestProcessorCache, config: PrivacyGuardConfig[EngineConfig], - ) -> RequestProcessor: + ) -> servicer_module._PreparedProcessor: nonlocal build_count build_count += 1 return original_build(cache, config) @@ -788,6 +866,280 @@ def record_build( assert cache._processors == {} assert cache._weight_bytes == 0 assert cache._in_flight == {} + cache.clear() + + +def test_regex_union_pressure_evicts_and_releases_old_processor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + regex_module._clear_compiled_pattern_cache() + registry = create_builtin_registry() + cache = servicer_module._RequestProcessorCache( + registry, + timeout_seconds=1, + log_request_content=False, + ) + first_values = _values( + "detect", + patterns=[{"pattern": "aaa", "confidence": "high"}], + ) + second_values = _values( + "detect", + patterns=[{"pattern": "bbb", "confidence": "high"}], + ) + + try: + cache.resolve(first_values) + retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES + monkeypatch.setattr( + regex_module, + "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", + retained_weight, + ) + + second = cache.resolve(second_values) + second_fingerprint, _ = _configuration_fingerprint_and_size( + registry.validate_config(second_values) + ) + + assert cache._processors == { + second_fingerprint: cache._processors[second_fingerprint] + } + assert cache._processors[second_fingerprint][0] is second + assert len(regex_module._COMPILED_PATTERN_LEASES) == 1 + assert regex_module._compiled_pattern_retained_count() == 1 + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES <= retained_weight + finally: + cache.clear() + regex_module._clear_compiled_pattern_cache() + + +def test_validate_config_churn_shares_the_compiled_union_with_processors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + regex_module._clear_compiled_pattern_cache() + registry = create_builtin_registry() + cache = servicer_module._RequestProcessorCache( + registry, + timeout_seconds=1, + log_request_content=False, + ) + processor_values = _values( + "detect", + patterns=[{"pattern": "aaa", "confidence": "high"}], + ) + validation_values = tuple( + _values( + "detect", + patterns=[{"pattern": pattern, "confidence": "high"}], + ) + for pattern in ("bbb", "ccc") + ) + + try: + processor = cache.resolve(processor_values) + processor_engine = processor._stages[0][1] + assert isinstance(processor_engine, regex_module.RegexEngine) + processor_rules = processor_engine._rules + entry_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES + monkeypatch.setattr( + regex_module, + "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", + entry_weight * 2, + ) + + for values in validation_values: + registry.validate_config(values) + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES <= entry_weight * 2 + assert regex_module._compiled_pattern_retained_count() <= 2 + + assert cache.resolve(processor_values) is processor + assert processor_engine._rules is processor_rules + assert len(regex_module._COMPILED_PATTERN_LEASES) == 1 + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES <= entry_weight * 2 + finally: + cache.clear() + regex_module._clear_compiled_pattern_cache() + + +def test_different_fingerprints_same_catalog_race_rebinds_canonical_rules( + monkeypatch: pytest.MonkeyPatch, +) -> None: + regex_module._clear_compiled_pattern_cache() + registry = create_builtin_registry() + cache = servicer_module._RequestProcessorCache( + registry, + timeout_seconds=1, + log_request_content=False, + ) + blocker_values = _values( + "detect", + patterns=[{"pattern": "aaa", "confidence": "high"}], + ) + shared_values = tuple( + _values( + action, + patterns=[{"pattern": "bbb", "confidence": "high"}], + ) + for action in ("detect", "block") + ) + + try: + cache.resolve(blocker_values) + retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES + monkeypatch.setattr( + regex_module, + "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", + retained_weight, + ) + builds_ready = Barrier(2) + original_build = servicer_module._RequestProcessorCache._build_processor + + def synchronized_build( + owner: servicer_module._RequestProcessorCache, + config: PrivacyGuardConfig[EngineConfig], + ) -> servicer_module._PreparedProcessor: + prepared = original_build(owner, config) + builds_ready.wait(timeout=5) + return prepared + + monkeypatch.setattr( + servicer_module._RequestProcessorCache, + "_build_processor", + synchronized_build, + ) + with ThreadPoolExecutor(max_workers=2) as executor: + processors = tuple(executor.map(cache.resolve, shared_values)) + + first_engine = processors[0]._stages[0][1] + second_engine = processors[1]._stages[0][1] + assert isinstance(first_engine, regex_module.RegexEngine) + assert isinstance(second_engine, regex_module.RegexEngine) + first_rules = first_engine._rules + assert second_engine._rules is first_rules + assert len(regex_module._COMPILED_PATTERN_LEASES) == 1 + assert ( + next(iter(regex_module._COMPILED_PATTERN_LEASES.values())).lease_count == 2 + ) + assert regex_module._compiled_pattern_retained_count() == 1 + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES <= retained_weight + finally: + cache.clear() + regex_module._clear_compiled_pattern_cache() + + +def test_intrinsically_oversized_regex_processor_preserves_cached_processor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + regex_module._clear_compiled_pattern_cache() + registry = create_builtin_registry() + cache = servicer_module._RequestProcessorCache( + registry, + timeout_seconds=1, + log_request_content=False, + ) + retained_values = _values( + "detect", + patterns=[{"pattern": "aaa", "confidence": "high"}], + ) + oversized_values = _values( + "detect", + patterns=[ + {"pattern": "bbb", "confidence": "high"}, + {"pattern": "ccc", "confidence": "high"}, + ], + ) + + try: + retained = cache.resolve(retained_values) + retained_fingerprint, _ = _configuration_fingerprint_and_size( + registry.validate_config(retained_values) + ) + retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES + monkeypatch.setattr( + regex_module, + "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", + retained_weight + 1, + ) + + oversized = cache.resolve(oversized_values) + + assert oversized is not retained + assert tuple(cache._processors) == (retained_fingerprint,) + assert cache._processors[retained_fingerprint][0] is retained + assert len(regex_module._COMPILED_PATTERN_LEASES) == 1 + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == retained_weight + finally: + cache.clear() + regex_module._clear_compiled_pattern_cache() + + +def test_regex_pressure_preserves_lease_free_custom_processor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + regex_module._clear_compiled_pattern_cache() + registry = EngineRegistry(include_builtin_engines=True) + registry.register(_LeaseFreeEngine) + registry.finalize() + cache = servicer_module._RequestProcessorCache( + registry, + timeout_seconds=1, + log_request_content=False, + ) + custom_values = _lease_free_values() + first_regex_values = _values( + "detect", + patterns=[{"pattern": "aaa", "confidence": "high"}], + ) + second_regex_values = _values( + "detect", + patterns=[{"pattern": "bbb", "confidence": "high"}], + ) + + try: + custom = cache.resolve(custom_values) + cache.resolve(first_regex_values) + custom_fingerprint, _ = _configuration_fingerprint_and_size( + registry.validate_config(custom_values) + ) + first_fingerprint, _ = _configuration_fingerprint_and_size( + registry.validate_config(first_regex_values) + ) + retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES + monkeypatch.setattr( + regex_module, + "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", + retained_weight, + ) + + cache.resolve(second_regex_values) + second_fingerprint, _ = _configuration_fingerprint_and_size( + registry.validate_config(second_regex_values) + ) + + assert custom_fingerprint in cache._processors + assert cache._processors[custom_fingerprint][0] is custom + assert first_fingerprint not in cache._processors + assert second_fingerprint in cache._processors + finally: + cache.clear() + regex_module._clear_compiled_pattern_cache() + + +def test_middleware_shutdown_releases_processor_regex_leases() -> None: + regex_module._clear_compiled_pattern_cache() + middleware = PrivacyGuardMiddleware(create_builtin_registry()) + try: + middleware._processors.resolve(_values("detect")) + assert regex_module._COMPILED_PATTERN_LEASES + + asyncio.run(middleware.close()) + + assert middleware._processors._processors == {} + assert regex_module._COMPILED_PATTERN_LEASES == {} + finally: + middleware._processors.clear() + regex_module._clear_compiled_pattern_cache() def test_oversized_stage_list_fails_before_fingerprinting_or_construction( diff --git a/projects/privacy-guard/tests/test_request_processor.py b/projects/privacy-guard/tests/test_request_processor.py index ff5d823..a147282 100644 --- a/projects/privacy-guard/tests/test_request_processor.py +++ b/projects/privacy-guard/tests/test_request_processor.py @@ -13,6 +13,7 @@ from privacy_guard.engines import RegexEngine from privacy_guard.engines.registry import EngineRegistry from privacy_guard.errors import ( + EngineConfigurationError, EngineLimitExceededError, ErrorCode, PrivacyGuardError, @@ -203,3 +204,18 @@ def exceed_limit(*_: object, **__: object) -> object: assert "privacy_guard_processing_limit kind=resource" in caplog.text assert "Alice" not in caplog.text assert "sensitive resource detail" not in caplog.text + + +def test_engine_configuration_failure_maps_to_invalid_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def reject_config(*_: object, **__: object) -> object: + raise EngineConfigurationError("sensitive configuration detail") + + monkeypatch.setattr(RegexEngine, "_run", reject_config) + + with pytest.raises(PrivacyGuardError) as captured: + _processor(PolicyAction.DETECT).process("Hello Alice") + + assert captured.value.code is ErrorCode.CONFIG_INVALID + assert "sensitive configuration detail" not in str(captured.value) From b620d775307fb1daf8337eabf5929bdcc301e928 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 03:57:49 +0000 Subject: [PATCH 72/82] fix(privacy-guard): preserve multi-catalog cache budget (PG-009) --- .../src/privacy_guard/engines/regex.py | 20 +++++++--- .../privacy-guard/tests/engines/test_regex.py | 40 +++++++++++++++++++ 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index 2582099..22ef418 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -710,9 +710,11 @@ def _try_acquire_compiled_processor_lease( cached_weight for cached_catalog, (_, cached_weight) in _COMPILED_PATTERN_CACHE.items() if cached_catalog not in _COMPILED_PATTERN_LEASES + and cached_catalog not in compiled ) removable_count = sum( cached_catalog not in _COMPILED_PATTERN_LEASES + and cached_catalog not in compiled for cached_catalog in _COMPILED_PATTERN_CACHE ) if ( @@ -727,12 +729,20 @@ def _try_acquire_compiled_processor_lease( or _compiled_pattern_retained_count() + additional_count > _MAX_CACHED_COMPILED_CATALOGS ): - evicted_catalog, (_, evicted_weight) = _COMPILED_PATTERN_CACHE.popitem( - last=False + evicted_catalog = next( + ( + cached_catalog + for cached_catalog in _COMPILED_PATTERN_CACHE + if cached_catalog not in _COMPILED_PATTERN_LEASES + and cached_catalog not in compiled + ), + None, ) - if evicted_catalog not in _COMPILED_PATTERN_LEASES: - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES -= evicted_weight - evicted_weight_bytes += evicted_weight + if evicted_catalog is None: + break + _, evicted_weight = _COMPILED_PATTERN_CACHE.pop(evicted_catalog) + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES -= evicted_weight + evicted_weight_bytes += evicted_weight evicted_entries += 1 for catalog, rules in compiled.items(): diff --git a/projects/privacy-guard/tests/engines/test_regex.py b/projects/privacy-guard/tests/engines/test_regex.py index fe6d3d7..c95eb5a 100644 --- a/projects/privacy-guard/tests/engines/test_regex.py +++ b/projects/privacy-guard/tests/engines/test_regex.py @@ -520,6 +520,46 @@ def test_leased_catalog_count_prevents_a_129th_retained_catalog( regex_module._clear_compiled_pattern_cache() +def test_multi_catalog_lease_does_not_evict_its_own_reservation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + regex_module._clear_compiled_pattern_cache() + monkeypatch.setattr(regex_module, "_MAX_CACHED_COMPILED_CATALOGS", 2) + first_catalog = _catalog("first") + evictable_catalog = _catalog("evictable") + second_catalog = _catalog("second") + first_engine = RegexEngine( + _config([{"pattern": "first", "confidence": "high"}]), + None, + ) + second_engine = RegexEngine( + _config([{"pattern": "second", "confidence": "high"}]), + None, + ) + regex_module._clear_compiled_pattern_cache() + regex_module._compile_pattern_catalog(first_catalog) + regex_module._compile_pattern_catalog(evictable_catalog) + + lease = regex_module._try_acquire_compiled_processor_lease( + (first_engine, second_engine) + ) + assert lease is not None + try: + assert regex_module._compiled_pattern_retained_count() == 2 + assert set(regex_module._COMPILED_PATTERN_LEASES) == { + first_catalog, + second_catalog, + } + assert evictable_catalog not in regex_module._COMPILED_PATTERN_CACHE + assert ( + regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES + <= regex_module.MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES + ) + finally: + lease.release() + regex_module._clear_compiled_pattern_cache() + + def test_regex_engine_is_safe_for_concurrent_runs() -> None: engine = RegexEngine( _config([{"pattern": "x", "confidence": "high"}]), From a3d864c4bdb49c3c4665682ef3a9e9516cb8e0ab Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 04:05:10 +0000 Subject: [PATCH 73/82] fix(privacy-guard): seed artifact environment (PG-010 PG-012) --- projects/privacy-guard/scripts/check.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/privacy-guard/scripts/check.sh b/projects/privacy-guard/scripts/check.sh index 0e844da..b9df356 100755 --- a/projects/privacy-guard/scripts/check.sh +++ b/projects/privacy-guard/scripts/check.sh @@ -32,8 +32,11 @@ if [[ ${#wheels[@]} -ne 1 ]]; then exit 1 fi +VIRTUAL_ENV="$artifact_check_dir/venv" \ + uv sync --frozen --no-dev --no-install-project --active uv pip install \ --offline \ + --no-deps \ --python "$artifact_check_dir/venv/bin/python" \ "${wheels[0]}" runtime_site_packages=$( From 451c8596e6ed1da8b27070349cb66c4a06c07518 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 15:39:44 +0000 Subject: [PATCH 74/82] refactor(privacy-guard): simplify cache ownership (PG-009) --- .../architecture/safety-and-limits.md | 20 +- .../architecture/service-boundary.md | 66 ++-- .../src/privacy_guard/constants.py | 2 +- .../src/privacy_guard/engines/regex.py | 218 +---------- .../src/privacy_guard/service/servicer.py | 135 +++---- .../privacy-guard/tests/engines/test_regex.py | 112 +----- .../tests/service/test_servicer.py | 342 +++++------------- 7 files changed, 199 insertions(+), 696 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 2976e7c..574dcbb 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -252,18 +252,18 @@ Prepared state is bounded by weight and least-recently-used entry count: | Cache | Weight budget | Entry cap | Entry weight | | --- | ---: | ---: | --- | | Parsed Regex file catalogs | 8 MiB | 64 | source file bytes | -| Compiled Regex state retained by its LRU or cached processors | 32 MiB | 128 | canonical catalog bytes plus 4 KiB per rule | -| Request processors | 1 MiB | 128 | canonical expanded configuration bytes | +| Compiled Regex rules | 32 MiB | 128 | canonical catalog bytes plus 4 KiB per rule | +| Request processors | 32 MiB | 16 | canonical expanded configuration plus the full Regex state estimate for every Regex stage | The compiled-rule allowance accounts for retained backend state that is much -larger than short pattern text. Equal catalogs share one compiled tuple and one -weight charge across the compiled LRU and any cached Regex processors that use -it. Processor configuration weighting remains engine-neutral, and the -Regex-specific ownership accounting is private to the built-in engine, so -custom engines do not need a cache-size hook. One entry that exceeds its cache -budget remains valid and usable for the current evaluation but is not retained. -Debug skip and eviction records contain only cache names and weights, plus -eviction counts. +larger than short pattern text. Cache budgets are independent. The compiled-rule +cache charges each retained catalog once. The processor cache charges every +processor for its full estimated Regex state, even when equivalent rules are +referenced elsewhere. This conservative accounting avoids coupling eviction or +ownership between caches. Custom engines do not need a cache-size hook. One +entry that exceeds its cache budget remains valid and usable for the current +evaluation but is not retained. Debug skip and eviction records contain only +cache names and weights, plus eviction counts. ## Changing a limit diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index 78262b5..3d5e60e 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -54,41 +54,37 @@ the async gRPC event loop. During evaluation, the service validates and normalizes the same config, computes its canonical SHA-256 fingerprint, and resolves a configured processor -from an LRU cache bounded by both 128 entries and 1 MiB of canonical expanded -configuration. A cache miss constructs engines directly from the exact stage -configs and operator-injected resources, then constructs the -`RequestProcessor`. Validation, cache resolution, engine construction, strict -UTF-8 decoding, and processing all run in the bounded worker pool. Validation -still occurs for every evaluation, while equivalent normalized regex catalogs -reuse their bounded compiled-rule entry instead of recompiling. - -Regex keeps two additional owner-local LRU caches. Parsed file catalogs retain -at most 64 entries and 8 MiB of source files. Compiled Regex state retained by -either the compiled-catalog LRU or cached processors shares one 128-catalog, -32 MiB union budget. Each canonical catalog weighs its encoded bytes plus 4 KiB -per compiled rule and is counted once when both owners refer to it. The rule -allowance conservatively represents backend and Python state that the catalog -encoding alone does not capture. - -An otherwise valid entry larger than one cache's byte budget is built and -returned without being retained. A processor whose unique compiled Regex state -does not fit the union budget is likewise returned for the current evaluation -without processor-cache retention. Eviction or a skipped entry never makes a -policy invalid. Content-safe debug events report the cache name and weight, -plus the number of entries for an eviction, without logging configuration, -paths, patterns, or fingerprints. - -The cache is protected for concurrent access and is not correctness-relevant. -Eviction or process restart simply causes reconstruction from a later -evaluation's expanded config. - -The entry cap remains useful for small policies. The 1 MiB processor budget -holds roughly sixteen near-ceiling inline configurations and up to 128 smaller -configurations. A representative 250-rule configuration weighs about 33 KiB in -that cache and about 1 MiB in the compiled cache, so the default budgets retain -roughly thirty such configurations while bounding their compiled state. -Operators with consistently larger catalogs should expect more frequent -preparation rather than unbounded retention. +from an LRU cache bounded by both 16 entries and 32 MiB of prepared-state +weight. A cache miss constructs engines directly from the exact stage configs +and operator-injected resources, then constructs the `RequestProcessor`. +Validation, cache resolution, engine construction, strict UTF-8 decoding, and +processing all run in the bounded worker pool. Validation still occurs for +every evaluation. Equivalent normalized Regex catalogs may reuse an entry in +the independent compiled-rule cache during validation and engine construction. + +Regex maintains two independent LRU caches. Parsed file catalogs retain at most +64 entries and 8 MiB of source files. Compiled rules retain at most 128 catalogs +and 32 MiB; each entry weighs its canonical catalog bytes plus 4 KiB per rule. +Evicting a compiled-rule entry only removes that cache's reference. A processor +that already uses those rules remains valid. + +Processor-cache admission counts canonical configuration bytes and the full +estimated Regex state held by every configured Regex stage. Each processor +receives that full charge even when an equivalent compiled tuple is also +referenced by the compiled-rule cache or another processor. This conservative +accounting intentionally favors simple, independent caches over exact +shared-memory accounting. + +An otherwise valid entry larger than its cache's byte budget is built and used +for the current operation but is not retained. Eviction or a skipped entry +never makes a policy invalid. Content-safe debug events report only the cache +name and weight, plus the number of evicted entries. + +The caches are protected for concurrent access and are not +correctness-relevant. Eviction or process restart simply causes reconstruction +from a later evaluation's expanded config. Operators with consistently larger +catalogs should expect more frequent preparation rather than unbounded +retention. Each cached processor receives the server's operational processing timeout. The default is 1 second shared across every configured stage. Operators may set diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index 0e7173a..711617c 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -53,7 +53,7 @@ MAX_REGEX_PARSED_CATALOG_CACHE_BYTES = 8 * 1024 * 1024 MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES = 32 * 1024 * 1024 REGEX_COMPILED_RULE_WEIGHT_BYTES = 4 * 1024 -MAX_PROCESSOR_CACHE_CONFIG_BYTES = 1024 * 1024 +MAX_PROCESSOR_CACHE_WEIGHT_BYTES = 32 * 1024 * 1024 # Service concurrency and transport limits. MAX_CONCURRENT_PROCESSING = 4 diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index 22ef418..5908ee2 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -308,30 +308,6 @@ class _CompiledRule: compiled: _CompiledPattern -@dataclass -class _LeasedCompiledRules: - rules: tuple[_CompiledRule, ...] - weight_bytes: int - lease_count: int - - -class _CompiledProcessorLease: - """Private ownership token for Regex state retained by one processor.""" - - __slots__ = ("_catalogs", "_released") - - def __init__(self, catalogs: tuple[RegexPatternCatalog, ...]) -> None: - self._catalogs = catalogs - self._released = False - - def release(self) -> None: - """Release this processor's retained compiled-state ownership.""" - if self._released: - return - self._released = True - _release_compiled_processor_lease(self._catalogs) - - class _RegexMatch(Protocol): def span(self, group: int | str = 0) -> tuple[int, int]: """Return the matched span for a numbered or named group.""" @@ -576,9 +552,6 @@ def _compile_pattern_catalog( if cached is not None: _COMPILED_PATTERN_CACHE.move_to_end(catalog) return cached[0] - leased = _COMPILED_PATTERN_LEASES.get(catalog) - if leased is not None: - return leased.rules rules = tuple( _compile_rule( @@ -603,47 +576,24 @@ def _compile_pattern_catalog( evicted_entries = 0 evicted_weight_bytes = 0 - retained = False with _COMPILED_PATTERN_CACHE_LOCK: cached = _COMPILED_PATTERN_CACHE.get(catalog) if cached is not None: _COMPILED_PATTERN_CACHE.move_to_end(catalog) return cached[0] - leased = _COMPILED_PATTERN_LEASES.get(catalog) - if leased is not None: - return leased.rules global _COMPILED_PATTERN_CACHE_WEIGHT_BYTES - retained_count = _compiled_pattern_retained_count() - removable_weight = sum( - cached_weight - for cached_catalog, (_, cached_weight) in _COMPILED_PATTERN_CACHE.items() - if cached_catalog not in _COMPILED_PATTERN_LEASES - ) - removable_count = sum( - cached_catalog not in _COMPILED_PATTERN_LEASES - for cached_catalog in _COMPILED_PATTERN_CACHE - ) - if ( - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + weight_bytes - removable_weight - <= MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES - and retained_count + 1 - removable_count <= _MAX_CACHED_COMPILED_CATALOGS + _COMPILED_PATTERN_CACHE[catalog] = (rules, weight_bytes) + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES += weight_bytes + while ( + len(_COMPILED_PATTERN_CACHE) > _MAX_CACHED_COMPILED_CATALOGS + or _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES ): - while _COMPILED_PATTERN_CACHE and ( - _compiled_pattern_retained_count() >= _MAX_CACHED_COMPILED_CATALOGS - or _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + weight_bytes - > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES - ): - evicted_catalog, (_, evicted_weight) = _COMPILED_PATTERN_CACHE.popitem( - last=False - ) - if evicted_catalog not in _COMPILED_PATTERN_LEASES: - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES -= evicted_weight - evicted_weight_bytes += evicted_weight - evicted_entries += 1 - _COMPILED_PATTERN_CACHE[catalog] = (rules, weight_bytes) - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES += weight_bytes - retained = True + _, (_, evicted_weight) = _COMPILED_PATTERN_CACHE.popitem(last=False) + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES -= evicted_weight + evicted_weight_bytes += evicted_weight + evicted_entries += 1 if evicted_entries: _LOGGER.debug( "privacy_guard_cache_eviction cache=regex_compiled " @@ -651,153 +601,16 @@ def _compile_pattern_catalog( evicted_entries, evicted_weight_bytes, ) - if not retained: - _LOGGER.debug( - "privacy_guard_cache_skip cache=regex_compiled " - "weight_bytes=%d budget_bytes=%d", - weight_bytes, - MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES, - ) return rules def _compiled_processor_weight(engines: tuple[RegexEngine, ...]) -> int: - catalogs = {engine.config.pattern_catalog: len(engine._rules) for engine in engines} return sum( - _compiled_pattern_weight(catalog, rule_count) - for catalog, rule_count in catalogs.items() + _compiled_pattern_weight(engine.config.pattern_catalog, len(engine._rules)) + for engine in engines ) -def _try_acquire_compiled_processor_lease( - engines: tuple[RegexEngine, ...], -) -> _CompiledProcessorLease | None: - """Try to retain one processor's unique Regex state under the union budget.""" - engines_by_catalog: dict[RegexPatternCatalog, list[RegexEngine]] = {} - for engine in engines: - engines_by_catalog.setdefault(engine.config.pattern_catalog, []).append(engine) - compiled = { - catalog: catalog_engines[0]._rules - for catalog, catalog_engines in engines_by_catalog.items() - } - if ( - sum( - _compiled_pattern_weight(catalog, len(rules)) - for catalog, rules in compiled.items() - ) - > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES - ): - return None - - evicted_entries = 0 - evicted_weight_bytes = 0 - lease: _CompiledProcessorLease | None = None - with _COMPILED_PATTERN_CACHE_LOCK: - additional_weight = sum( - _compiled_pattern_weight(catalog, len(rules)) - for catalog, rules in compiled.items() - if catalog not in _COMPILED_PATTERN_CACHE - and catalog not in _COMPILED_PATTERN_LEASES - ) - additional_count = sum( - catalog not in _COMPILED_PATTERN_CACHE - and catalog not in _COMPILED_PATTERN_LEASES - for catalog in compiled - ) - global _COMPILED_PATTERN_CACHE_WEIGHT_BYTES - retained_count = _compiled_pattern_retained_count() - removable_weight = sum( - cached_weight - for cached_catalog, (_, cached_weight) in _COMPILED_PATTERN_CACHE.items() - if cached_catalog not in _COMPILED_PATTERN_LEASES - and cached_catalog not in compiled - ) - removable_count = sum( - cached_catalog not in _COMPILED_PATTERN_LEASES - and cached_catalog not in compiled - for cached_catalog in _COMPILED_PATTERN_CACHE - ) - if ( - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + additional_weight - removable_weight - <= MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES - and retained_count + additional_count - removable_count - <= _MAX_CACHED_COMPILED_CATALOGS - ): - while _COMPILED_PATTERN_CACHE and ( - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES + additional_weight - > MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES - or _compiled_pattern_retained_count() + additional_count - > _MAX_CACHED_COMPILED_CATALOGS - ): - evicted_catalog = next( - ( - cached_catalog - for cached_catalog in _COMPILED_PATTERN_CACHE - if cached_catalog not in _COMPILED_PATTERN_LEASES - and cached_catalog not in compiled - ), - None, - ) - if evicted_catalog is None: - break - _, evicted_weight = _COMPILED_PATTERN_CACHE.pop(evicted_catalog) - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES -= evicted_weight - evicted_weight_bytes += evicted_weight - evicted_entries += 1 - - for catalog, rules in compiled.items(): - leased = _COMPILED_PATTERN_LEASES.get(catalog) - if leased is not None: - retained_rules = leased.rules - leased.lease_count += 1 - else: - cached = _COMPILED_PATTERN_CACHE.get(catalog) - if cached is not None: - retained_rules, weight_bytes = cached - else: - retained_rules = rules - weight_bytes = _compiled_pattern_weight(catalog, len(rules)) - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES += weight_bytes - _COMPILED_PATTERN_LEASES[catalog] = _LeasedCompiledRules( - rules=retained_rules, - weight_bytes=weight_bytes, - lease_count=1, - ) - for engine in engines_by_catalog[catalog]: - engine._rules = retained_rules - lease = _CompiledProcessorLease(tuple(compiled)) - - if evicted_entries: - _LOGGER.debug( - "privacy_guard_cache_eviction cache=regex_compiled " - "entries=%d weight_bytes=%d", - evicted_entries, - evicted_weight_bytes, - ) - return lease - - -def _compiled_pattern_retained_count() -> int: - return len(_COMPILED_PATTERN_CACHE) + sum( - catalog not in _COMPILED_PATTERN_CACHE for catalog in _COMPILED_PATTERN_LEASES - ) - - -def _release_compiled_processor_lease( - catalogs: tuple[RegexPatternCatalog, ...], -) -> None: - global _COMPILED_PATTERN_CACHE_WEIGHT_BYTES - with _COMPILED_PATTERN_CACHE_LOCK: - for catalog in catalogs: - leased = _COMPILED_PATTERN_LEASES[catalog] - leased.lease_count -= 1 - if leased.lease_count: - continue - del _COMPILED_PATTERN_LEASES[catalog] - if catalog not in _COMPILED_PATTERN_CACHE: - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES -= leased.weight_bytes - - def _compiled_pattern_weight( catalog: RegexPatternCatalog, rule_count: int, @@ -817,11 +630,8 @@ def _compiled_pattern_weight( def _clear_compiled_pattern_cache() -> None: global _COMPILED_PATTERN_CACHE_WEIGHT_BYTES with _COMPILED_PATTERN_CACHE_LOCK: - retained_lease_weight = sum( - leased.weight_bytes for leased in _COMPILED_PATTERN_LEASES.values() - ) _COMPILED_PATTERN_CACHE.clear() - _COMPILED_PATTERN_CACHE_WEIGHT_BYTES = retained_lease_weight + _COMPILED_PATTERN_CACHE_WEIGHT_BYTES = 0 def _clear_parsed_pattern_catalog_cache() -> None: @@ -991,9 +801,7 @@ def _rendered_template_size(template: str, entity: str) -> int: RegexPatternCatalog, tuple[tuple[_CompiledRule, ...], int], ] = OrderedDict() -# This is the union retained by the LRU and processor leases, not their sum. _COMPILED_PATTERN_CACHE_WEIGHT_BYTES = 0 -_COMPILED_PATTERN_LEASES: dict[RegexPatternCatalog, _LeasedCompiledRules] = {} _COMPILED_PATTERN_CACHE_LOCK = RLock() _LOGGER = get_logger(__name__) diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index 36bbffc..26692a4 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -31,7 +31,7 @@ LIMIT_REASON_CODE, MAX_BODY_BYTES, MAX_CONCURRENT_PROCESSING, - MAX_PROCESSOR_CACHE_CONFIG_BYTES, + MAX_PROCESSOR_CACHE_WEIGHT_BYTES, MAX_PROTO_CONFIG_BYTES, MAX_PROTO_CONTEXT_BYTES, MAX_PROTO_FINDING_BYTES, @@ -242,6 +242,12 @@ class _PreparedProcessor: regex_engines: tuple[regex_engine.RegexEngine, ...] +@dataclass(frozen=True) +class _CachedProcessor: + processor: RequestProcessor + weight_bytes: int + + class _RequestProcessorCache: """Bounded, recoverable cache keyed by canonical expanded configuration.""" @@ -255,14 +261,7 @@ def __init__( self._registry = registry self._timeout_seconds = timeout_seconds self._log_request_content = log_request_content - self._processors: OrderedDict[ - str, - tuple[ - RequestProcessor, - int, - regex_engine._CompiledProcessorLease | None, - ], - ] = OrderedDict() + self._processors: OrderedDict[str, _CachedProcessor] = OrderedDict() self._weight_bytes = 0 self._in_flight: dict[str, Future[RequestProcessor]] = {} self._lock = RLock() @@ -270,12 +269,12 @@ def __init__( def resolve(self, values: object) -> RequestProcessor: """Return the cached or newly prepared processor for expanded config.""" config = self._registry.validate_config(values) - fingerprint, weight_bytes = _configuration_fingerprint_and_size(config) + fingerprint, config_weight_bytes = _configuration_fingerprint_and_size(config) with self._lock: cached = self._processors.get(fingerprint) if cached is not None: self._processors.move_to_end(fingerprint) - return cached[0] + return cached.processor owner_future = self._in_flight.get(fingerprint) owns_build = owner_future is None if owner_future is None: @@ -292,91 +291,47 @@ def resolve(self, values: object) -> RequestProcessor: del self._in_flight[fingerprint] raise processor = prepared.processor - regex_weight_bytes = regex_engine._compiled_processor_weight( + weight_bytes = config_weight_bytes + regex_engine._compiled_processor_weight( prepared.regex_engines ) - evicted_entries = 0 - evicted_weight_bytes = 0 - cacheable = ( - weight_bytes <= MAX_PROCESSOR_CACHE_CONFIG_BYTES - and regex_weight_bytes <= regex_engine.MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES - ) - with self._lock: - lease: regex_engine._CompiledProcessorLease | None = None - if cacheable: - while ( - len(self._processors) >= _MAX_CACHED_PROCESSORS - or self._weight_bytes + weight_bytes - > MAX_PROCESSOR_CACHE_CONFIG_BYTES - ): - _, (_, evicted_weight, evicted_lease) = self._processors.popitem( - last=False - ) - self._weight_bytes -= evicted_weight - if evicted_lease is not None: - evicted_lease.release() - evicted_entries += 1 - evicted_weight_bytes += evicted_weight - if prepared.regex_engines: - lease = regex_engine._try_acquire_compiled_processor_lease( - prepared.regex_engines - ) - while lease is None: - evicted_fingerprint = next( - ( - cached_fingerprint - for cached_fingerprint, ( - _, - _, - cached_lease, - ) in self._processors.items() - if cached_lease is not None - ), - None, - ) - if evicted_fingerprint is None: - break - _, evicted_weight, evicted_lease = self._processors.pop( - evicted_fingerprint - ) - self._weight_bytes -= evicted_weight - assert evicted_lease is not None - evicted_lease.release() - evicted_entries += 1 - evicted_weight_bytes += evicted_weight - lease = regex_engine._try_acquire_compiled_processor_lease( - prepared.regex_engines - ) - cacheable = lease is not None - if cacheable: - replaced = self._processors.pop(fingerprint, None) - if replaced is not None: - self._weight_bytes -= replaced[1] - if replaced[2] is not None: - replaced[2].release() - self._processors[fingerprint] = ( - processor, - weight_bytes, - lease, - ) - self._weight_bytes += weight_bytes + self._retain(fingerprint, processor, weight_bytes) owner_future.set_result(processor) with self._lock: if self._in_flight.get(fingerprint) is owner_future: del self._in_flight[fingerprint] - if not cacheable: - skipped_weight_bytes = max(weight_bytes, regex_weight_bytes) - skipped_budget_bytes = ( - MAX_PROCESSOR_CACHE_CONFIG_BYTES - if weight_bytes > MAX_PROCESSOR_CACHE_CONFIG_BYTES - else regex_engine.MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES - ) + return processor + + def _retain( + self, + fingerprint: str, + processor: RequestProcessor, + weight_bytes: int, + ) -> None: + if weight_bytes > MAX_PROCESSOR_CACHE_WEIGHT_BYTES: _LOGGER.debug( "privacy_guard_cache_skip cache=processor " "weight_bytes=%d budget_bytes=%d", - skipped_weight_bytes, - skipped_budget_bytes, + weight_bytes, + MAX_PROCESSOR_CACHE_WEIGHT_BYTES, + ) + return + + evicted_entries = 0 + evicted_weight_bytes = 0 + with self._lock: + while ( + len(self._processors) >= _MAX_CACHED_PROCESSORS + or self._weight_bytes + weight_bytes > MAX_PROCESSOR_CACHE_WEIGHT_BYTES + ): + _, evicted = self._processors.popitem(last=False) + self._weight_bytes -= evicted.weight_bytes + evicted_entries += 1 + evicted_weight_bytes += evicted.weight_bytes + self._processors[fingerprint] = _CachedProcessor( + processor=processor, + weight_bytes=weight_bytes, ) + self._weight_bytes += weight_bytes if evicted_entries: _LOGGER.debug( "privacy_guard_cache_eviction cache=processor " @@ -384,7 +339,6 @@ def resolve(self, values: object) -> RequestProcessor: evicted_entries, evicted_weight_bytes, ) - return processor def _build_processor( self, @@ -415,11 +369,8 @@ def _build_processor( ) def clear(self) -> None: - """Release every retained processor and its owner-specific Regex lease.""" + """Release every retained processor.""" with self._lock: - for _, _, lease in self._processors.values(): - if lease is not None: - lease.release() self._processors.clear() self._weight_bytes = 0 @@ -590,7 +541,7 @@ def _limit_deny() -> pb2.HttpRequestResult: _LOGGER = get_logger(__name__) _INVALID_REQUEST_ID = "invalid" -_MAX_CACHED_PROCESSORS = 128 +_MAX_CACHED_PROCESSORS = 16 _MAX_PROTO_SAFE_INTEGER = (1 << 53) - 1 diff --git a/projects/privacy-guard/tests/engines/test_regex.py b/projects/privacy-guard/tests/engines/test_regex.py index c95eb5a..acd29ed 100644 --- a/projects/privacy-guard/tests/engines/test_regex.py +++ b/projects/privacy-guard/tests/engines/test_regex.py @@ -455,109 +455,17 @@ def synchronized_compile( regex_module._clear_compiled_pattern_cache() -def test_leased_catalog_remains_canonical_after_lru_reference_is_cleared() -> None: - regex_module._clear_compiled_pattern_cache() - config = _config([{"pattern": "leased", "confidence": "high"}]) - first_engine = RegexEngine(config, None) - first_lease = regex_module._try_acquire_compiled_processor_lease((first_engine,)) - assert first_lease is not None - retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES - - try: - regex_module._clear_compiled_pattern_cache() - assert regex_module._COMPILED_PATTERN_CACHE == {} - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == retained_weight - - second_config = _config([{"pattern": "leased", "confidence": "high"}]) - second_engine = RegexEngine(second_config, None) - second_lease = regex_module._try_acquire_compiled_processor_lease( - (second_engine,) - ) - assert second_lease is not None - try: - assert second_engine._rules is first_engine._rules - assert len(regex_module._COMPILED_PATTERN_LEASES) == 1 - assert ( - next(iter(regex_module._COMPILED_PATTERN_LEASES.values())).lease_count - == 2 - ) - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == retained_weight - finally: - second_lease.release() - finally: - first_lease.release() - regex_module._clear_compiled_pattern_cache() - - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == 0 - assert regex_module._COMPILED_PATTERN_LEASES == {} - - -def test_leased_catalog_count_prevents_a_129th_retained_catalog( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - regex_module._clear_compiled_pattern_cache() - monkeypatch.setattr(regex_module, "_MAX_CACHED_COMPILED_CATALOGS", 1) - config = _config([{"pattern": "leased", "confidence": "high"}]) - engine = RegexEngine(config, None) - lease = regex_module._try_acquire_compiled_processor_lease((engine,)) - assert lease is not None - - try: - regex_module._clear_compiled_pattern_cache() - retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES - with caplog.at_level(logging.DEBUG, logger="privacy_guard.engines.regex"): - uncached_rules = regex_module._compile_pattern_catalog(_catalog("other")) - - assert uncached_rules - assert regex_module._COMPILED_PATTERN_CACHE == {} - assert regex_module._compiled_pattern_retained_count() == 1 - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == retained_weight - assert "privacy_guard_cache_skip cache=regex_compiled" in caplog.text - assert "other" not in caplog.text - finally: - lease.release() - regex_module._clear_compiled_pattern_cache() - - -def test_multi_catalog_lease_does_not_evict_its_own_reservation( - monkeypatch: pytest.MonkeyPatch, -) -> None: - regex_module._clear_compiled_pattern_cache() - monkeypatch.setattr(regex_module, "_MAX_CACHED_COMPILED_CATALOGS", 2) - first_catalog = _catalog("first") - evictable_catalog = _catalog("evictable") - second_catalog = _catalog("second") - first_engine = RegexEngine( - _config([{"pattern": "first", "confidence": "high"}]), - None, +def test_processor_weight_counts_each_regex_stage() -> None: + config = _config([{"pattern": "shared", "confidence": "high"}]) + first = RegexEngine(config, None) + second = RegexEngine(config, None) + + assert regex_module._compiled_processor_weight( + (first, second) + ) == 2 * regex_module._compiled_pattern_weight( + first.config.pattern_catalog, + len(first._rules), ) - second_engine = RegexEngine( - _config([{"pattern": "second", "confidence": "high"}]), - None, - ) - regex_module._clear_compiled_pattern_cache() - regex_module._compile_pattern_catalog(first_catalog) - regex_module._compile_pattern_catalog(evictable_catalog) - - lease = regex_module._try_acquire_compiled_processor_lease( - (first_engine, second_engine) - ) - assert lease is not None - try: - assert regex_module._compiled_pattern_retained_count() == 2 - assert set(regex_module._COMPILED_PATTERN_LEASES) == { - first_catalog, - second_catalog, - } - assert evictable_catalog not in regex_module._COMPILED_PATTERN_CACHE - assert ( - regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES - <= regex_module.MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES - ) - finally: - lease.release() - regex_module._clear_compiled_pattern_cache() def test_regex_engine_is_safe_for_concurrent_runs() -> None: diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index c36b4ac..da4c2c6 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -7,7 +7,7 @@ from concurrent.futures import Future, ThreadPoolExecutor from copy import deepcopy from threading import Barrier, Event, Lock, get_ident -from typing import Literal, Never +from typing import Never import grpc import pytest @@ -32,9 +32,6 @@ ) from privacy_guard.engines import ( EngineConfig, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, ) from privacy_guard.engines import regex as regex_module from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry @@ -47,25 +44,6 @@ ) from privacy_guard.service import servicer as servicer_module from privacy_guard.service.servicer import PrivacyGuardMiddleware -from privacy_guard.timeout import Timeout - - -class _LeaseFreeEngineConfig(EngineConfig): - engine: Literal["lease-free"] = "lease-free" - - -class _LeaseFreeEngine(EntityProcessingEngine[_LeaseFreeEngineConfig]): - supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - del strategy, timeout - return TextProcessingResult(text=text, detections=()) def _values( @@ -107,21 +85,31 @@ def _values( } +def _expected_processor_cache_weight( + registry: EngineRegistry, + values: dict[str, object], +) -> int: + config = registry.validate_config(values) + _, config_weight = _configuration_fingerprint_and_size(config) + regex_weight = sum( + regex_module._compiled_pattern_weight( + stage.config.pattern_catalog, + sum( + len(entity.patterns) for entity in stage.config.pattern_catalog.entities + ), + ) + for stage in config.entity_processing.stages + if isinstance(stage.config, regex_module.RegexEngineConfig) + ) + return config_weight + regex_weight + + def _proto_config(values: dict[str, object]) -> Message: result = pb2.ValidateConfigRequest().config json_format.ParseDict(values, result) return result -def _lease_free_values() -> dict[str, object]: - return { - "entity_processing": { - "stages": [{"config": {"engine": "lease-free"}}], - }, - "on_detection": {"action": "detect"}, - } - - def _request(body: bytes, *, action: str = "replace") -> pb2.HttpRequestEvaluation: return pb2.HttpRequestEvaluation( phase=pb2.SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS, @@ -578,8 +566,9 @@ def synchronized_build( "_build_processor", synchronized_build, ) + registry = create_builtin_registry() cache = servicer_module._RequestProcessorCache( - create_builtin_registry(), + registry, timeout_seconds=1, log_request_content=False, ) @@ -593,10 +582,10 @@ def resolve() -> RequestProcessor: assert build_count == 1 assert all(processor is processors[0] for processor in processors) - _, expected_weight = _configuration_fingerprint_and_size( - create_builtin_registry().validate_config(_values()) + assert cache._weight_bytes == _expected_processor_cache_weight( + registry, + _values(), ) - assert cache._weight_bytes == expected_weight assert cache._in_flight == {} cache.clear() @@ -619,8 +608,9 @@ def synchronized_build( "_build_processor", synchronized_build, ) + registry = create_builtin_registry() cache = servicer_module._RequestProcessorCache( - create_builtin_registry(), + registry, timeout_seconds=1, log_request_content=False, ) @@ -634,11 +624,9 @@ def synchronized_build( assert processors[0] is not processors[1] assert len(cache._processors) == 2 + values = (_values(action="detect"), _values(action="block")) expected_weight = sum( - _configuration_fingerprint_and_size( - create_builtin_registry().validate_config(values) - )[1] - for values in (_values(action="detect"), _values(action="block")) + _expected_processor_cache_weight(registry, item) for item in values ) assert cache._weight_bytes == expected_weight assert cache._in_flight == {} @@ -713,7 +701,7 @@ def capture_failure() -> PrivacyGuardError: cache.clear() -def test_processor_cache_evicts_least_recently_used_config_by_weight( +def test_processor_cache_evicts_least_recently_used_processor_by_weight( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: @@ -726,21 +714,21 @@ def test_processor_cache_evicts_least_recently_used_config_by_weight( _configuration_fingerprint_and_size(registry.validate_config(item)) for item in values ) - assert len({weight for _, weight in identities}) == 1 - entry_weight = identities[0][1] - monkeypatch.setattr( - servicer_module, - "MAX_PROCESSOR_CACHE_CONFIG_BYTES", - entry_weight * 2, - ) cache = servicer_module._RequestProcessorCache( registry, timeout_seconds=1, log_request_content=False, ) + first = cache.resolve(values[0]) + entry_weight = cache._processors[identities[0][0]].weight_bytes + assert entry_weight > identities[0][1] + monkeypatch.setattr( + servicer_module, + "MAX_PROCESSOR_CACHE_WEIGHT_BYTES", + entry_weight * 2, + ) with caplog.at_level(logging.DEBUG, logger="privacy_guard.service.servicer"): - first = cache.resolve(values[0]) cache.resolve(values[1]) assert cache.resolve(values[0]) is first cache.resolve(values[2]) @@ -752,19 +740,47 @@ def test_processor_cache_evicts_least_recently_used_config_by_weight( cache.clear() -def test_processor_cache_builds_but_does_not_retain_oversized_config( +def test_processor_cache_evicts_least_recently_used_processor_by_count( + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = create_builtin_registry() + values = tuple( + _values(action="detect", stage_name=f"stage-{suffix}") + for suffix in ("a", "b", "c") + ) + fingerprints = tuple( + _configuration_fingerprint_and_size(registry.validate_config(item))[0] + for item in values + ) + monkeypatch.setattr(servicer_module, "_MAX_CACHED_PROCESSORS", 2) + cache = servicer_module._RequestProcessorCache( + registry, + timeout_seconds=1, + log_request_content=False, + ) + + first = cache.resolve(values[0]) + cache.resolve(values[1]) + assert cache.resolve(values[0]) is first + cache.resolve(values[2]) + + assert tuple(cache._processors) == (fingerprints[0], fingerprints[2]) + assert cache._weight_bytes == sum( + entry.weight_bytes for entry in cache._processors.values() + ) + cache.clear() + + +def test_processor_cache_builds_but_does_not_retain_oversized_processor( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: registry = create_builtin_registry() values = _values(action="detect", stage_name="sensitive-oversized-stage") - _, weight_bytes = _configuration_fingerprint_and_size( - registry.validate_config(values) - ) monkeypatch.setattr( servicer_module, - "MAX_PROCESSOR_CACHE_CONFIG_BYTES", - weight_bytes - 1, + "MAX_PROCESSOR_CACHE_WEIGHT_BYTES", + 1, ) build_count = 0 original_build = servicer_module._RequestProcessorCache._build_processor @@ -820,13 +836,10 @@ def set_result(self, result: RequestProcessor) -> None: registry = create_builtin_registry() values = _values(action="detect", stage_name="oversized-single-flight") - _, weight_bytes = _configuration_fingerprint_and_size( - registry.validate_config(values) - ) monkeypatch.setattr( servicer_module, - "MAX_PROCESSOR_CACHE_CONFIG_BYTES", - weight_bytes - 1, + "MAX_PROCESSOR_CACHE_WEIGHT_BYTES", + 1, ) monkeypatch.setattr(servicer_module, "Future", _PublicationPausingFuture) build_count = 0 @@ -869,7 +882,7 @@ def record_build( cache.clear() -def test_regex_union_pressure_evicts_and_releases_old_processor( +def test_compiled_cache_eviction_does_not_invalidate_cached_processor( monkeypatch: pytest.MonkeyPatch, ) -> None: regex_module._clear_compiled_pattern_cache() @@ -879,156 +892,36 @@ def test_regex_union_pressure_evicts_and_releases_old_processor( timeout_seconds=1, log_request_content=False, ) - first_values = _values( + processor_values = _values( "detect", patterns=[{"pattern": "aaa", "confidence": "high"}], ) - second_values = _values( + validation_values = _values( "detect", patterns=[{"pattern": "bbb", "confidence": "high"}], ) - try: - cache.resolve(first_values) - retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES - monkeypatch.setattr( - regex_module, - "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", - retained_weight, - ) - - second = cache.resolve(second_values) - second_fingerprint, _ = _configuration_fingerprint_and_size( - registry.validate_config(second_values) - ) - - assert cache._processors == { - second_fingerprint: cache._processors[second_fingerprint] - } - assert cache._processors[second_fingerprint][0] is second - assert len(regex_module._COMPILED_PATTERN_LEASES) == 1 - assert regex_module._compiled_pattern_retained_count() == 1 - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES <= retained_weight - finally: - cache.clear() - regex_module._clear_compiled_pattern_cache() - - -def test_validate_config_churn_shares_the_compiled_union_with_processors( - monkeypatch: pytest.MonkeyPatch, -) -> None: - regex_module._clear_compiled_pattern_cache() - registry = create_builtin_registry() - cache = servicer_module._RequestProcessorCache( - registry, - timeout_seconds=1, - log_request_content=False, - ) - processor_values = _values( - "detect", - patterns=[{"pattern": "aaa", "confidence": "high"}], - ) - validation_values = tuple( - _values( - "detect", - patterns=[{"pattern": pattern, "confidence": "high"}], - ) - for pattern in ("bbb", "ccc") - ) - try: processor = cache.resolve(processor_values) - processor_engine = processor._stages[0][1] - assert isinstance(processor_engine, regex_module.RegexEngine) - processor_rules = processor_engine._rules entry_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES monkeypatch.setattr( regex_module, "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", - entry_weight * 2, + entry_weight, ) - for values in validation_values: - registry.validate_config(values) - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES <= entry_weight * 2 - assert regex_module._compiled_pattern_retained_count() <= 2 + registry.validate_config(validation_values) + result = processor.process("aaa") + assert len(result.detection_summaries) == 1 assert cache.resolve(processor_values) is processor - assert processor_engine._rules is processor_rules - assert len(regex_module._COMPILED_PATTERN_LEASES) == 1 - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES <= entry_weight * 2 + assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES <= entry_weight finally: cache.clear() regex_module._clear_compiled_pattern_cache() -def test_different_fingerprints_same_catalog_race_rebinds_canonical_rules( - monkeypatch: pytest.MonkeyPatch, -) -> None: - regex_module._clear_compiled_pattern_cache() - registry = create_builtin_registry() - cache = servicer_module._RequestProcessorCache( - registry, - timeout_seconds=1, - log_request_content=False, - ) - blocker_values = _values( - "detect", - patterns=[{"pattern": "aaa", "confidence": "high"}], - ) - shared_values = tuple( - _values( - action, - patterns=[{"pattern": "bbb", "confidence": "high"}], - ) - for action in ("detect", "block") - ) - - try: - cache.resolve(blocker_values) - retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES - monkeypatch.setattr( - regex_module, - "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", - retained_weight, - ) - builds_ready = Barrier(2) - original_build = servicer_module._RequestProcessorCache._build_processor - - def synchronized_build( - owner: servicer_module._RequestProcessorCache, - config: PrivacyGuardConfig[EngineConfig], - ) -> servicer_module._PreparedProcessor: - prepared = original_build(owner, config) - builds_ready.wait(timeout=5) - return prepared - - monkeypatch.setattr( - servicer_module._RequestProcessorCache, - "_build_processor", - synchronized_build, - ) - with ThreadPoolExecutor(max_workers=2) as executor: - processors = tuple(executor.map(cache.resolve, shared_values)) - - first_engine = processors[0]._stages[0][1] - second_engine = processors[1]._stages[0][1] - assert isinstance(first_engine, regex_module.RegexEngine) - assert isinstance(second_engine, regex_module.RegexEngine) - first_rules = first_engine._rules - assert second_engine._rules is first_rules - assert len(regex_module._COMPILED_PATTERN_LEASES) == 1 - assert ( - next(iter(regex_module._COMPILED_PATTERN_LEASES.values())).lease_count == 2 - ) - assert regex_module._compiled_pattern_retained_count() == 1 - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES <= retained_weight - finally: - cache.clear() - regex_module._clear_compiled_pattern_cache() - - -def test_intrinsically_oversized_regex_processor_preserves_cached_processor( +def test_oversized_regex_processor_preserves_cached_processor( monkeypatch: pytest.MonkeyPatch, ) -> None: regex_module._clear_compiled_pattern_cache() @@ -1055,10 +948,10 @@ def test_intrinsically_oversized_regex_processor_preserves_cached_processor( retained_fingerprint, _ = _configuration_fingerprint_and_size( registry.validate_config(retained_values) ) - retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES + retained_weight = cache._processors[retained_fingerprint].weight_bytes monkeypatch.setattr( - regex_module, - "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", + servicer_module, + "MAX_PROCESSOR_CACHE_WEIGHT_BYTES", retained_weight + 1, ) @@ -1066,77 +959,24 @@ def test_intrinsically_oversized_regex_processor_preserves_cached_processor( assert oversized is not retained assert tuple(cache._processors) == (retained_fingerprint,) - assert cache._processors[retained_fingerprint][0] is retained - assert len(regex_module._COMPILED_PATTERN_LEASES) == 1 - assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES == retained_weight - finally: - cache.clear() - regex_module._clear_compiled_pattern_cache() - - -def test_regex_pressure_preserves_lease_free_custom_processor( - monkeypatch: pytest.MonkeyPatch, -) -> None: - regex_module._clear_compiled_pattern_cache() - registry = EngineRegistry(include_builtin_engines=True) - registry.register(_LeaseFreeEngine) - registry.finalize() - cache = servicer_module._RequestProcessorCache( - registry, - timeout_seconds=1, - log_request_content=False, - ) - custom_values = _lease_free_values() - first_regex_values = _values( - "detect", - patterns=[{"pattern": "aaa", "confidence": "high"}], - ) - second_regex_values = _values( - "detect", - patterns=[{"pattern": "bbb", "confidence": "high"}], - ) - - try: - custom = cache.resolve(custom_values) - cache.resolve(first_regex_values) - custom_fingerprint, _ = _configuration_fingerprint_and_size( - registry.validate_config(custom_values) - ) - first_fingerprint, _ = _configuration_fingerprint_and_size( - registry.validate_config(first_regex_values) - ) - retained_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES - monkeypatch.setattr( - regex_module, - "MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES", - retained_weight, - ) - - cache.resolve(second_regex_values) - second_fingerprint, _ = _configuration_fingerprint_and_size( - registry.validate_config(second_regex_values) - ) - - assert custom_fingerprint in cache._processors - assert cache._processors[custom_fingerprint][0] is custom - assert first_fingerprint not in cache._processors - assert second_fingerprint in cache._processors + assert cache._processors[retained_fingerprint].processor is retained + assert cache._weight_bytes == retained_weight finally: cache.clear() regex_module._clear_compiled_pattern_cache() -def test_middleware_shutdown_releases_processor_regex_leases() -> None: +def test_middleware_shutdown_clears_processor_cache() -> None: regex_module._clear_compiled_pattern_cache() middleware = PrivacyGuardMiddleware(create_builtin_registry()) try: middleware._processors.resolve(_values("detect")) - assert regex_module._COMPILED_PATTERN_LEASES + assert middleware._processors._processors asyncio.run(middleware.close()) assert middleware._processors._processors == {} - assert regex_module._COMPILED_PATTERN_LEASES == {} + assert middleware._processors._weight_bytes == 0 finally: middleware._processors.clear() regex_module._clear_compiled_pattern_cache() From 4bbfb088b66dcbade8be28feae1f82de42c14998 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 16:05:46 +0000 Subject: [PATCH 75/82] refactor(privacy-guard): address review feedback --- projects/privacy-guard/README.md | 3 +- projects/privacy-guard/scripts/check.sh | 37 ---- .../privacy-guard/scripts/check_artifacts.py | 204 ------------------ .../scripts/external_engine_typing.py | 66 ------ .../privacy-guard/src/privacy_guard/config.py | 12 +- .../privacy-guard/src/privacy_guard/py.typed | 1 - .../src/privacy_guard/request_processor.py | 40 ++-- .../src/privacy_guard/service/servicer.py | 4 +- .../tests/service/test_servicer.py | 12 +- projects/privacy-guard/tests/test_config.py | 20 +- 10 files changed, 42 insertions(+), 357 deletions(-) delete mode 100644 projects/privacy-guard/scripts/check_artifacts.py delete mode 100644 projects/privacy-guard/scripts/external_engine_typing.py delete mode 100644 projects/privacy-guard/src/privacy_guard/py.typed diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 8be2264..828fa17 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -282,4 +282,5 @@ make check-py311 ``` `make check` delegates to `scripts/check.sh`, the authoritative local and CI -gate. It runs tests, formatting, lint, `ty`, import smoke, and package builds. +gate. It runs tests, formatting, lint, `ty`, an import smoke check, and a +dependency audit. diff --git a/projects/privacy-guard/scripts/check.sh b/projects/privacy-guard/scripts/check.sh index b9df356..c82e637 100755 --- a/projects/privacy-guard/scripts/check.sh +++ b/projects/privacy-guard/scripts/check.sh @@ -4,14 +4,12 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." uv_run=(uv run --frozen) -artifact_python=() if [[ $# -gt 0 ]]; then if [[ $1 != "--python" || $# -ne 2 ]]; then echo "usage: scripts/check.sh [--python VERSION]" >&2 exit 2 fi uv_run+=(--python "$2") - artifact_python+=(--python "$2") fi "${uv_run[@]}" pytest -q @@ -19,41 +17,6 @@ fi "${uv_run[@]}" ruff check . "${uv_run[@]}" ty check "${uv_run[@]}" python -c "import privacy_guard" -uv build -"${uv_run[@]}" python scripts/check_artifacts.py - -artifact_check_dir=$(mktemp -d) -trap 'rm -rf "$artifact_check_dir"' EXIT -uv venv --clear "${artifact_python[@]}" "$artifact_check_dir/venv" - -wheels=(dist/*.whl) -if [[ ${#wheels[@]} -ne 1 ]]; then - echo "expected exactly one built wheel, found ${#wheels[@]}" >&2 - exit 1 -fi - -VIRTUAL_ENV="$artifact_check_dir/venv" \ - uv sync --frozen --no-dev --no-install-project --active -uv pip install \ - --offline \ - --no-deps \ - --python "$artifact_check_dir/venv/bin/python" \ - "${wheels[0]}" -runtime_site_packages=$( - "$artifact_check_dir/venv/bin/python" \ - -c 'import sysconfig; print(sysconfig.get_path("purelib"))' -) -"${uv_run[@]}" pip-audit \ - --cache-dir "$artifact_check_dir/pip-audit-cache" \ - --progress-spinner off \ - --path "$runtime_site_packages" - -cp scripts/external_engine_typing.py "$artifact_check_dir/external_engine_typing.py" -"${uv_run[@]}" ty check \ - --project "$artifact_check_dir" \ - --python "$artifact_check_dir/venv" \ - "$artifact_check_dir/external_engine_typing.py" "${uv_run[@]}" pip-audit \ - --cache-dir "$artifact_check_dir/pip-audit-cache" \ --progress-spinner off \ --local diff --git a/projects/privacy-guard/scripts/check_artifacts.py b/projects/privacy-guard/scripts/check_artifacts.py deleted file mode 100644 index 75d348b..0000000 --- a/projects/privacy-guard/scripts/check_artifacts.py +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env python3 -"""Verify release semantics that generic packaging checks do not cover.""" - -from __future__ import annotations - -import re -import tarfile -import tomllib -import zipfile -from collections.abc import Iterable -from email import policy -from email.message import Message -from email.parser import BytesParser -from pathlib import Path - -PROJECT_ROOT = Path(__file__).resolve().parent.parent -REPOSITORY_ROOT = PROJECT_ROOT.parent.parent -DIST_DIRECTORY = PROJECT_ROOT / "dist" - -CANONICAL_README_TARGETS = { - "https://github.com/NVIDIA/OpenShell-Research/blob/main/" - "projects/privacy-guard/examples/regex-engine/README.md": ( - REPOSITORY_ROOT / "projects/privacy-guard/examples/regex-engine/README.md" - ), - "https://github.com/NVIDIA/OpenShell-Research/blob/main/" - "projects/privacy-guard/examples/custom-engine/README.md": ( - REPOSITORY_ROOT / "projects/privacy-guard/examples/custom-engine/README.md" - ), - "https://github.com/NVIDIA/OpenShell-Research/blob/main/" - "projects/openshell-middleware-kit/README.md": ( - REPOSITORY_ROOT / "projects/openshell-middleware-kit/README.md" - ), -} -REPOSITORY_RELATIVE_README_TARGETS = ( - "(examples/regex-engine/README.md)", - "(examples/custom-engine/README.md)", - "(../openshell-middleware-kit/README.md)", -) - - -def main() -> None: - """Inspect the wheel and source distribution produced for this project.""" - project = _load_project_metadata() - distribution_name = re.sub(r"[-_.]+", "_", project["name"]) - version = project["version"] - archive_stem = f"{distribution_name}-{version}" - - wheel = _find_one(DIST_DIRECTORY.glob(f"{archive_stem}-*.whl"), "wheel") - source_distribution = _find_one( - DIST_DIRECTORY.glob(f"{archive_stem}.tar.gz"), - "source distribution", - ) - - _check_readme_targets() - _check_wheel(wheel, archive_stem) - _check_source_distribution(source_distribution, archive_stem) - - -def _load_project_metadata() -> dict[str, str]: - with (PROJECT_ROOT / "pyproject.toml").open("rb") as file: - project = tomllib.load(file)["project"] - return {"name": project["name"], "version": project["version"]} - - -def _find_one(paths: Iterable[Path], description: str) -> Path: - matches = sorted(paths) - if len(matches) != 1: - raise SystemExit(f"expected exactly one {description}, found {len(matches)}") - return matches[0] - - -def _check_readme_targets() -> None: - missing = [ - str(target.relative_to(REPOSITORY_ROOT)) - for target in CANONICAL_README_TARGETS.values() - if not target.is_file() - ] - if missing: - raise SystemExit( - "canonical README link targets are missing: " + ", ".join(missing) - ) - - -def _check_wheel(wheel: Path, archive_stem: str) -> None: - metadata_member = f"{archive_stem}.dist-info/METADATA" - license_member = f"{archive_stem}.dist-info/licenses/LICENSE" - typed_marker = "privacy_guard/py.typed" - - with zipfile.ZipFile(wheel) as archive: - members = set(archive.namelist()) - _require_members( - wheel.name, - members, - (metadata_member, license_member, typed_marker), - ) - _require_exact_license(wheel.name, archive.read(license_member)) - metadata = _parse_metadata(archive.read(metadata_member)) - _require_license_metadata(wheel.name, metadata) - _require_readme_links(wheel.name, _metadata_description(metadata)) - - print(f"{wheel.name}: {license_member}, {typed_marker}, License-File: LICENSE") - - -def _check_source_distribution( - source_distribution: Path, - archive_stem: str, -) -> None: - metadata_member = f"{archive_stem}/PKG-INFO" - license_member = f"{archive_stem}/LICENSE" - readme_member = f"{archive_stem}/README.md" - typed_marker = f"{archive_stem}/src/privacy_guard/py.typed" - - with tarfile.open(source_distribution, "r:gz") as archive: - members = set(archive.getnames()) - _require_members( - source_distribution.name, - members, - (metadata_member, license_member, readme_member, typed_marker), - ) - _require_exact_license( - source_distribution.name, - _read_tar_member(archive, license_member), - ) - metadata = _parse_metadata(_read_tar_member(archive, metadata_member)) - _require_license_metadata(source_distribution.name, metadata) - _require_readme_links( - f"{source_distribution.name}:{readme_member}", - _read_tar_member(archive, readme_member).decode("utf-8"), - ) - _require_readme_links( - f"{source_distribution.name}:{metadata_member}", - _metadata_description(metadata), - ) - - print( - f"{source_distribution.name}: {license_member}, {typed_marker}, " - "License-File: LICENSE" - ) - - -def _require_members( - artifact: str, - members: set[str], - required: tuple[str, ...], -) -> None: - missing = [member for member in required if member not in members] - if missing: - raise SystemExit(f"{artifact} is missing: {', '.join(missing)}") - - -def _require_exact_license(artifact: str, packaged_license: bytes) -> None: - expected_license = (PROJECT_ROOT / "LICENSE").read_bytes() - if packaged_license != expected_license: - raise SystemExit(f"{artifact} does not contain the exact project LICENSE") - - -def _parse_metadata(contents: bytes) -> Message: - return BytesParser(policy=policy.default).parsebytes(contents) - - -def _require_license_metadata(artifact: str, metadata: Message) -> None: - if metadata["License-Expression"] != "Apache-2.0": - raise SystemExit( - f"{artifact} metadata does not declare License-Expression: Apache-2.0" - ) - license_files = metadata.get_all("License-File") or [] - if "LICENSE" not in license_files: - raise SystemExit(f"{artifact} metadata is missing License-File: LICENSE") - - -def _metadata_description(metadata: Message) -> str: - description = metadata.get_payload() - if not isinstance(description, str): - raise SystemExit("package metadata description is not text") - return description - - -def _require_readme_links(artifact: str, readme: str) -> None: - missing = [url for url in CANONICAL_README_TARGETS if url not in readme] - if missing: - raise SystemExit( - f"{artifact} README is missing canonical links: {', '.join(missing)}" - ) - retained = [ - relative - for relative in REPOSITORY_RELATIVE_README_TARGETS - if relative in readme - ] - if retained: - raise SystemExit( - f"{artifact} README retains repository-relative links: " - + ", ".join(retained) - ) - - -def _read_tar_member(archive: tarfile.TarFile, member: str) -> bytes: - extracted = archive.extractfile(member) - if extracted is None: - raise SystemExit(f"could not read source distribution member {member}") - return extracted.read() - - -if __name__ == "__main__": - main() diff --git a/projects/privacy-guard/scripts/external_engine_typing.py b/projects/privacy-guard/scripts/external_engine_typing.py deleted file mode 100644 index 3089295..0000000 --- a/projects/privacy-guard/scripts/external_engine_typing.py +++ /dev/null @@ -1,66 +0,0 @@ -"""External consumer fixture checked only against an installed wheel.""" - -from __future__ import annotations - -from typing import Literal, assert_type - -from privacy_guard.engines import ( - EngineConfig, - EngineResources, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, -) -from privacy_guard.timeout import Timeout - - -class ExternalConfig(EngineConfig): - engine: Literal["external"] = "external" - keyword: str - - -class ExternalClient: - def find(self, text: str, keyword: str) -> bool: - return keyword in text - - -class ExternalResources(EngineResources): - __slots__ = ("client",) - - def __init__(self, client: ExternalClient) -> None: - self.client = client - - -class ExternalEngine(EntityProcessingEngine[ExternalConfig, ExternalResources]): - supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - assert_type(self.config, ExternalConfig) - assert_type(self.config.keyword, str) - assert_type(self.resources, ExternalResources) - assert_type(self.resources.client, ExternalClient) - self.resources.client.find(text, self.config.keyword) - result = TextProcessingResult(text=text, detections=()) - assert_type(result, TextProcessingResult) - return result - - -engine = ExternalEngine( - ExternalConfig(keyword="secret"), - ExternalResources(ExternalClient()), -) -assert_type(engine.config, ExternalConfig) -assert_type(engine.resources, ExternalResources) -processed = engine.run( - "text", - strategy=EntityProcessingStrategy.DETECT, - timeout=Timeout.from_seconds(1), -) -assert_type(processed, TextProcessingResult) -assert_type(processed.text, str) diff --git a/projects/privacy-guard/src/privacy_guard/config.py b/projects/privacy-guard/src/privacy_guard/config.py index 40e5f82..7395ea9 100644 --- a/projects/privacy-guard/src/privacy_guard/config.py +++ b/projects/privacy-guard/src/privacy_guard/config.py @@ -113,15 +113,7 @@ class PrivacyGuardConfig( on_detection: OnDetection = Field(repr=False) -def configuration_fingerprint( - config: PrivacyGuardConfig[EngineConfig], -) -> str: - """Return the canonical SHA-256 fingerprint of an expanded policy config.""" - fingerprint, _ = _configuration_fingerprint_and_size(config) - return fingerprint - - -def _configuration_fingerprint_and_size( +def configuration_fingerprint_and_size( config: PrivacyGuardConfig[EngineConfig], ) -> tuple[str, int]: """Return the canonical fingerprint and encoded size of an expanded config.""" @@ -141,5 +133,5 @@ def _configuration_fingerprint_and_size( "OnDetection", "PolicyAction", "PrivacyGuardConfig", - "configuration_fingerprint", + "configuration_fingerprint_and_size", ] diff --git a/projects/privacy-guard/src/privacy_guard/py.typed b/projects/privacy-guard/src/privacy_guard/py.typed deleted file mode 100644 index 8b13789..0000000 --- a/projects/privacy-guard/src/privacy_guard/py.typed +++ /dev/null @@ -1 +0,0 @@ - diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index b12ad91..78af758 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -5,7 +5,6 @@ from collections import OrderedDict from collections.abc import Sequence from enum import StrEnum -from typing import Protocol from pydantic import Field @@ -21,6 +20,8 @@ from privacy_guard.engines import ( ConfidenceLevel, EngineConfig, + EngineResources, + EntityProcessingEngine, EntityProcessingStrategy, TextProcessingResult, ) @@ -69,21 +70,26 @@ class RequestProcessor: def __init__( self, config: PrivacyGuardConfig[EngineConfig], - stages: Sequence[tuple[str, _RunnableEngine]], + configured_engines: Sequence[ + tuple[ + str, + EntityProcessingEngine[EngineConfig, EngineResources | None], + ] + ], *, timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, log_request_content: bool = False, ) -> None: - configured_stages = tuple(stages) - if len(configured_stages) != len(config.entity_processing.stages): - raise ValueError("configured stages do not match the policy") - if not configured_stages: - raise ValueError("at least one configured stage is required") - sources = tuple(source for source, _ in configured_stages) + engines = tuple(configured_engines) + if len(engines) != len(config.entity_processing.stages): + raise ValueError("configured engines do not match the policy") + if not engines: + raise ValueError("at least one configured engine is required") + sources = tuple(source for source, _ in engines) if any(not source for source in sources) or len(sources) != len(set(sources)): - raise ValueError("stage sources must be non-empty and unique") + raise ValueError("engine sources must be non-empty and unique") self._config = config - self._stages = configured_stages + self._engines = engines self._timeout_seconds = validate_timeout_seconds(timeout_seconds) self._log_request_content = log_request_content @@ -108,7 +114,7 @@ def process(self, text: str) -> RequestProcessingResult: current_text = input_text stage_results: list[tuple[str, TextProcessingResult]] = [] try: - for source, engine in self._stages: + for source, engine in self._engines: _LOGGER.debug( "privacy_guard_stage_run source=%s strategy=%s", source, @@ -196,18 +202,6 @@ def _aggregate_detections( ) -class _RunnableEngine(Protocol): - """The engine behavior needed by request orchestration.""" - - def run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: ... - - _LOGGER = get_logger(__name__) diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index 26692a4..f53dca3 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -21,7 +21,7 @@ from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc from privacy_guard.config import ( PrivacyGuardConfig, - _configuration_fingerprint_and_size, + configuration_fingerprint_and_size, ) from privacy_guard.constants import ( BLOCK_REASON, @@ -269,7 +269,7 @@ def __init__( def resolve(self, values: object) -> RequestProcessor: """Return the cached or newly prepared processor for expanded config.""" config = self._registry.validate_config(values) - fingerprint, config_weight_bytes = _configuration_fingerprint_and_size(config) + fingerprint, config_weight_bytes = configuration_fingerprint_and_size(config) with self._lock: cached = self._processors.get(fingerprint) if cached is not None: diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index da4c2c6..c8165be 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -17,7 +17,7 @@ from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.config import ( PrivacyGuardConfig, - _configuration_fingerprint_and_size, + configuration_fingerprint_and_size, ) from privacy_guard.constants import ( LIMIT_REASON, @@ -90,7 +90,7 @@ def _expected_processor_cache_weight( values: dict[str, object], ) -> int: config = registry.validate_config(values) - _, config_weight = _configuration_fingerprint_and_size(config) + _, config_weight = configuration_fingerprint_and_size(config) regex_weight = sum( regex_module._compiled_pattern_weight( stage.config.pattern_catalog, @@ -711,7 +711,7 @@ def test_processor_cache_evicts_least_recently_used_processor_by_weight( for suffix in ("a", "b", "c") ) identities = tuple( - _configuration_fingerprint_and_size(registry.validate_config(item)) + configuration_fingerprint_and_size(registry.validate_config(item)) for item in values ) cache = servicer_module._RequestProcessorCache( @@ -749,7 +749,7 @@ def test_processor_cache_evicts_least_recently_used_processor_by_count( for suffix in ("a", "b", "c") ) fingerprints = tuple( - _configuration_fingerprint_and_size(registry.validate_config(item))[0] + configuration_fingerprint_and_size(registry.validate_config(item))[0] for item in values ) monkeypatch.setattr(servicer_module, "_MAX_CACHED_PROCESSORS", 2) @@ -945,7 +945,7 @@ def test_oversized_regex_processor_preserves_cached_processor( try: retained = cache.resolve(retained_values) - retained_fingerprint, _ = _configuration_fingerprint_and_size( + retained_fingerprint, _ = configuration_fingerprint_and_size( registry.validate_config(retained_values) ) retained_weight = cache._processors[retained_fingerprint].weight_bytes @@ -993,7 +993,7 @@ def unexpected_call(*args: object, **kwargs: object) -> object: monkeypatch.setattr( servicer_module, - "_configuration_fingerprint_and_size", + "configuration_fingerprint_and_size", unexpected_call, ) monkeypatch.setattr( diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py index 878832f..ad3a826 100644 --- a/projects/privacy-guard/tests/test_config.py +++ b/projects/privacy-guard/tests/test_config.py @@ -15,7 +15,7 @@ import privacy_guard.engines.regex as regex_module from privacy_guard.config import ( PolicyAction, - configuration_fingerprint, + configuration_fingerprint_and_size, ) from privacy_guard.engines import ( RegexEngine, @@ -182,9 +182,9 @@ def test_catalog_file_and_inline_catalog_produce_the_same_config( file_config = registry.validate_config(file_values) assert file_config == inline_config - assert configuration_fingerprint(file_config) == configuration_fingerprint( - inline_config - ) + assert configuration_fingerprint_and_size( + file_config + ) == configuration_fingerprint_and_size(inline_config) serialized_catalog = file_config.model_dump(mode="json")["entity_processing"][ "stages" ][0]["config"]["pattern_catalog"] @@ -214,7 +214,9 @@ def test_catalog_file_change_produces_a_new_fingerprint( catalog_path.write_text(yaml.safe_dump(catalog), encoding="utf-8") second = registry.validate_config(values) - assert configuration_fingerprint(first) != configuration_fingerprint(second) + assert configuration_fingerprint_and_size( + first + ) != configuration_fingerprint_and_size(second) def test_parsed_catalog_cache_evicts_least_recently_used_file_by_weight( @@ -663,8 +665,12 @@ def test_canonical_fingerprint_covers_concrete_expanded_config() -> None: ][0]["patterns"][0]["confidence"] = "low" changed = registry.validate_config(changed_values) - assert configuration_fingerprint(first) == configuration_fingerprint(equivalent) - assert configuration_fingerprint(first) != configuration_fingerprint(changed) + assert configuration_fingerprint_and_size( + first + ) == configuration_fingerprint_and_size(equivalent) + assert configuration_fingerprint_and_size( + first + ) != configuration_fingerprint_and_size(changed) def test_models_are_frozen_and_hide_engine_configuration_from_repr() -> None: From 96683e1f8802b41fee359f5ccdab274ba15dc5d2 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 16:18:14 +0000 Subject: [PATCH 76/82] refactor(privacy-guard): use one active policy (PG-008 PG-009) --- .../architecture/configuration.md | 36 +- .../privacy-guard/architecture/index.md | 16 +- .../architecture/request-lifecycle.md | 22 +- .../architecture/safety-and-limits.md | 30 +- .../architecture/service-boundary.md | 100 +-- projects/privacy-guard/README.md | 21 +- .../privacy-guard/src/privacy_guard/config.py | 17 - .../src/privacy_guard/constants.py | 1 - .../src/privacy_guard/engines/regex.py | 7 - .../src/privacy_guard/service/servicer.py | 147 +---- .../privacy-guard/tests/engines/test_regex.py | 13 - .../tests/service/test_grpc_integration.py | 8 + .../tests/service/test_server.py | 4 +- .../tests/service/test_servicer.py | 596 ++++++------------ projects/privacy-guard/tests/test_cli.py | 4 +- projects/privacy-guard/tests/test_config.py | 20 +- 16 files changed, 354 insertions(+), 688 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/configuration.md b/docs/documentation/privacy-guard/architecture/configuration.md index 68bede7..811734b 100644 --- a/docs/documentation/privacy-guard/architecture/configuration.md +++ b/docs/documentation/privacy-guard/architecture/configuration.md @@ -84,11 +84,11 @@ traversal, and symlinks are rejected. Catalog files must be bounded UTF-8 YAML without aliases, duplicate keys, or unsafe tags. File-backed and inline inputs normalize to the same `RegexPatternCatalog`. -Serialization and canonical fingerprints contain the validated structured -catalog rather than its source path. File metadata keys a bounded parser cache; -the normalized immutable catalog keys a bounded compiled-rule cache. A content -change produces a newly validated configuration, compiled rule set, and -processor fingerprint. +The complete validated configuration contains the structured catalog rather +than its source path. File metadata keys a bounded parser cache; the normalized +immutable catalog keys a bounded compiled-rule cache. A content change produces +a different validated configuration and causes the next evaluation to prepare +an active-policy replacement. Privacy Guard maintains the catalog schema and safety limits but does not ship an authoritative pattern set. Repository catalogs are examples to copy and @@ -113,9 +113,10 @@ the middleware process. The service rejects an encoded configuration above that limit before protobuf conversion or policy validation. A policy may contain at most ten ordered entity-processing stages. The service validates the complete bounded -configuration, computes its canonical fingerprint, and uses a bounded internal -`RequestProcessor` cache. Caching avoids repeated engine initialization but -does not increase either limit. +configuration and compares it with the single active policy. Equal validated +configurations reuse the active processor; a different valid configuration pays +full engine and processor preparation before atomic replacement. Reuse avoids +repeated engine initialization but does not increase either limit. `Struct` carries every number as a double. At this transport boundary only, Privacy Guard converts finite integral values in the safe integer range @@ -129,14 +130,19 @@ A future self-contained transport for larger expanded catalogs requires an upstream OpenShell contract for preparing configuration and referring to it during evaluation. Privacy Guard must not create a private protocol fork. -## Configuration identity +## Policy identity -Canonical serialization includes every concrete engine field and nested -replacement variant. Mapping keys are sorted, compact JSON encoding is used, -and the SHA-256 fingerprint is computed over the resulting UTF-8 bytes. +Privacy Guard compares the complete immutable validated configuration, +including every concrete engine field, nested replacement variant, and expanded +Regex catalog. Equal configurations reuse the active processor. A different +configuration is an update candidate and replaces the active policy only after +complete preparation succeeds. -Equivalent structured configurations therefore share a processor cache entry. -Cache state is only an optimization; eviction or restart reconstructs the -processor from configuration supplied by a later evaluation. +The current protocol carries neither an explicit update operation nor a policy +version. Privacy Guard therefore cannot distinguish an intentional user update +from any other changed per-evaluation configuration. OpenShell must send one +consistent policy stream to a process; interleaved old and new configurations +can repeatedly replace one another. A future protocol can make preparation and +activation explicit and let evaluations refer to a versioned policy. [Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/index.md b/docs/documentation/privacy-guard/architecture/index.md index 3015e8f..2e8d932 100644 --- a/docs/documentation/privacy-guard/architecture/index.md +++ b/docs/documentation/privacy-guard/architecture/index.md @@ -51,9 +51,9 @@ Source paths on these pages are relative to `projects/privacy-guard/src/privacy_guard/`. - `service/` owns gRPC, protobuf conversion, UTF-8 decoding and encoding, - bounded worker scheduling, processor caching, and finding serialization. - Outside generated `bindings/`, no other package imports gRPC or generated - bindings. + bounded worker scheduling, active-processor preparation and replacement, and + finding serialization. Outside generated `bindings/`, no other package + imports gRPC or generated bindings. - `cli.py` owns command parsing, registry-factory loading, engine discovery, configuration-schema output, logging options, and the adapter that starts the programmatic server. Top-level `logging.py` provides the shared, @@ -66,8 +66,7 @@ Source paths on these pages are relative to implementations and operator-owned resources, builds the exact Pydantic discriminated union, and contains the built-in Regex implementation. Each engine owns its detection and replacement algorithms. -- `config.py` defines ordered stages, the required policy action, canonical - configuration serialization, and fingerprints. +- `config.py` defines ordered stages and the required policy action. - top-level `base.py` defines the package-wide strict immutable domain-model base. - `string_validators.py` defines shared string validators and field types. @@ -87,7 +86,8 @@ HttpRequestEvaluation protobuf PrivacyGuardMiddleware validates phase and body size validates and normalizes policy configuration - resolves or builds a cached RequestProcessor + reuses the matching active RequestProcessor or prepares and activates + a replacement decodes a non-empty body as strict UTF-8 | v @@ -165,7 +165,7 @@ representations. extension contract and built-in engines. - [Configuration and text boundary](configuration.md) covers the one-text contract, configuration ownership, and current catalog limits. -- [Service boundary](service-boundary.md) covers gRPC adaptation, caching, and - concurrency. +- [Service boundary](service-boundary.md) covers gRPC adaptation, policy + activation, and concurrency. - [Safety and limits](safety-and-limits.md) records failure behavior and resource bounds. diff --git a/docs/documentation/privacy-guard/architecture/request-lifecycle.md b/docs/documentation/privacy-guard/architecture/request-lifecycle.md index fc5b2fd..19e0791 100644 --- a/docs/documentation/privacy-guard/architecture/request-lifecycle.md +++ b/docs/documentation/privacy-guard/architecture/request-lifecycle.md @@ -63,23 +63,27 @@ For each evaluation under the current OpenShell protocol, the service: 3. validates each concrete config against its registered implementation and injected resources 4. validates the action/replacement compatibility -5. computes a SHA-256 fingerprint of canonical expanded configuration -6. resolves a cached `RequestProcessor`, or constructs the engines and - processor and adds it to the bounded cache +5. reuses the active `RequestProcessor` when its complete immutable validated + configuration is equal +6. otherwise serializes preparation, constructs every configured engine and a + candidate processor, and atomically activates the candidate only after full + success -`ValidateConfig` performs the validation steps without populating this cache. -Preparation is repeatable; cache state is an optimization and never required -for correctness. +The first evaluation establishes the active policy and pays its preparation +cost. A failed validation or preparation leaves an existing active processor +unchanged and fails the triggering evaluation. `ValidateConfig` performs +validation without constructing engines or changing the active policy. There is no separate execution-plan abstraction. The validated stage order -already contains the necessary policy structure, and the prepared processor +already contains the necessary policy structure, and the active processor privately retains the corresponding ordered engine instances. ## Text input The service validates the pre-credentials phase and the request body byte -limit before processing. It still validates configuration for an empty body, -then immediately allows that body without invoking an engine. +limit before processing. It still validates configuration and resolves the +active processor for an empty body, then immediately allows that body without +invoking an engine. A non-empty body must decode as strict UTF-8. The decoded `str` is the only request input passed to `RequestProcessor`; headers, content type, request ID, diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 574dcbb..35ae855 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -243,27 +243,31 @@ validation without a concrete failure mode and a clear owning layer. ## State and retention -Cross-request entity memory is not implemented. Privacy Guard retains validated -configuration and immutable engine state in its processor cache, but never -retains request text, detections, or replacement mappings there. +Cross-request entity memory is not implemented. Privacy Guard retains one active +validated policy and its immutable configured engine state, but never request +text, detections, or replacement mappings. The active processor is deliberate +process state, not an LRU entry: it remains until successful replacement or +shutdown. -Prepared state is bounded by weight and least-recently-used entry count: +During an update, the fully prepared candidate may coexist with the active +processor. Evaluations that already captured the prior processor may keep it +alive until they finish. A failed candidate leaves active state unchanged. + +Regex's supporting caches remain independently bounded: | Cache | Weight budget | Entry cap | Entry weight | | --- | ---: | ---: | --- | | Parsed Regex file catalogs | 8 MiB | 64 | source file bytes | | Compiled Regex rules | 32 MiB | 128 | canonical catalog bytes plus 4 KiB per rule | -| Request processors | 32 MiB | 16 | canonical expanded configuration plus the full Regex state estimate for every Regex stage | The compiled-rule allowance accounts for retained backend state that is much -larger than short pattern text. Cache budgets are independent. The compiled-rule -cache charges each retained catalog once. The processor cache charges every -processor for its full estimated Regex state, even when equivalent rules are -referenced elsewhere. This conservative accounting avoids coupling eviction or -ownership between caches. Custom engines do not need a cache-size hook. One -entry that exceeds its cache budget remains valid and usable for the current -evaluation but is not retained. Debug skip and eviction records contain only -cache names and weights, plus eviction counts. +larger than short pattern text. Cache budgets are independent. One entry that +exceeds a Regex cache budget remains valid and usable for the current operation +but is not retained there. This does not invalidate an active or candidate +processor holding those rules. The active policy is not rejected by cache +admission; policy and schema limits bound its shape, while custom-engine +prepared memory remains an operator responsibility. Regex cache skip and +eviction records contain only cache names and weights, plus eviction counts. ## Changing a limit diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index 3d5e60e..3c260f1 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -1,6 +1,6 @@ --- title: Service boundary -description: gRPC adaptation, configuration caching, worker scheduling, and server lifecycle. +description: gRPC adaptation, policy activation, worker scheduling, and server lifecycle. agent_markdown: true --- @@ -23,7 +23,7 @@ Privacy Guard implements the three methods currently defined by | --- | --- | | `Describe` | Advertises service identity, one pre-credentials HTTP binding, and the 4 MiB body limit | | `ValidateConfig` | Validates supplied policy configuration and registered resources | -| `EvaluateHttpRequest` | Validates transport input, resolves a processor, processes text, and returns a decision | +| `EvaluateHttpRequest` | Validates transport input, resolves the active processor, processes text, and returns a decision | `Describe` advertises only `SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS`. Evaluations at any other phase @@ -47,20 +47,24 @@ validation and every request evaluation. 5. validates replacement support for a replace action 6. returns `valid=true`, or a content-safe reason -It does not construct engines, populate the processor cache, contact model -providers, download resources, or write artifacts. Validation runs in the -bounded worker pool so catalog parsing and engine-specific checks do not block -the async gRPC event loop. - -During evaluation, the service validates and normalizes the same config, -computes its canonical SHA-256 fingerprint, and resolves a configured processor -from an LRU cache bounded by both 16 entries and 32 MiB of prepared-state -weight. A cache miss constructs engines directly from the exact stage configs -and operator-injected resources, then constructs the `RequestProcessor`. -Validation, cache resolution, engine construction, strict UTF-8 decoding, and -processing all run in the bounded worker pool. Validation still occurs for -every evaluation. Equivalent normalized Regex catalogs may reuse an entry in -the independent compiled-rule cache during validation and engine construction. +It does not construct engines, change the active policy, contact model providers, +download resources, or write artifacts. Validation runs in the bounded worker +pool so catalog parsing and engine-specific checks do not block the async gRPC +event loop. + +During evaluation, the service validates and normalizes the same config. It +retains one active pair: the complete immutable validated configuration and its +configured `RequestProcessor`. Equal config reuses the processor. When no +processor is active or config differs, one serialized update path constructs +engines from the exact stage configs and operator-injected resources, constructs +a candidate processor, and atomically replaces the active pair only after +preparation succeeds. Failure preserves the prior active pair and fails the +triggering evaluation. + +Validation, resolution or preparation, strict UTF-8 decoding, and processing all +run in the bounded worker pool. Validation still occurs for every evaluation. +Equivalent normalized Regex catalogs may reuse an entry in the independent +compiled-rule cache during validation and engine construction. Regex maintains two independent LRU caches. Parsed file catalogs retain at most 64 entries and 8 MiB of source files. Compiled rules retain at most 128 catalogs @@ -68,32 +72,21 @@ and 32 MiB; each entry weighs its canonical catalog bytes plus 4 KiB per rule. Evicting a compiled-rule entry only removes that cache's reference. A processor that already uses those rules remains valid. -Processor-cache admission counts canonical configuration bytes and the full -estimated Regex state held by every configured Regex stage. Each processor -receives that full charge even when an equivalent compiled tuple is also -referenced by the compiled-rule cache or another processor. This conservative -accounting intentionally favors simple, independent caches over exact -shared-memory accounting. - -An otherwise valid entry larger than its cache's byte budget is built and used -for the current operation but is not retained. Eviction or a skipped entry -never makes a policy invalid. Content-safe debug events report only the cache -name and weight, plus the number of evicted entries. - -The caches are protected for concurrent access and are not -correctness-relevant. Eviction or process restart simply causes reconstruction -from a later evaluation's expanded config. Operators with consistently larger -catalogs should expect more frequent preparation rather than unbounded -retention. - -Each cached processor receives the server's operational processing timeout. +An otherwise valid Regex cache entry larger than its byte budget is built and +used for the current operation but is not retained in that cache. The active +processor itself has no LRU admission budget: it is deliberate process state +retained until replacement or shutdown. Regex cache eviction or a skipped entry +never invalidates it. Process restart discards the active processor, and the +first later evaluation reconstructs it from supplied configuration. + +Each active processor receives the server's operational processing timeout. The default is 1 second shared across every configured stage. Operators may set up to 30 seconds with `privacy-guard serve --timeout-seconds`; programmatic applications pass the same `timeout_seconds` argument to `PrivacyGuardServer`. OpenShell's outer middleware `timeout` must include the processing timeout plus additional headroom for worker queueing, repeated configuration validation, -cache resolution, and engine construction so the supervisor does not end the -RPC first. +and occasional candidate preparation so the supervisor does not end the RPC +first. ## Incoming requests @@ -104,7 +97,7 @@ For each evaluation, the service: (32 KiB), headers (128 entries and 64 KiB), and body (4 MiB) limits 3. acquires one bounded worker slot 4. converts transport-safe integral numbers, validates the maximum ten stages, - and resolves the supplied policy config in that worker + and resolves or updates the active policy in that worker 5. allows an empty body without invoking an engine 6. decodes a non-empty body as strict UTF-8 7. calls `RequestProcessor.process(text)` in the same worker @@ -160,7 +153,7 @@ gRPC event loop | v configuration validation -and processor resolution +and active-policy resolution | v RequestProcessor.process @@ -172,9 +165,13 @@ ordered engine pipeline This keeps synchronous configuration and engine work off the async event loop and bounds the number of active operations. -Cached processors, engine instances, and injected resources may be used by +The active processor, engine instances, and injected resources may be used by multiple worker threads. They must retain no mutable per-request state and must -be safe for concurrent access. +be safe for concurrent access. Policy preparation is serialized. A candidate is +fully prepared before the active reference is swapped. An evaluation that +already captured the old processor may finish with it while later evaluations +use the replacement; this is an atomic reference cutover, not a draining +cutover. ## Cancellation @@ -227,7 +224,8 @@ that registry to `PrivacyGuardServer`. The registry is an explicit application-scoped dependency, not a global singleton. `PrivacyGuardServer` and `PrivacyGuardMiddleware` reject unfinalized registries. A deployment creates and finalizes one registry during startup; -cached processors then construct configured stage engines from that registry. +the middleware constructs configured stage engines and the active processor +from that registry. Different middleware applications in the same process may intentionally use different engine inventories or runtime resources. @@ -312,17 +310,20 @@ not policy-controlled behavior. The intended large-catalog and discovery experience requires coordinated OpenShell changes for: -- a preparation operation that accepts expanded configuration larger than the - current 64 KiB evaluation limit -- canonical configuration fingerprints on evaluations -- typed cache-miss recovery +- explicit preparation and activation that accepts expanded configuration + larger than the current 64 KiB evaluation limit +- versioned policy references on evaluations - finalized policy schema and engine discovery in the manifest - a dedicated finding source field These are protocol evolution items, not local implementation hooks. Until they land upstream, Privacy Guard continues to validate the per-evaluation config, -repopulate its local cache when needed, expose discovery through the CLI, and -render stage provenance inside the finding label. +treat a changed valid config as an active-policy update, expose discovery +through the CLI, and render stage provenance inside the finding label. + +The protocol cannot distinguish an intentional update from any other changed +configuration. Each process must receive one consistent policy stream. +Interleaved old and new configurations can repeatedly switch the active policy. ## Testing the boundary @@ -332,7 +333,8 @@ Service tests should cover: - manifest fields supported by the current protocol - pure policy validation - phase, body-size, and UTF-8 validation -- cache hit and reconstruction behavior +- first activation, equal-config reuse, serialized replacement, failed-update + preservation, and concurrent cutover - detect, replacement, block, and limit serialization - finding encoding and limits - gRPC status mapping diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 828fa17..b84d5a1 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -51,9 +51,10 @@ passes its catalog as `patterns.yaml`. ```text OpenShell HttpRequestEvaluation - -> strict UTF-8 decode -> finalized Pydantic policy union (config.engine discriminator) - -> canonical config fingerprint and bounded processor cache + -> reuse the matching active RequestProcessor, or fully prepare and + atomically activate a replacement + -> strict UTF-8 decode -> RequestProcessor.process(one text string) -> stage 1 engine.run(current text) -> stage 2 engine.run(stage 1 text) @@ -70,10 +71,18 @@ Blocking is a request-level disposition owned by `RequestProcessor`. The copied `proto/supervisor_middleware.proto` and generated bindings are owned by OpenShell. Update them only through the repository's middleware-kit workflow; never hand-edit them. Today's protocol carries a `google.protobuf.Struct` -configuration on each evaluation, so Privacy Guard validates and caches it -internally. Large-catalog preparation RPCs, evaluation fingerprints, manifest -schema fields, and a dedicated finding-source field require a coordinated -change in the canonical OpenShell protocol rather than a private proto fork. +configuration on each evaluation. Privacy Guard validates it every time and +retains one active configured processor. The first evaluation prepares that +processor. A different validated configuration is fully prepared and atomically +replaces it only after preparation succeeds; failure leaves the active processor +unchanged. + +The protocol has no explicit update or policy-version marker, so any changed +per-evaluation configuration is treated as an update. Each Privacy Guard process +must therefore receive one consistent policy stream. Large-catalog preparation +RPCs, versioned policy references, manifest schema fields, and a dedicated +finding-source field require a coordinated change in the canonical OpenShell +protocol rather than a private proto fork. ## Built-in engines diff --git a/projects/privacy-guard/src/privacy_guard/config.py b/projects/privacy-guard/src/privacy_guard/config.py index 7395ea9..0862880 100644 --- a/projects/privacy-guard/src/privacy_guard/config.py +++ b/projects/privacy-guard/src/privacy_guard/config.py @@ -7,9 +7,7 @@ from __future__ import annotations -import json from enum import StrEnum -from hashlib import sha256 from typing import Generic, Self, TypeVar from pydantic import ( @@ -113,25 +111,10 @@ class PrivacyGuardConfig( on_detection: OnDetection = Field(repr=False) -def configuration_fingerprint_and_size( - config: PrivacyGuardConfig[EngineConfig], -) -> tuple[str, int]: - """Return the canonical fingerprint and encoded size of an expanded config.""" - serialized = json.dumps( - config.model_dump(mode="json"), - allow_nan=False, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - return sha256(serialized).hexdigest(), len(serialized) - - __all__ = [ "EntityProcessingStage", "EntityProcessingStages", "OnDetection", "PolicyAction", "PrivacyGuardConfig", - "configuration_fingerprint_and_size", ] diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index 711617c..fbd4677 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -53,7 +53,6 @@ MAX_REGEX_PARSED_CATALOG_CACHE_BYTES = 8 * 1024 * 1024 MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES = 32 * 1024 * 1024 REGEX_COMPILED_RULE_WEIGHT_BYTES = 4 * 1024 -MAX_PROCESSOR_CACHE_WEIGHT_BYTES = 32 * 1024 * 1024 # Service concurrency and transport limits. MAX_CONCURRENT_PROCESSING = 4 diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index 5908ee2..5315765 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -604,13 +604,6 @@ def _compile_pattern_catalog( return rules -def _compiled_processor_weight(engines: tuple[RegexEngine, ...]) -> int: - return sum( - _compiled_pattern_weight(engine.config.pattern_catalog, len(engine._rules)) - for engine in engines - ) - - def _compiled_pattern_weight( catalog: RegexPatternCatalog, rule_count: int, diff --git a/projects/privacy-guard/src/privacy_guard/service/servicer.py b/projects/privacy-guard/src/privacy_guard/service/servicer.py index f53dca3..68b3058 100644 --- a/projects/privacy-guard/src/privacy_guard/service/servicer.py +++ b/projects/privacy-guard/src/privacy_guard/service/servicer.py @@ -1,4 +1,4 @@ -"""gRPC boundary for cached entity-processing configurations.""" +"""gRPC boundary for active entity-processing policy evaluation.""" from __future__ import annotations @@ -6,11 +6,9 @@ import json import math import time -from collections import OrderedDict from collections.abc import Callable, Iterable from concurrent.futures import Future, ThreadPoolExecutor -from dataclasses import dataclass -from threading import RLock +from threading import Lock from typing import Never, Protocol, TypedDict, TypeVar import grpc @@ -19,10 +17,7 @@ from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 from privacy_guard.bindings import supervisor_middleware_pb2_grpc as pb2_grpc -from privacy_guard.config import ( - PrivacyGuardConfig, - configuration_fingerprint_and_size, -) +from privacy_guard.config import PrivacyGuardConfig from privacy_guard.constants import ( BLOCK_REASON, BLOCK_REASON_CODE, @@ -31,7 +26,6 @@ LIMIT_REASON_CODE, MAX_BODY_BYTES, MAX_CONCURRENT_PROCESSING, - MAX_PROCESSOR_CACHE_WEIGHT_BYTES, MAX_PROTO_CONFIG_BYTES, MAX_PROTO_CONTEXT_BYTES, MAX_PROTO_FINDING_BYTES, @@ -44,7 +38,6 @@ SERVICE_VERSION, ) from privacy_guard.engines import EngineConfig -from privacy_guard.engines import regex as regex_engine from privacy_guard.engines.registry import EngineRegistry from privacy_guard.errors import ( EngineRegistryError, @@ -76,7 +69,7 @@ def __init__( if not registry.is_finalized: raise EngineRegistryError("middleware requires a finalized engine registry") self._registry = registry - self._processors = _RequestProcessorCache( + self._policy = _ActivePolicy( registry, timeout_seconds=validate_timeout_seconds(timeout_seconds), log_request_content=log_request_content, @@ -90,7 +83,7 @@ def __init__( async def close(self) -> None: """Wait for in-flight synchronous engines during shutdown.""" self._processing_executor.shutdown(wait=True, cancel_futures=True) - self._processors.clear() + self._policy.clear() async def Describe( self, @@ -211,7 +204,7 @@ def _prepare_and_process( body: bytes, ) -> RequestProcessingResult: values = _mapping_from_proto(config) - processor = self._processors.resolve(values) + processor = self._policy.processor_for(values) if not body: return RequestProcessingResult(decision=RequestDecision.ALLOW) try: @@ -236,20 +229,8 @@ async def _run_in_worker( return await asyncio.shield(future) -@dataclass(frozen=True) -class _PreparedProcessor: - processor: RequestProcessor - regex_engines: tuple[regex_engine.RegexEngine, ...] - - -@dataclass(frozen=True) -class _CachedProcessor: - processor: RequestProcessor - weight_bytes: int - - -class _RequestProcessorCache: - """Bounded, recoverable cache keyed by canonical expanded configuration.""" +class _ActivePolicy: + """Own the process's active policy and its prepared processor.""" def __init__( self, @@ -261,89 +242,25 @@ def __init__( self._registry = registry self._timeout_seconds = timeout_seconds self._log_request_content = log_request_content - self._processors: OrderedDict[str, _CachedProcessor] = OrderedDict() - self._weight_bytes = 0 - self._in_flight: dict[str, Future[RequestProcessor]] = {} - self._lock = RLock() + self._config: PrivacyGuardConfig[EngineConfig] | None = None + self._processor: RequestProcessor | None = None + self._lock = Lock() - def resolve(self, values: object) -> RequestProcessor: - """Return the cached or newly prepared processor for expanded config.""" + def processor_for(self, values: object) -> RequestProcessor: + """Return the processor for the requested policy, activating it if needed.""" config = self._registry.validate_config(values) - fingerprint, config_weight_bytes = configuration_fingerprint_and_size(config) - with self._lock: - cached = self._processors.get(fingerprint) - if cached is not None: - self._processors.move_to_end(fingerprint) - return cached.processor - owner_future = self._in_flight.get(fingerprint) - owns_build = owner_future is None - if owner_future is None: - owner_future = Future() - self._in_flight[fingerprint] = owner_future - if not owns_build: - return owner_future.result() - try: - prepared = self._build_processor(config) - except Exception as error: - owner_future.set_exception(error) - with self._lock: - if self._in_flight.get(fingerprint) is owner_future: - del self._in_flight[fingerprint] - raise - processor = prepared.processor - weight_bytes = config_weight_bytes + regex_engine._compiled_processor_weight( - prepared.regex_engines - ) - self._retain(fingerprint, processor, weight_bytes) - owner_future.set_result(processor) with self._lock: - if self._in_flight.get(fingerprint) is owner_future: - del self._in_flight[fingerprint] - return processor - - def _retain( - self, - fingerprint: str, - processor: RequestProcessor, - weight_bytes: int, - ) -> None: - if weight_bytes > MAX_PROCESSOR_CACHE_WEIGHT_BYTES: - _LOGGER.debug( - "privacy_guard_cache_skip cache=processor " - "weight_bytes=%d budget_bytes=%d", - weight_bytes, - MAX_PROCESSOR_CACHE_WEIGHT_BYTES, - ) - return - - evicted_entries = 0 - evicted_weight_bytes = 0 - with self._lock: - while ( - len(self._processors) >= _MAX_CACHED_PROCESSORS - or self._weight_bytes + weight_bytes > MAX_PROCESSOR_CACHE_WEIGHT_BYTES - ): - _, evicted = self._processors.popitem(last=False) - self._weight_bytes -= evicted.weight_bytes - evicted_entries += 1 - evicted_weight_bytes += evicted.weight_bytes - self._processors[fingerprint] = _CachedProcessor( - processor=processor, - weight_bytes=weight_bytes, - ) - self._weight_bytes += weight_bytes - if evicted_entries: - _LOGGER.debug( - "privacy_guard_cache_eviction cache=processor " - "entries=%d weight_bytes=%d", - evicted_entries, - evicted_weight_bytes, - ) + if config == self._config and self._processor is not None: + return self._processor + processor = self._build_processor(config) + self._config = config + self._processor = processor + return processor def _build_processor( self, config: PrivacyGuardConfig[EngineConfig], - ) -> _PreparedProcessor: + ) -> RequestProcessor: stages = tuple( ( stage.diagnostic_name(index), @@ -354,25 +271,18 @@ def _build_processor( start=1, ) ) - return _PreparedProcessor( - processor=RequestProcessor( - config, - stages, - timeout_seconds=self._timeout_seconds, - log_request_content=self._log_request_content, - ), - regex_engines=tuple( - engine - for _, engine in stages - if isinstance(engine, regex_engine.RegexEngine) - ), + return RequestProcessor( + config, + stages, + timeout_seconds=self._timeout_seconds, + log_request_content=self._log_request_content, ) def clear(self) -> None: - """Release every retained processor.""" + """Release the active policy.""" with self._lock: - self._processors.clear() - self._weight_bytes = 0 + self._config = None + self._processor = None _WorkerResultT = TypeVar("_WorkerResultT") @@ -541,7 +451,6 @@ def _limit_deny() -> pb2.HttpRequestResult: _LOGGER = get_logger(__name__) _INVALID_REQUEST_ID = "invalid" -_MAX_CACHED_PROCESSORS = 16 _MAX_PROTO_SAFE_INTEGER = (1 << 53) - 1 diff --git a/projects/privacy-guard/tests/engines/test_regex.py b/projects/privacy-guard/tests/engines/test_regex.py index acd29ed..9127124 100644 --- a/projects/privacy-guard/tests/engines/test_regex.py +++ b/projects/privacy-guard/tests/engines/test_regex.py @@ -455,19 +455,6 @@ def synchronized_compile( regex_module._clear_compiled_pattern_cache() -def test_processor_weight_counts_each_regex_stage() -> None: - config = _config([{"pattern": "shared", "confidence": "high"}]) - first = RegexEngine(config, None) - second = RegexEngine(config, None) - - assert regex_module._compiled_processor_weight( - (first, second) - ) == 2 * regex_module._compiled_pattern_weight( - first.config.pattern_catalog, - len(first._rules), - ) - - def test_regex_engine_is_safe_for_concurrent_runs() -> None: engine = RegexEngine( _config([{"pattern": "x", "confidence": "high"}]), diff --git a/projects/privacy-guard/tests/service/test_grpc_integration.py b/projects/privacy-guard/tests/service/test_grpc_integration.py index 2f8d684..b67a20c 100644 --- a/projects/privacy-guard/tests/service/test_grpc_integration.py +++ b/projects/privacy-guard/tests/service/test_grpc_integration.py @@ -208,16 +208,24 @@ async def test_generated_stub_maps_contextual_zero_width_to_invalid_config() -> middleware = PrivacyGuardMiddleware(create_builtin_registry()) async with _running_stub(middleware) as stub: + before = await stub.EvaluateHttpRequest( + _evaluation(b"contact a@b.com", action="detect") + ) validation = await stub.ValidateConfig(config) with pytest.raises(grpc.aio.AioRpcError) as evaluation_error: await stub.EvaluateHttpRequest(evaluation) + after = await stub.EvaluateHttpRequest( + _evaluation(b"contact a@b.com", action="detect") + ) details = evaluation_error.value.details() or "" + assert len(before.findings) == 1 assert validation.valid is True assert evaluation_error.value.code() is grpc.StatusCode.INVALID_ARGUMENT assert "config_invalid" in details assert "engine_execution_failed" not in details assert report_pattern not in details + assert len(after.findings) == 1 class _NumericNestedConfig(StrictDomainModel): diff --git a/projects/privacy-guard/tests/service/test_server.py b/projects/privacy-guard/tests/service/test_server.py index 2539740..3147cfe 100644 --- a/projects/privacy-guard/tests/service/test_server.py +++ b/projects/privacy-guard/tests/service/test_server.py @@ -88,8 +88,8 @@ async def record_serve(self: PrivacyGuardServer, listen: str) -> None: assert served == [(server, "127.0.0.1:50051")] assert server._middleware._registry is registry - assert server._middleware._processors._timeout_seconds == 4.5 - assert server._middleware._processors._log_request_content is True + assert server._middleware._policy._timeout_seconds == 4.5 + assert server._middleware._policy._log_request_content is True def test_programmatic_server_requires_an_explicit_finalized_registry() -> None: diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index c8165be..e48b9ce 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -4,7 +4,7 @@ import asyncio import logging -from concurrent.futures import Future, ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor from copy import deepcopy from threading import Barrier, Event, Lock, get_ident from typing import Never @@ -15,10 +15,7 @@ from google.protobuf.message import Message from privacy_guard.bindings import supervisor_middleware_pb2 as pb2 -from privacy_guard.config import ( - PrivacyGuardConfig, - configuration_fingerprint_and_size, -) +from privacy_guard.config import PrivacyGuardConfig from privacy_guard.constants import ( LIMIT_REASON, LIMIT_REASON_CODE, @@ -34,7 +31,7 @@ EngineConfig, ) from privacy_guard.engines import regex as regex_module -from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry +from privacy_guard.engines.registry import create_builtin_registry from privacy_guard.errors import ErrorCode, PrivacyGuardError from privacy_guard.request_processor import ( EntityDetectionSummary, @@ -85,25 +82,6 @@ def _values( } -def _expected_processor_cache_weight( - registry: EngineRegistry, - values: dict[str, object], -) -> int: - config = registry.validate_config(values) - _, config_weight = configuration_fingerprint_and_size(config) - regex_weight = sum( - regex_module._compiled_pattern_weight( - stage.config.pattern_catalog, - sum( - len(entity.patterns) for entity in stage.config.pattern_catalog.entities - ), - ) - for stage in config.entity_processing.stages - if isinstance(stage.config, regex_module.RegexEngineConfig) - ) - return config_weight + regex_weight - - def _proto_config(values: dict[str, object]) -> Message: result = pb2.ValidateConfigRequest().config json_format.ParseDict(values, result) @@ -124,26 +102,6 @@ async def abort(self, code: grpc.StatusCode, details: str) -> Never: raise AssertionError("successful evaluation unexpectedly aborted") -def _waiter_observing_future_type( - *, - expected_waiters: int, - waiters_ready: Event, -) -> type[Future[RequestProcessor]]: - waiter_count = 0 - waiter_count_lock = Lock() - - class _WaiterObservingFuture(Future[RequestProcessor]): - def result(self, timeout: float | None = None) -> RequestProcessor: - nonlocal waiter_count - with waiter_count_lock: - waiter_count += 1 - if waiter_count == expected_waiters: - waiters_ready.set() - return super().result(timeout) - - return _WaiterObservingFuture - - def test_copied_proto_remains_the_current_openshell_contract() -> None: evaluation = pb2.HttpRequestEvaluation() finding = pb2.Finding() @@ -153,19 +111,43 @@ def test_copied_proto_remains_the_current_openshell_contract() -> None: assert not hasattr(finding, "source") -def test_validate_config_is_pure_and_reports_invalid_config() -> None: +def test_validate_config_is_pure_and_reports_invalid_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: middleware = PrivacyGuardMiddleware(create_builtin_registry()) + active = middleware._policy.processor_for(_values(action="replace")) + processor_build_count = 0 + original_build = servicer_module._ActivePolicy._build_processor - valid = middleware._validate_config( - pb2.ValidateConfigRequest(config=_proto_config(_values())) - ) - invalid = middleware._validate_config( - pb2.ValidateConfigRequest(config=_proto_config({"on_detection": {}})) + def record_processor_build( + policy: servicer_module._ActivePolicy, + config: PrivacyGuardConfig[EngineConfig], + ) -> RequestProcessor: + nonlocal processor_build_count + processor_build_count += 1 + return original_build(policy, config) + + monkeypatch.setattr( + servicer_module._ActivePolicy, + "_build_processor", + record_processor_build, ) + try: + valid = middleware._validate_config( + pb2.ValidateConfigRequest(config=_proto_config(_values("detect"))) + ) + invalid = middleware._validate_config( + pb2.ValidateConfigRequest(config=_proto_config({"on_detection": {}})) + ) + still_active = middleware._policy.processor_for(_values(action="replace")) + finally: + asyncio.run(middleware.close()) assert valid.valid is True assert invalid.valid is False assert "config_invalid" in invalid.reason + assert still_active is active + assert processor_build_count == 0 def test_validate_config_rejects_oversized_proto_before_registry_validation( @@ -435,13 +417,13 @@ async def evaluate() -> None: assert records[0].getMessage().count(" action=") == 1 -def test_middleware_applies_configured_timeout_to_cached_processors() -> None: +def test_middleware_applies_configured_timeout_to_active_processor() -> None: middleware = PrivacyGuardMiddleware( create_builtin_registry(), timeout_seconds=4.5, ) try: - processor = middleware._processors.resolve(_values()) + processor = middleware._policy.processor_for(_values()) finally: asyncio.run(middleware.close()) @@ -471,19 +453,19 @@ def test_evaluation_prepares_configuration_off_the_event_loop( ) -> None: event_loop_thread = get_ident() preparation_threads: list[int] = [] - original_resolve = servicer_module._RequestProcessorCache.resolve + original_processor_for = servicer_module._ActivePolicy.processor_for - def record_resolve( - cache: servicer_module._RequestProcessorCache, + def record_preparation( + policy: servicer_module._ActivePolicy, values: object, ) -> RequestProcessor: preparation_threads.append(get_ident()) - return original_resolve(cache, values) + return original_processor_for(policy, values) monkeypatch.setattr( - servicer_module._RequestProcessorCache, - "resolve", - record_resolve, + servicer_module._ActivePolicy, + "processor_for", + record_preparation, ) async def evaluate() -> None: @@ -499,7 +481,7 @@ async def evaluate() -> None: assert preparation_threads[0] != event_loop_thread -def test_evaluation_revalidates_configuration_before_reusing_cached_processor( +def test_evaluation_revalidates_configuration_before_reusing_active_processor( monkeypatch: pytest.MonkeyPatch, ) -> None: validation_count = 0 @@ -533,361 +515,202 @@ async def evaluate_twice() -> None: assert validation_count == 2 -def test_same_fingerprint_misses_build_one_shared_processor( - monkeypatch: pytest.MonkeyPatch, -) -> None: - worker_count = 4 - workers_ready = Barrier(worker_count) - waiters_ready = Event() - build_count = 0 - build_count_lock = Lock() - original_build = servicer_module._RequestProcessorCache._build_processor - - def synchronized_build( - cache: servicer_module._RequestProcessorCache, - config: PrivacyGuardConfig[EngineConfig], - ) -> servicer_module._PreparedProcessor: - nonlocal build_count - with build_count_lock: - build_count += 1 - assert waiters_ready.wait(timeout=5) - return original_build(cache, config) - - monkeypatch.setattr( - servicer_module, - "Future", - _waiter_observing_future_type( - expected_waiters=worker_count - 1, - waiters_ready=waiters_ready, - ), - ) - monkeypatch.setattr( - servicer_module._RequestProcessorCache, - "_build_processor", - synchronized_build, - ) - registry = create_builtin_registry() - cache = servicer_module._RequestProcessorCache( - registry, +def test_active_policy_reuses_only_the_current_configuration() -> None: + policy = servicer_module._ActivePolicy( + create_builtin_registry(), timeout_seconds=1, log_request_content=False, ) + first_values = _values(action="detect") + second_values = _values(action="block") - def resolve() -> RequestProcessor: - workers_ready.wait(timeout=5) - return cache.resolve(_values()) + first = policy.processor_for(first_values) + same = policy.processor_for(deepcopy(first_values)) + second = policy.processor_for(second_values) + rebuilt_first = policy.processor_for(first_values) - with ThreadPoolExecutor(max_workers=worker_count) as executor: - processors = tuple(executor.map(lambda _: resolve(), range(worker_count))) + assert same is first + assert second is not first + assert rebuilt_first is not first + assert rebuilt_first is not second - assert build_count == 1 - assert all(processor is processors[0] for processor in processors) - assert cache._weight_bytes == _expected_processor_cache_weight( - registry, - _values(), - ) - assert cache._in_flight == {} - cache.clear() - -def test_different_fingerprints_build_concurrently( - monkeypatch: pytest.MonkeyPatch, -) -> None: - builds_ready = Barrier(2) - original_build = servicer_module._RequestProcessorCache._build_processor - - def synchronized_build( - cache: servicer_module._RequestProcessorCache, - config: PrivacyGuardConfig[EngineConfig], - ) -> servicer_module._PreparedProcessor: - builds_ready.wait(timeout=5) - return original_build(cache, config) - - monkeypatch.setattr( - servicer_module._RequestProcessorCache, - "_build_processor", - synchronized_build, - ) - registry = create_builtin_registry() - cache = servicer_module._RequestProcessorCache( - registry, - timeout_seconds=1, - log_request_content=False, - ) - with ThreadPoolExecutor(max_workers=2) as executor: - processors = tuple( - executor.map( - cache.resolve, - (_values(action="detect"), _values(action="block")), - ) - ) - - assert processors[0] is not processors[1] - assert len(cache._processors) == 2 - values = (_values(action="detect"), _values(action="block")) - expected_weight = sum( - _expected_processor_cache_weight(registry, item) for item in values - ) - assert cache._weight_bytes == expected_weight - assert cache._in_flight == {} - cache.clear() - - -def test_failed_processor_build_wakes_waiters_and_allows_retry( +@pytest.mark.parametrize("initial_action", [None, "detect"]) +def test_concurrent_requests_for_the_same_policy_build_once( monkeypatch: pytest.MonkeyPatch, + initial_action: str | None, ) -> None: worker_count = 4 workers_ready = Barrier(worker_count) - waiters_ready = Event() + build_started = Event() + release_build = Event() build_count = 0 build_count_lock = Lock() - failure = PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) - original_build = servicer_module._RequestProcessorCache._build_processor + original_build = servicer_module._ActivePolicy._build_processor + policy = servicer_module._ActivePolicy( + create_builtin_registry(), + timeout_seconds=1, + log_request_content=False, + ) + initial = ( + policy.processor_for(_values(action=initial_action)) + if initial_action is not None + else None + ) + requested_values = _values( + action="block" if initial_action is not None else "detect" + ) - def fail_first_build( - cache: servicer_module._RequestProcessorCache, + def pause_build( + active_policy: servicer_module._ActivePolicy, config: PrivacyGuardConfig[EngineConfig], - ) -> servicer_module._PreparedProcessor: + ) -> RequestProcessor: nonlocal build_count with build_count_lock: build_count += 1 - current_build = build_count - if current_build == 1: - assert waiters_ready.wait(timeout=5) - raise failure - return original_build(cache, config) + build_started.set() + assert release_build.wait(timeout=5) + return original_build(active_policy, config) monkeypatch.setattr( - servicer_module, - "Future", - _waiter_observing_future_type( - expected_waiters=worker_count - 1, - waiters_ready=waiters_ready, - ), - ) - monkeypatch.setattr( - servicer_module._RequestProcessorCache, + servicer_module._ActivePolicy, "_build_processor", - fail_first_build, - ) - cache = servicer_module._RequestProcessorCache( - create_builtin_registry(), - timeout_seconds=1, - log_request_content=False, + pause_build, ) - def capture_failure() -> PrivacyGuardError: + def resolve_policy() -> RequestProcessor: workers_ready.wait(timeout=5) - with pytest.raises(PrivacyGuardError) as captured: - cache.resolve(_values()) - return captured.value + return policy.processor_for(requested_values) with ThreadPoolExecutor(max_workers=worker_count) as executor: - failures = tuple(executor.map(lambda _: capture_failure(), range(worker_count))) + futures = tuple(executor.submit(resolve_policy) for _ in range(worker_count)) + assert build_started.wait(timeout=5) + assert all(not future.done() for future in futures) + release_build.set() + processors = tuple(future.result(timeout=5) for future in futures) assert build_count == 1 - assert all(error is failure for error in failures) - assert cache._processors == {} - assert cache._weight_bytes == 0 - assert cache._in_flight == {} - - processor = cache.resolve(_values()) - - assert isinstance(processor, RequestProcessor) - assert build_count == 2 - assert len(cache._processors) == 1 - assert cache._weight_bytes > 0 - assert cache._in_flight == {} - cache.clear() + assert all(processor is processors[0] for processor in processors) + assert processors[0] is not initial + assert policy.processor_for(requested_values) is processors[0] -def test_processor_cache_evicts_least_recently_used_processor_by_weight( +def test_different_policy_updates_are_serialized( monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, ) -> None: - registry = create_builtin_registry() - values = tuple( - _values(action="detect", stage_name=f"sensitive-stage-{suffix}") - for suffix in ("a", "b", "c") - ) - identities = tuple( - configuration_fingerprint_and_size(registry.validate_config(item)) - for item in values - ) - cache = servicer_module._RequestProcessorCache( - registry, + first_build_started = Event() + release_first_build = Event() + second_build_started = Event() + build_actions: list[str] = [] + active_builds = 0 + maximum_active_builds = 0 + build_count_lock = Lock() + original_build = servicer_module._ActivePolicy._build_processor + policy = servicer_module._ActivePolicy( + create_builtin_registry(), timeout_seconds=1, log_request_content=False, ) - first = cache.resolve(values[0]) - entry_weight = cache._processors[identities[0][0]].weight_bytes - assert entry_weight > identities[0][1] + initial = policy.processor_for(_values(action="detect")) + + def control_build( + active_policy: servicer_module._ActivePolicy, + config: PrivacyGuardConfig[EngineConfig], + ) -> RequestProcessor: + nonlocal active_builds, maximum_active_builds + action = config.on_detection.action.value + with build_count_lock: + active_builds += 1 + maximum_active_builds = max(maximum_active_builds, active_builds) + build_actions.append(action) + try: + if action == "block": + first_build_started.set() + assert release_first_build.wait(timeout=5) + elif action == "replace": + second_build_started.set() + return original_build(active_policy, config) + finally: + with build_count_lock: + active_builds -= 1 + monkeypatch.setattr( - servicer_module, - "MAX_PROCESSOR_CACHE_WEIGHT_BYTES", - entry_weight * 2, + servicer_module._ActivePolicy, + "_build_processor", + control_build, ) - with caplog.at_level(logging.DEBUG, logger="privacy_guard.service.servicer"): - cache.resolve(values[1]) - assert cache.resolve(values[0]) is first - cache.resolve(values[2]) + with ThreadPoolExecutor(max_workers=2) as executor: + first_update = executor.submit(policy.processor_for, _values(action="block")) + assert first_build_started.wait(timeout=5) + second_update = executor.submit( + policy.processor_for, + _values(action="replace"), + ) + assert not second_build_started.wait(timeout=0.1) + release_first_build.set() + first_processor = first_update.result(timeout=5) + second_processor = second_update.result(timeout=5) - assert tuple(cache._processors) == (identities[0][0], identities[2][0]) - assert cache._weight_bytes == entry_weight * 2 - assert "privacy_guard_cache_eviction cache=processor entries=1" in caplog.text - assert "sensitive-stage" not in caplog.text - cache.clear() + assert second_build_started.is_set() + assert build_actions == ["block", "replace"] + assert maximum_active_builds == 1 + assert first_processor is not initial + assert second_processor is not first_processor + assert policy.processor_for(_values(action="replace")) is second_processor -def test_processor_cache_evicts_least_recently_used_processor_by_count( +def test_failed_policy_update_preserves_the_active_processor( monkeypatch: pytest.MonkeyPatch, ) -> None: - registry = create_builtin_registry() - values = tuple( - _values(action="detect", stage_name=f"stage-{suffix}") - for suffix in ("a", "b", "c") - ) - fingerprints = tuple( - configuration_fingerprint_and_size(registry.validate_config(item))[0] - for item in values - ) - monkeypatch.setattr(servicer_module, "_MAX_CACHED_PROCESSORS", 2) - cache = servicer_module._RequestProcessorCache( - registry, + failure = PrivacyGuardError(ErrorCode.UNEXPECTED_SERVICE_FAILURE) + original_build = servicer_module._ActivePolicy._build_processor + policy = servicer_module._ActivePolicy( + create_builtin_registry(), timeout_seconds=1, log_request_content=False, ) + active_values = _values(action="detect") + update_values = _values(action="block") + active = policy.processor_for(active_values) - first = cache.resolve(values[0]) - cache.resolve(values[1]) - assert cache.resolve(values[0]) is first - cache.resolve(values[2]) - - assert tuple(cache._processors) == (fingerprints[0], fingerprints[2]) - assert cache._weight_bytes == sum( - entry.weight_bytes for entry in cache._processors.values() - ) - cache.clear() - - -def test_processor_cache_builds_but_does_not_retain_oversized_processor( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - registry = create_builtin_registry() - values = _values(action="detect", stage_name="sensitive-oversized-stage") - monkeypatch.setattr( - servicer_module, - "MAX_PROCESSOR_CACHE_WEIGHT_BYTES", - 1, - ) - build_count = 0 - original_build = servicer_module._RequestProcessorCache._build_processor - - def record_build( - cache: servicer_module._RequestProcessorCache, + def fail_update( + active_policy: servicer_module._ActivePolicy, config: PrivacyGuardConfig[EngineConfig], - ) -> servicer_module._PreparedProcessor: - nonlocal build_count - build_count += 1 - return original_build(cache, config) + ) -> RequestProcessor: + if config.on_detection.action.value == "block": + raise failure + return original_build(active_policy, config) monkeypatch.setattr( - servicer_module._RequestProcessorCache, + servicer_module._ActivePolicy, "_build_processor", - record_build, - ) - cache = servicer_module._RequestProcessorCache( - registry, - timeout_seconds=1, - log_request_content=False, + fail_update, ) - with caplog.at_level(logging.DEBUG, logger="privacy_guard.service.servicer"): - first = cache.resolve(values) - second = cache.resolve(values) - - assert first is not second - assert build_count == 2 - assert cache._processors == {} - assert cache._weight_bytes == 0 - assert caplog.text.count("privacy_guard_cache_skip cache=processor") == 2 - assert "sensitive-oversized-stage" not in caplog.text - cache.clear() - - -def test_oversized_same_fingerprint_misses_remain_single_flight( - monkeypatch: pytest.MonkeyPatch, -) -> None: - publication_started = Event() - waiter_ready = Event() - release_publication = Event() - - class _PublicationPausingFuture(Future[RequestProcessor]): - def result(self, timeout: float | None = None) -> RequestProcessor: - waiter_ready.set() - return super().result(timeout) - - def set_result(self, result: RequestProcessor) -> None: - publication_started.set() - assert release_publication.wait(timeout=5) - super().set_result(result) - - registry = create_builtin_registry() - values = _values(action="detect", stage_name="oversized-single-flight") - monkeypatch.setattr( - servicer_module, - "MAX_PROCESSOR_CACHE_WEIGHT_BYTES", - 1, - ) - monkeypatch.setattr(servicer_module, "Future", _PublicationPausingFuture) - build_count = 0 - original_build = servicer_module._RequestProcessorCache._build_processor + with pytest.raises(PrivacyGuardError) as captured: + policy.processor_for(update_values) - def record_build( - cache: servicer_module._RequestProcessorCache, - config: PrivacyGuardConfig[EngineConfig], - ) -> servicer_module._PreparedProcessor: - nonlocal build_count - build_count += 1 - return original_build(cache, config) + assert captured.value is failure + assert policy.processor_for(active_values) is active monkeypatch.setattr( - servicer_module._RequestProcessorCache, + servicer_module._ActivePolicy, "_build_processor", - record_build, + original_build, ) - cache = servicer_module._RequestProcessorCache( - registry, - timeout_seconds=1, - log_request_content=False, - ) - - with ThreadPoolExecutor(max_workers=2) as executor: - owner = executor.submit(cache.resolve, values) - assert publication_started.wait(timeout=5) - waiter = executor.submit(cache.resolve, values) - try: - assert waiter_ready.wait(timeout=5) - finally: - release_publication.set() - processors = (owner.result(timeout=5), waiter.result(timeout=5)) + updated = policy.processor_for(update_values) - assert processors[0] is processors[1] - assert build_count == 1 - assert cache._processors == {} - assert cache._weight_bytes == 0 - assert cache._in_flight == {} - cache.clear() + assert updated is not active + assert policy.processor_for(update_values) is updated -def test_compiled_cache_eviction_does_not_invalidate_cached_processor( +def test_compiled_cache_eviction_does_not_invalidate_active_processor( monkeypatch: pytest.MonkeyPatch, ) -> None: regex_module._clear_compiled_pattern_cache() registry = create_builtin_registry() - cache = servicer_module._RequestProcessorCache( + policy = servicer_module._ActivePolicy( registry, timeout_seconds=1, log_request_content=False, @@ -902,7 +725,7 @@ def test_compiled_cache_eviction_does_not_invalidate_cached_processor( ) try: - processor = cache.resolve(processor_values) + processor = policy.processor_for(processor_values) entry_weight = regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES monkeypatch.setattr( regex_module, @@ -914,75 +737,29 @@ def test_compiled_cache_eviction_does_not_invalidate_cached_processor( result = processor.process("aaa") assert len(result.detection_summaries) == 1 - assert cache.resolve(processor_values) is processor + assert policy.processor_for(processor_values) is processor assert regex_module._COMPILED_PATTERN_CACHE_WEIGHT_BYTES <= entry_weight finally: - cache.clear() + policy.clear() regex_module._clear_compiled_pattern_cache() -def test_oversized_regex_processor_preserves_cached_processor( - monkeypatch: pytest.MonkeyPatch, -) -> None: - regex_module._clear_compiled_pattern_cache() - registry = create_builtin_registry() - cache = servicer_module._RequestProcessorCache( - registry, - timeout_seconds=1, - log_request_content=False, - ) - retained_values = _values( - "detect", - patterns=[{"pattern": "aaa", "confidence": "high"}], - ) - oversized_values = _values( - "detect", - patterns=[ - {"pattern": "bbb", "confidence": "high"}, - {"pattern": "ccc", "confidence": "high"}, - ], - ) - - try: - retained = cache.resolve(retained_values) - retained_fingerprint, _ = configuration_fingerprint_and_size( - registry.validate_config(retained_values) - ) - retained_weight = cache._processors[retained_fingerprint].weight_bytes - monkeypatch.setattr( - servicer_module, - "MAX_PROCESSOR_CACHE_WEIGHT_BYTES", - retained_weight + 1, - ) - - oversized = cache.resolve(oversized_values) - - assert oversized is not retained - assert tuple(cache._processors) == (retained_fingerprint,) - assert cache._processors[retained_fingerprint].processor is retained - assert cache._weight_bytes == retained_weight - finally: - cache.clear() - regex_module._clear_compiled_pattern_cache() - - -def test_middleware_shutdown_clears_processor_cache() -> None: +def test_middleware_shutdown_clears_active_policy() -> None: regex_module._clear_compiled_pattern_cache() middleware = PrivacyGuardMiddleware(create_builtin_registry()) try: - middleware._processors.resolve(_values("detect")) - assert middleware._processors._processors + middleware._policy.processor_for(_values("detect")) asyncio.run(middleware.close()) - assert middleware._processors._processors == {} - assert middleware._processors._weight_bytes == 0 + assert middleware._policy._config is None + assert middleware._policy._processor is None finally: - middleware._processors.clear() + middleware._policy.clear() regex_module._clear_compiled_pattern_cache() -def test_oversized_stage_list_fails_before_fingerprinting_or_construction( +def test_oversized_stage_list_fails_before_engine_construction( monkeypatch: pytest.MonkeyPatch, ) -> None: values = _values(action="detect", stage_count=10_000) @@ -991,11 +768,6 @@ def unexpected_call(*args: object, **kwargs: object) -> object: del args, kwargs raise AssertionError("oversized stage list reached preparation") - monkeypatch.setattr( - servicer_module, - "configuration_fingerprint_and_size", - unexpected_call, - ) monkeypatch.setattr( servicer_module.EngineRegistry, "create_engine", @@ -1004,7 +776,7 @@ def unexpected_call(*args: object, **kwargs: object) -> object: middleware = PrivacyGuardMiddleware(create_builtin_registry()) try: with pytest.raises(PrivacyGuardError) as captured: - middleware._processors.resolve(values) + middleware._policy.processor_for(values) finally: asyncio.run(middleware.close()) diff --git a/projects/privacy-guard/tests/test_cli.py b/projects/privacy-guard/tests/test_cli.py index eaa470b..5565a1d 100644 --- a/projects/privacy-guard/tests/test_cli.py +++ b/projects/privacy-guard/tests/test_cli.py @@ -186,8 +186,8 @@ def record_serve_sync(self: PrivacyGuardServer, listen: str) -> None: calls.append( ( listen, - self._middleware._processors._timeout_seconds, - self._middleware._processors._log_request_content, + self._middleware._policy._timeout_seconds, + self._middleware._policy._log_request_content, ) ) diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py index ad3a826..4b91784 100644 --- a/projects/privacy-guard/tests/test_config.py +++ b/projects/privacy-guard/tests/test_config.py @@ -15,7 +15,6 @@ import privacy_guard.engines.regex as regex_module from privacy_guard.config import ( PolicyAction, - configuration_fingerprint_and_size, ) from privacy_guard.engines import ( RegexEngine, @@ -182,9 +181,6 @@ def test_catalog_file_and_inline_catalog_produce_the_same_config( file_config = registry.validate_config(file_values) assert file_config == inline_config - assert configuration_fingerprint_and_size( - file_config - ) == configuration_fingerprint_and_size(inline_config) serialized_catalog = file_config.model_dump(mode="json")["entity_processing"][ "stages" ][0]["config"]["pattern_catalog"] @@ -195,7 +191,7 @@ def test_catalog_file_and_inline_catalog_produce_the_same_config( assert isinstance(serialized_catalog, dict) -def test_catalog_file_change_produces_a_new_fingerprint( +def test_catalog_file_change_produces_a_different_validated_config( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -214,9 +210,7 @@ def test_catalog_file_change_produces_a_new_fingerprint( catalog_path.write_text(yaml.safe_dump(catalog), encoding="utf-8") second = registry.validate_config(values) - assert configuration_fingerprint_and_size( - first - ) != configuration_fingerprint_and_size(second) + assert first != second def test_parsed_catalog_cache_evicts_least_recently_used_file_by_weight( @@ -655,7 +649,7 @@ def test_regex_pattern_names_are_optional_but_supplied_names_are_unique() -> Non _registry().validate_config(values) -def test_canonical_fingerprint_covers_concrete_expanded_config() -> None: +def test_validated_config_equality_covers_concrete_expanded_config() -> None: registry = _registry() first = registry.validate_config(_config()) equivalent = registry.validate_config(deepcopy(_config())) @@ -665,12 +659,8 @@ def test_canonical_fingerprint_covers_concrete_expanded_config() -> None: ][0]["patterns"][0]["confidence"] = "low" changed = registry.validate_config(changed_values) - assert configuration_fingerprint_and_size( - first - ) == configuration_fingerprint_and_size(equivalent) - assert configuration_fingerprint_and_size( - first - ) != configuration_fingerprint_and_size(changed) + assert first == equivalent + assert first != changed def test_models_are_frozen_and_hide_engine_configuration_from_repr() -> None: From aa0103c41055097ed583ade2eed47f4aa7b89d03 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 16:20:53 +0000 Subject: [PATCH 77/82] fix(privacy-guard): restore release artifact gate (PG-010 PG-011 PG-013) --- projects/privacy-guard/README.md | 5 +- projects/privacy-guard/scripts/check.sh | 35 +++++ .../privacy-guard/scripts/check_artifacts.py | 120 ++++++++++++++++++ .../scripts/external_engine_typing.py | 66 ++++++++++ .../privacy-guard/src/privacy_guard/py.typed | 1 + 5 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 projects/privacy-guard/scripts/check_artifacts.py create mode 100644 projects/privacy-guard/scripts/external_engine_typing.py create mode 100644 projects/privacy-guard/src/privacy_guard/py.typed diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index b84d5a1..e5dd85c 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -291,5 +291,6 @@ make check-py311 ``` `make check` delegates to `scripts/check.sh`, the authoritative local and CI -gate. It runs tests, formatting, lint, `ty`, an import smoke check, and a -dependency audit. +gate. It runs tests, formatting, lint, `ty`, an import smoke check, package and +artifact validation, an installed-wheel custom-engine type check, and runtime +and development dependency audits. diff --git a/projects/privacy-guard/scripts/check.sh b/projects/privacy-guard/scripts/check.sh index c82e637..965d252 100755 --- a/projects/privacy-guard/scripts/check.sh +++ b/projects/privacy-guard/scripts/check.sh @@ -4,12 +4,14 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." uv_run=(uv run --frozen) +artifact_python=() if [[ $# -gt 0 ]]; then if [[ $1 != "--python" || $# -ne 2 ]]; then echo "usage: scripts/check.sh [--python VERSION]" >&2 exit 2 fi uv_run+=(--python "$2") + artifact_python+=(--python "$2") fi "${uv_run[@]}" pytest -q @@ -17,6 +19,39 @@ fi "${uv_run[@]}" ruff check . "${uv_run[@]}" ty check "${uv_run[@]}" python -c "import privacy_guard" +uv build +"${uv_run[@]}" python scripts/check_artifacts.py + +artifact_check_dir=$(mktemp -d) +trap 'rm -rf "$artifact_check_dir"' EXIT +uv venv --clear "${artifact_python[@]}" "$artifact_check_dir/venv" + +wheels=(dist/*.whl) +if [[ ${#wheels[@]} -ne 1 ]]; then + echo "expected one built wheel, found ${#wheels[@]}" >&2 + exit 1 +fi + +VIRTUAL_ENV="$artifact_check_dir/venv" \ + uv sync --frozen --no-dev --no-install-project --active +uv pip install \ + --offline \ + --no-deps \ + --python "$artifact_check_dir/venv/bin/python" \ + "${wheels[0]}" +runtime_site_packages=$( + "$artifact_check_dir/venv/bin/python" \ + -c 'import sysconfig; print(sysconfig.get_path("purelib"))' +) +"${uv_run[@]}" pip-audit \ + --progress-spinner off \ + --path "$runtime_site_packages" + +cp scripts/external_engine_typing.py "$artifact_check_dir/external_engine_typing.py" +"${uv_run[@]}" ty check \ + --project "$artifact_check_dir" \ + --python "$artifact_check_dir/venv" \ + "$artifact_check_dir/external_engine_typing.py" "${uv_run[@]}" pip-audit \ --progress-spinner off \ --local diff --git a/projects/privacy-guard/scripts/check_artifacts.py b/projects/privacy-guard/scripts/check_artifacts.py new file mode 100644 index 0000000..291f76c --- /dev/null +++ b/projects/privacy-guard/scripts/check_artifacts.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Check the release details that the package builder cannot validate.""" + +from __future__ import annotations + +import tarfile +import zipfile +from collections.abc import Iterable +from email import policy +from email.message import Message +from email.parser import BytesParser +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +REPOSITORY_ROOT = PROJECT_ROOT.parent.parent +DIST_DIRECTORY = PROJECT_ROOT / "dist" +README_LINKS = { + "https://github.com/NVIDIA/OpenShell-Research/blob/main/" + "projects/privacy-guard/examples/regex-engine/README.md": ( + REPOSITORY_ROOT / "projects/privacy-guard/examples/regex-engine/README.md" + ), + "https://github.com/NVIDIA/OpenShell-Research/blob/main/" + "projects/privacy-guard/examples/custom-engine/README.md": ( + REPOSITORY_ROOT / "projects/privacy-guard/examples/custom-engine/README.md" + ), + "https://github.com/NVIDIA/OpenShell-Research/blob/main/" + "projects/openshell-middleware-kit/README.md": ( + REPOSITORY_ROOT / "projects/openshell-middleware-kit/README.md" + ), +} + + +def main() -> None: + """Inspect the one wheel and source distribution produced by ``uv build``.""" + wheel = _one(DIST_DIRECTORY.glob("privacy_guard-*.whl"), "wheel") + source = _one(DIST_DIRECTORY.glob("privacy_guard-*.tar.gz"), "sdist") + missing_targets = [ + str(path) for path in README_LINKS.values() if not path.is_file() + ] + if missing_targets: + raise SystemExit(f"README targets do not exist: {', '.join(missing_targets)}") + _check_wheel(wheel) + _check_sdist(source) + + +def _one(paths: Iterable[Path], description: str) -> Path: + matches = tuple(paths) + if len(matches) != 1: + raise SystemExit(f"expected one {description}, found {len(matches)}") + return matches[0] + + +def _check_wheel(path: Path) -> None: + with zipfile.ZipFile(path) as archive: + names = archive.namelist() + metadata_name = _member_ending(names, ".dist-info/METADATA") + license_name = _member_ending(names, ".dist-info/licenses/LICENSE") + _require_member(names, "privacy_guard/py.typed", path) + _require_license(archive.read(license_name), path) + _require_metadata(archive.read(metadata_name), path) + + +def _check_sdist(path: Path) -> None: + with tarfile.open(path) as archive: + names = archive.getnames() + metadata_name = _member_ending(names, "/PKG-INFO") + license_name = _member_ending(names, "/LICENSE") + readme_name = _member_ending(names, "/README.md") + _member_ending(names, "/src/privacy_guard/py.typed") + _require_license(_read_tar(archive, license_name), path) + _require_metadata(_read_tar(archive, metadata_name), path) + _require_readme(_read_tar(archive, readme_name).decode(), path) + + +def _member_ending(names: list[str], suffix: str) -> str: + matches = [name for name in names if name.endswith(suffix)] + if len(matches) != 1: + raise SystemExit(f"expected one archive member ending in {suffix!r}") + return matches[0] + + +def _require_member(names: list[str], member: str, artifact: Path) -> None: + if member not in names: + raise SystemExit(f"{artifact.name} is missing {member}") + + +def _require_license(contents: bytes, artifact: Path) -> None: + if contents != (PROJECT_ROOT / "LICENSE").read_bytes(): + raise SystemExit(f"{artifact.name} does not contain the exact LICENSE") + + +def _require_metadata(contents: bytes, artifact: Path) -> None: + metadata: Message = BytesParser(policy=policy.default).parsebytes(contents) + if metadata["License-Expression"] != "Apache-2.0": + raise SystemExit(f"{artifact.name} has the wrong license expression") + if "LICENSE" not in (metadata.get_all("License-File") or []): + raise SystemExit(f"{artifact.name} lacks License-File: LICENSE") + description = metadata.get_payload() + if not isinstance(description, str): + raise SystemExit(f"{artifact.name} has no text README") + _require_readme(description, artifact) + + +def _require_readme(contents: str, artifact: Path) -> None: + missing = [url for url in README_LINKS if url not in contents] + if missing: + raise SystemExit( + f"{artifact.name} README is missing canonical links: {', '.join(missing)}" + ) + + +def _read_tar(archive: tarfile.TarFile, member: str) -> bytes: + file = archive.extractfile(member) + if file is None: + raise SystemExit(f"could not read {member}") + return file.read() + + +if __name__ == "__main__": + main() diff --git a/projects/privacy-guard/scripts/external_engine_typing.py b/projects/privacy-guard/scripts/external_engine_typing.py new file mode 100644 index 0000000..3089295 --- /dev/null +++ b/projects/privacy-guard/scripts/external_engine_typing.py @@ -0,0 +1,66 @@ +"""External consumer fixture checked only against an installed wheel.""" + +from __future__ import annotations + +from typing import Literal, assert_type + +from privacy_guard.engines import ( + EngineConfig, + EngineResources, + EntityProcessingEngine, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.timeout import Timeout + + +class ExternalConfig(EngineConfig): + engine: Literal["external"] = "external" + keyword: str + + +class ExternalClient: + def find(self, text: str, keyword: str) -> bool: + return keyword in text + + +class ExternalResources(EngineResources): + __slots__ = ("client",) + + def __init__(self, client: ExternalClient) -> None: + self.client = client + + +class ExternalEngine(EntityProcessingEngine[ExternalConfig, ExternalResources]): + supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + assert_type(self.config, ExternalConfig) + assert_type(self.config.keyword, str) + assert_type(self.resources, ExternalResources) + assert_type(self.resources.client, ExternalClient) + self.resources.client.find(text, self.config.keyword) + result = TextProcessingResult(text=text, detections=()) + assert_type(result, TextProcessingResult) + return result + + +engine = ExternalEngine( + ExternalConfig(keyword="secret"), + ExternalResources(ExternalClient()), +) +assert_type(engine.config, ExternalConfig) +assert_type(engine.resources, ExternalResources) +processed = engine.run( + "text", + strategy=EntityProcessingStrategy.DETECT, + timeout=Timeout.from_seconds(1), +) +assert_type(processed, TextProcessingResult) +assert_type(processed.text, str) diff --git a/projects/privacy-guard/src/privacy_guard/py.typed b/projects/privacy-guard/src/privacy_guard/py.typed new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/py.typed @@ -0,0 +1 @@ + From 00bc7fa4ea664afb04a535e2533e4b3630e41bed Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 16:23:14 +0000 Subject: [PATCH 78/82] chore(privacy-guard): keep release checks streamlined --- projects/privacy-guard/README.md | 5 +- projects/privacy-guard/scripts/check.sh | 35 ----- .../privacy-guard/scripts/check_artifacts.py | 120 ------------------ .../scripts/external_engine_typing.py | 66 ---------- .../privacy-guard/src/privacy_guard/py.typed | 1 - 5 files changed, 2 insertions(+), 225 deletions(-) delete mode 100644 projects/privacy-guard/scripts/check_artifacts.py delete mode 100644 projects/privacy-guard/scripts/external_engine_typing.py delete mode 100644 projects/privacy-guard/src/privacy_guard/py.typed diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index e5dd85c..b84d5a1 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -291,6 +291,5 @@ make check-py311 ``` `make check` delegates to `scripts/check.sh`, the authoritative local and CI -gate. It runs tests, formatting, lint, `ty`, an import smoke check, package and -artifact validation, an installed-wheel custom-engine type check, and runtime -and development dependency audits. +gate. It runs tests, formatting, lint, `ty`, an import smoke check, and a +dependency audit. diff --git a/projects/privacy-guard/scripts/check.sh b/projects/privacy-guard/scripts/check.sh index 965d252..c82e637 100755 --- a/projects/privacy-guard/scripts/check.sh +++ b/projects/privacy-guard/scripts/check.sh @@ -4,14 +4,12 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." uv_run=(uv run --frozen) -artifact_python=() if [[ $# -gt 0 ]]; then if [[ $1 != "--python" || $# -ne 2 ]]; then echo "usage: scripts/check.sh [--python VERSION]" >&2 exit 2 fi uv_run+=(--python "$2") - artifact_python+=(--python "$2") fi "${uv_run[@]}" pytest -q @@ -19,39 +17,6 @@ fi "${uv_run[@]}" ruff check . "${uv_run[@]}" ty check "${uv_run[@]}" python -c "import privacy_guard" -uv build -"${uv_run[@]}" python scripts/check_artifacts.py - -artifact_check_dir=$(mktemp -d) -trap 'rm -rf "$artifact_check_dir"' EXIT -uv venv --clear "${artifact_python[@]}" "$artifact_check_dir/venv" - -wheels=(dist/*.whl) -if [[ ${#wheels[@]} -ne 1 ]]; then - echo "expected one built wheel, found ${#wheels[@]}" >&2 - exit 1 -fi - -VIRTUAL_ENV="$artifact_check_dir/venv" \ - uv sync --frozen --no-dev --no-install-project --active -uv pip install \ - --offline \ - --no-deps \ - --python "$artifact_check_dir/venv/bin/python" \ - "${wheels[0]}" -runtime_site_packages=$( - "$artifact_check_dir/venv/bin/python" \ - -c 'import sysconfig; print(sysconfig.get_path("purelib"))' -) -"${uv_run[@]}" pip-audit \ - --progress-spinner off \ - --path "$runtime_site_packages" - -cp scripts/external_engine_typing.py "$artifact_check_dir/external_engine_typing.py" -"${uv_run[@]}" ty check \ - --project "$artifact_check_dir" \ - --python "$artifact_check_dir/venv" \ - "$artifact_check_dir/external_engine_typing.py" "${uv_run[@]}" pip-audit \ --progress-spinner off \ --local diff --git a/projects/privacy-guard/scripts/check_artifacts.py b/projects/privacy-guard/scripts/check_artifacts.py deleted file mode 100644 index 291f76c..0000000 --- a/projects/privacy-guard/scripts/check_artifacts.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -"""Check the release details that the package builder cannot validate.""" - -from __future__ import annotations - -import tarfile -import zipfile -from collections.abc import Iterable -from email import policy -from email.message import Message -from email.parser import BytesParser -from pathlib import Path - -PROJECT_ROOT = Path(__file__).resolve().parent.parent -REPOSITORY_ROOT = PROJECT_ROOT.parent.parent -DIST_DIRECTORY = PROJECT_ROOT / "dist" -README_LINKS = { - "https://github.com/NVIDIA/OpenShell-Research/blob/main/" - "projects/privacy-guard/examples/regex-engine/README.md": ( - REPOSITORY_ROOT / "projects/privacy-guard/examples/regex-engine/README.md" - ), - "https://github.com/NVIDIA/OpenShell-Research/blob/main/" - "projects/privacy-guard/examples/custom-engine/README.md": ( - REPOSITORY_ROOT / "projects/privacy-guard/examples/custom-engine/README.md" - ), - "https://github.com/NVIDIA/OpenShell-Research/blob/main/" - "projects/openshell-middleware-kit/README.md": ( - REPOSITORY_ROOT / "projects/openshell-middleware-kit/README.md" - ), -} - - -def main() -> None: - """Inspect the one wheel and source distribution produced by ``uv build``.""" - wheel = _one(DIST_DIRECTORY.glob("privacy_guard-*.whl"), "wheel") - source = _one(DIST_DIRECTORY.glob("privacy_guard-*.tar.gz"), "sdist") - missing_targets = [ - str(path) for path in README_LINKS.values() if not path.is_file() - ] - if missing_targets: - raise SystemExit(f"README targets do not exist: {', '.join(missing_targets)}") - _check_wheel(wheel) - _check_sdist(source) - - -def _one(paths: Iterable[Path], description: str) -> Path: - matches = tuple(paths) - if len(matches) != 1: - raise SystemExit(f"expected one {description}, found {len(matches)}") - return matches[0] - - -def _check_wheel(path: Path) -> None: - with zipfile.ZipFile(path) as archive: - names = archive.namelist() - metadata_name = _member_ending(names, ".dist-info/METADATA") - license_name = _member_ending(names, ".dist-info/licenses/LICENSE") - _require_member(names, "privacy_guard/py.typed", path) - _require_license(archive.read(license_name), path) - _require_metadata(archive.read(metadata_name), path) - - -def _check_sdist(path: Path) -> None: - with tarfile.open(path) as archive: - names = archive.getnames() - metadata_name = _member_ending(names, "/PKG-INFO") - license_name = _member_ending(names, "/LICENSE") - readme_name = _member_ending(names, "/README.md") - _member_ending(names, "/src/privacy_guard/py.typed") - _require_license(_read_tar(archive, license_name), path) - _require_metadata(_read_tar(archive, metadata_name), path) - _require_readme(_read_tar(archive, readme_name).decode(), path) - - -def _member_ending(names: list[str], suffix: str) -> str: - matches = [name for name in names if name.endswith(suffix)] - if len(matches) != 1: - raise SystemExit(f"expected one archive member ending in {suffix!r}") - return matches[0] - - -def _require_member(names: list[str], member: str, artifact: Path) -> None: - if member not in names: - raise SystemExit(f"{artifact.name} is missing {member}") - - -def _require_license(contents: bytes, artifact: Path) -> None: - if contents != (PROJECT_ROOT / "LICENSE").read_bytes(): - raise SystemExit(f"{artifact.name} does not contain the exact LICENSE") - - -def _require_metadata(contents: bytes, artifact: Path) -> None: - metadata: Message = BytesParser(policy=policy.default).parsebytes(contents) - if metadata["License-Expression"] != "Apache-2.0": - raise SystemExit(f"{artifact.name} has the wrong license expression") - if "LICENSE" not in (metadata.get_all("License-File") or []): - raise SystemExit(f"{artifact.name} lacks License-File: LICENSE") - description = metadata.get_payload() - if not isinstance(description, str): - raise SystemExit(f"{artifact.name} has no text README") - _require_readme(description, artifact) - - -def _require_readme(contents: str, artifact: Path) -> None: - missing = [url for url in README_LINKS if url not in contents] - if missing: - raise SystemExit( - f"{artifact.name} README is missing canonical links: {', '.join(missing)}" - ) - - -def _read_tar(archive: tarfile.TarFile, member: str) -> bytes: - file = archive.extractfile(member) - if file is None: - raise SystemExit(f"could not read {member}") - return file.read() - - -if __name__ == "__main__": - main() diff --git a/projects/privacy-guard/scripts/external_engine_typing.py b/projects/privacy-guard/scripts/external_engine_typing.py deleted file mode 100644 index 3089295..0000000 --- a/projects/privacy-guard/scripts/external_engine_typing.py +++ /dev/null @@ -1,66 +0,0 @@ -"""External consumer fixture checked only against an installed wheel.""" - -from __future__ import annotations - -from typing import Literal, assert_type - -from privacy_guard.engines import ( - EngineConfig, - EngineResources, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, -) -from privacy_guard.timeout import Timeout - - -class ExternalConfig(EngineConfig): - engine: Literal["external"] = "external" - keyword: str - - -class ExternalClient: - def find(self, text: str, keyword: str) -> bool: - return keyword in text - - -class ExternalResources(EngineResources): - __slots__ = ("client",) - - def __init__(self, client: ExternalClient) -> None: - self.client = client - - -class ExternalEngine(EntityProcessingEngine[ExternalConfig, ExternalResources]): - supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - assert_type(self.config, ExternalConfig) - assert_type(self.config.keyword, str) - assert_type(self.resources, ExternalResources) - assert_type(self.resources.client, ExternalClient) - self.resources.client.find(text, self.config.keyword) - result = TextProcessingResult(text=text, detections=()) - assert_type(result, TextProcessingResult) - return result - - -engine = ExternalEngine( - ExternalConfig(keyword="secret"), - ExternalResources(ExternalClient()), -) -assert_type(engine.config, ExternalConfig) -assert_type(engine.resources, ExternalResources) -processed = engine.run( - "text", - strategy=EntityProcessingStrategy.DETECT, - timeout=Timeout.from_seconds(1), -) -assert_type(processed, TextProcessingResult) -assert_type(processed.text, str) diff --git a/projects/privacy-guard/src/privacy_guard/py.typed b/projects/privacy-guard/src/privacy_guard/py.typed deleted file mode 100644 index 8b13789..0000000 --- a/projects/privacy-guard/src/privacy_guard/py.typed +++ /dev/null @@ -1 +0,0 @@ - From 8b6e89dcf692d502e0f67bcc2a093e69e4e193ca Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 16:33:59 +0000 Subject: [PATCH 79/82] refactor(privacy-guard): call regex entries rules --- .../architecture/configuration.md | 2 +- .../architecture/entity-processing-engines.md | 13 +-- .../architecture/request-lifecycle.md | 2 +- .../architecture/safety-and-limits.md | 12 +-- projects/privacy-guard/README.md | 10 +-- .../examples/regex-engine/README.md | 2 +- .../examples/regex-engine/patterns.yaml | 4 +- .../src/privacy_guard/constants.py | 4 +- .../src/privacy_guard/engines/__init__.py | 4 +- .../src/privacy_guard/engines/regex.py | 84 +++++++++---------- .../privacy-guard/src/privacy_guard/errors.py | 2 +- .../privacy-guard/tests/engines/test_base.py | 2 +- .../privacy-guard/tests/engines/test_regex.py | 22 ++--- .../tests/examples/test_regex_engine.py | 1 + .../tests/service/test_grpc_integration.py | 2 +- .../tests/service/test_servicer.py | 14 ++-- projects/privacy-guard/tests/test_config.py | 40 ++++----- .../tests/test_request_processor.py | 4 +- projects/privacy-guard/tests/test_timeout.py | 2 +- 19 files changed, 113 insertions(+), 113 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/configuration.md b/docs/documentation/privacy-guard/architecture/configuration.md index 811734b..52e9061 100644 --- a/docs/documentation/privacy-guard/architecture/configuration.md +++ b/docs/documentation/privacy-guard/architecture/configuration.md @@ -50,7 +50,7 @@ entity_processing: pattern_catalog: entities: - name: customer-id - patterns: + rules: - name: prefixed-eight-digit-id pattern: '\bCUST-[0-9]{8}\b' confidence: high diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md index 8c8cee0..d92d8c9 100644 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md @@ -269,24 +269,25 @@ concurrent requests. Keep request text, detections, and counters local to `RegexEngine` owns both regular-expression detection and deterministic template replacement. Its `RegexPatternCatalog` contains structured entities and ordered -patterns; Privacy Guard maintains the schema and limits but no authoritative -pattern set. +rules. Each rule combines one regex pattern with its optional diagnostic name, +confidence, and flags. Privacy Guard maintains the schema and limits but no +authoritative pattern set. Important properties: -- patterns compile once during configuration validation and initialization +- rules compile once during configuration validation and initialization - a non-capturing wrapper plus a private trailing named marker preserves numeric backreferences and proves the configured match completed - user-defined named groups and inline flags are rejected - `ignore_case`, `multiline`, `dot_all`, and `ascii` are explicit fields -- detection retains overlapping matches within and across patterns +- detection retains overlapping matches within and across rules - each backend search receives the shared remaining timeout -- pattern names are optional; unnamed patterns receive deterministic +- rule names are optional; unnamed rules receive deterministic diagnostic identities without changing serialized configuration - catalogs may be supplied inline or through a bounded relative YAML path and normalize to the same structured configuration - replacement resolves overlaps by categorical confidence, span length, - offsets, entity, and pattern identity + offsets, entity, and rule identity - templates allow literal text and `{entity}` only - replacement size is projected before output allocation diff --git a/docs/documentation/privacy-guard/architecture/request-lifecycle.md b/docs/documentation/privacy-guard/architecture/request-lifecycle.md index 19e0791..9a0cda9 100644 --- a/docs/documentation/privacy-guard/architecture/request-lifecycle.md +++ b/docs/documentation/privacy-guard/architecture/request-lifecycle.md @@ -26,7 +26,7 @@ entity_processing: pattern_catalog: entities: - name: email - patterns: + rules: - pattern: '...' confidence: high replacement: diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 35ae855..6664017 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -76,7 +76,7 @@ request IDs use the same content and size rules for logging; an invalid request ID is represented by a constant placeholder and does not change the evaluation result. The human-readable log message quotes request IDs and escapes ASCII spaces so their text cannot introduce another key/value token; structured log -records retain the exact validated ID. Regex entity and supplied pattern names +records retain the exact validated ID. Regex entity and supplied rule names have a stricter ASCII grammar and limit described below. The processor aggregates occurrences before the service applies protobuf @@ -87,13 +87,13 @@ deny with no partial findings. | Limit | Constant | Value | | --- | --- | ---: | -| Regex entity or supplied pattern name | `MAX_REGEX_NAME_BYTES` | 128 ASCII bytes | +| Regex entity or supplied rule name | `MAX_REGEX_NAME_BYTES` | 128 ASCII bytes | | Entities per catalog | `MAX_REGEX_ENTITIES_PER_CATALOG` | 2,000 | -| Patterns per catalog | `MAX_REGEX_PATTERNS_PER_CATALOG` | 10,000 | +| Rules per catalog | `MAX_REGEX_RULES_PER_CATALOG` | 10,000 | | Pattern string | `MAX_REGEX_PATTERN_BYTES` | 16 KiB | -Entity and pattern names use -`[A-Za-z_][A-Za-z0-9_-]*`. Pattern names are optional; their deterministic +Entity and rule names use +`[A-Za-z_][A-Za-z0-9_-]*`. Rule names are optional; their deterministic derived diagnostic identities are not serialized back into configuration. The per-evaluation config `Struct` is limited to 64 KiB. A catalog may satisfy @@ -139,7 +139,7 @@ The processor emits a content-safe `timeout` or `resource` limit kind, and the service adapter emits `resource` for representation limits. Neither log includes request content or collaborator error text. The deny reason directs users to check that log, then reduce the request or replacement size, simplify the -configured stages and patterns, or increase `--timeout-seconds` or +configured stages and rules, or increase `--timeout-seconds` or `PrivacyGuardServer(timeout_seconds=...)` up to 30 seconds. When increasing the processing timeout, users must give OpenShell's middleware timeout additional headroom for worker queueing and configuration or engine preparation. These diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index b84d5a1..4622f4b 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -23,7 +23,7 @@ entity_processing: pattern_catalog: entities: - name: email - patterns: + rules: - pattern: '(? ConfidenceLevel: class RegexEntity(StrictDomainModel): - """One entity name and its ordered, non-empty regex patterns.""" + """One entity name and its ordered, non-empty regex rules.""" name: str - patterns: tuple[RegexPattern, ...] = Field(repr=False) + rules: tuple[RegexRule, ...] = Field(repr=False) @field_validator("name") @classmethod def _name_is_safe(cls, value: str) -> str: return _validate_name(value) - @field_validator("patterns", mode="before") + @field_validator("rules", mode="before") @classmethod - def _patterns_are_non_empty(cls, value: object) -> object: + def _rules_are_non_empty(cls, value: object) -> object: if not isinstance(value, list | tuple) or not value: - raise ValueError("patterns must be a non-empty list") + raise ValueError("rules must be a non-empty list") return tuple(value) @model_validator(mode="after") - def _supplied_pattern_names_are_unique(self) -> Self: - supplied_names = [ - pattern.name for pattern in self.patterns if pattern.name is not None - ] + def _supplied_rule_names_are_unique(self) -> Self: + supplied_names = [rule.name for rule in self.rules if rule.name is not None] if len(supplied_names) != len(set(supplied_names)): - raise ValueError("supplied pattern names must be unique within an entity") + raise ValueError("supplied rule names must be unique within an entity") return self @@ -135,10 +133,10 @@ def _catalog_is_bounded_and_unambiguous(self) -> Self: if len(self.entities) > MAX_REGEX_ENTITIES_PER_CATALOG: raise ValueError("entity catalog exceeds the size limit") if ( - sum(len(entity.patterns) for entity in self.entities) - > MAX_REGEX_PATTERNS_PER_CATALOG + sum(len(entity.rules) for entity in self.entities) + > MAX_REGEX_RULES_PER_CATALOG ): - raise ValueError("pattern catalog exceeds the size limit") + raise ValueError("rule catalog exceeds the size limit") return self @@ -192,7 +190,7 @@ def _load_pattern_catalog( return value @model_validator(mode="after") - def _patterns_are_valid(self) -> Self: + def _rules_are_valid(self) -> Self: try: _compile_pattern_catalog(self.pattern_catalog) except (RecursionError, ValueError, regex.error): @@ -265,9 +263,9 @@ def _run( start=start, end=end, confidence=rule.confidence, - metadata={_PATTERN_METADATA_KEY: rule.pattern_identity}, + metadata={_RULE_METADATA_KEY: rule.rule_identity}, ) - detections_with_identity.append((detection, rule.pattern_identity)) + detections_with_identity.append((detection, rule.rule_identity)) if len(detections_with_identity) > MAX_DETECTIONS_PER_STAGE: raise EngineLimitExceededError( "regex detection count exceeds the limit" @@ -302,7 +300,7 @@ def _run( @dataclass(frozen=True) class _CompiledRule: entity: str - pattern_identity: str + rule_identity: str confidence: ConfidenceLevel marker: str compiled: _CompiledPattern @@ -556,12 +554,12 @@ def _compile_pattern_catalog( rules = tuple( _compile_rule( entity, - pattern, + rule, catalog_index=global_index, - entity_pattern_index=pattern_index, + entity_rule_index=rule_index, ) - for global_index, (entity, pattern_index, pattern) in enumerate( - _iter_catalog_patterns(catalog) + for global_index, (entity, rule_index, rule) in enumerate( + _iter_catalog_rules(catalog) ) ) weight_bytes = _compiled_pattern_weight(catalog, len(rules)) @@ -636,47 +634,47 @@ def _clear_parsed_pattern_catalog_cache() -> None: def _compile_rule( entity: RegexEntity, - pattern: RegexPattern, + rule: RegexRule, catalog_index: int, - entity_pattern_index: int, + entity_rule_index: int, ) -> _CompiledRule: flags = 0 - if pattern.ignore_case: + if rule.ignore_case: flags |= regex.IGNORECASE - if pattern.multiline: + if rule.multiline: flags |= regex.MULTILINE - if pattern.dot_all: + if rule.dot_all: flags |= regex.DOTALL - if pattern.ascii: + if rule.ascii: flags |= regex.ASCII - if _contains_inline_flags(pattern.pattern): + if _contains_inline_flags(rule.pattern): raise ValueError("inline flags are unsupported") - unmarked = regex.compile(pattern.pattern, flags) + unmarked = regex.compile(rule.pattern, flags) if unmarked.groupindex: raise ValueError("named groups are reserved") if unmarked.search("") is not None: raise ValueError("pattern must not match empty input") - marker = f"_pg_pattern_{catalog_index:06d}" - compiled = regex.compile(f"(?:{pattern.pattern})(?P<{marker}>)", flags) + marker = f"_pg_rule_{catalog_index:06d}" + compiled = regex.compile(f"(?:{rule.pattern})(?P<{marker}>)", flags) if marker not in compiled.groupindex: raise ValueError("internal marker is missing") - pattern_identity = pattern.name or f"{entity.name}.patterns[{entity_pattern_index}]" + rule_identity = rule.name or f"{entity.name}.rules[{entity_rule_index}]" return _CompiledRule( entity=entity.name, - pattern_identity=pattern_identity, - confidence=pattern.confidence, + rule_identity=rule_identity, + confidence=rule.confidence, marker=marker, compiled=compiled, ) -def _iter_catalog_patterns( +def _iter_catalog_rules( catalog: RegexPatternCatalog, -) -> tuple[tuple[RegexEntity, int, RegexPattern], ...]: +) -> tuple[tuple[RegexEntity, int, RegexRule], ...]: return tuple( - (entity, pattern_index, pattern) + (entity, rule_index, rule) for entity in catalog.entities - for pattern_index, pattern in enumerate(entity.patterns) + for rule_index, rule in enumerate(entity.rules) ) @@ -781,7 +779,7 @@ def _rendered_template_size(template: str, entity: str) -> int: ConfidenceLevel.MEDIUM: 1, ConfidenceLevel.HIGH: 2, } -_PATTERN_METADATA_KEY = "pattern" +_RULE_METADATA_KEY = "rule" _MAX_CACHED_PATTERN_CATALOGS = 64 _PATTERN_CATALOG_CACHE: OrderedDict[ tuple[str, int, int, int, int, int], @@ -803,7 +801,7 @@ def _rendered_template_size(template: str, entity: str) -> int: "RegexEngine", "RegexEngineConfig", "RegexEntity", - "RegexPattern", "RegexPatternCatalog", "RegexReplacement", + "RegexRule", ] diff --git a/projects/privacy-guard/src/privacy_guard/errors.py b/projects/privacy-guard/src/privacy_guard/errors.py index 9bfec62..84e58be 100644 --- a/projects/privacy-guard/src/privacy_guard/errors.py +++ b/projects/privacy-guard/src/privacy_guard/errors.py @@ -111,7 +111,7 @@ class TimeoutExpiredError(EntityProcessingError): def __init__(self) -> None: super().__init__( "Privacy Guard processing timed out. Reduce the request size or simplify " - "the configured stages and patterns, or increase the processing timeout " + "the configured stages and rules, or increase the processing timeout " f"to at most {MAX_TIMEOUT_SECONDS:g} seconds, then retry." ) diff --git a/projects/privacy-guard/tests/engines/test_base.py b/projects/privacy-guard/tests/engines/test_base.py index 9e12357..2dcae3d 100644 --- a/projects/privacy-guard/tests/engines/test_base.py +++ b/projects/privacy-guard/tests/engines/test_base.py @@ -102,7 +102,7 @@ def test_detection_confidence_and_metadata_are_strict_bounded_values() -> None: "start": 0, "end": 1, "confidence": "high", - "metadata": {"pattern": "email.patterns[0]"}, + "metadata": {"rule": "email.rules[0]"}, } ) assert categorical.confidence is ConfidenceLevel.HIGH diff --git a/projects/privacy-guard/tests/engines/test_regex.py b/projects/privacy-guard/tests/engines/test_regex.py index 9127124..54ddd3f 100644 --- a/projects/privacy-guard/tests/engines/test_regex.py +++ b/projects/privacy-guard/tests/engines/test_regex.py @@ -21,7 +21,7 @@ def _config( - patterns: list[dict[str, object]], + rules: list[dict[str, object]], *, replacement: dict[str, object] | None = None, ) -> RegexEngineConfig: @@ -31,7 +31,7 @@ def _config( "entities": [ { "name": "token", - "patterns": patterns, + "rules": rules, } ] }, @@ -56,7 +56,7 @@ def _run( detection.entity, detection.start, detection.end, - detection.metadata["pattern"], + detection.metadata["rule"], ) for detection in result.detections ] @@ -68,7 +68,7 @@ def _catalog(pattern: str) -> RegexPatternCatalog: "entities": [ { "name": "token", - "patterns": [ + "rules": [ { "pattern": pattern, "confidence": "high", @@ -112,7 +112,7 @@ def test_optional_names_derive_identity_without_affecting_internal_marker() -> N assert [item[3] for item in detections] == [ "same-name", "same_name", - "token.patterns[2]", + "token.rules[2]", ] @@ -215,7 +215,7 @@ def test_patterns_with_consuming_lookarounds_or_boundaries_remain_valid( assert [(item[1], item[2]) for item in detections] == [expected_span] -def test_duplicate_supplied_names_are_rejected_but_unnamed_patterns_are_not() -> None: +def test_duplicate_supplied_names_are_rejected_but_unnamed_rules_are_not() -> None: with pytest.raises(ValidationError): _config( [ @@ -230,7 +230,7 @@ def test_duplicate_supplied_names_are_rejected_but_unnamed_patterns_are_not() -> {"pattern": "y", "confidence": "high"}, ] ) - assert len(config.pattern_catalog.entities[0].patterns) == 2 + assert len(config.pattern_catalog.entities[0].rules) == 2 def test_replacement_selects_ranked_non_overlapping_winners() -> None: @@ -423,16 +423,16 @@ def test_compiled_catalog_same_key_race_accounts_once( def synchronized_compile( entity: regex_module.RegexEntity, - pattern: regex_module.RegexPattern, + rule: regex_module.RegexRule, catalog_index: int, - entity_pattern_index: int, + entity_rule_index: int, ) -> regex_module._CompiledRule: workers_ready.wait(timeout=5) return original_compile_rule( entity, - pattern, + rule, catalog_index, - entity_pattern_index, + entity_rule_index, ) monkeypatch.setattr(regex_module, "_compile_rule", synchronized_compile) diff --git a/projects/privacy-guard/tests/examples/test_regex_engine.py b/projects/privacy-guard/tests/examples/test_regex_engine.py index 7ef63ca..328249e 100644 --- a/projects/privacy-guard/tests/examples/test_regex_engine.py +++ b/projects/privacy-guard/tests/examples/test_regex_engine.py @@ -77,6 +77,7 @@ def test_builtin_registry_drives_documented_cli_discovery_and_schema() -> None: serialized_schema = json.loads(schema.stdout) assert "RegexEngineConfig" in serialized_schema["$defs"] assert "RegexPatternCatalog" in serialized_schema["$defs"] + assert "RegexRule" in serialized_schema["$defs"] assert "RegexReplacement" in serialized_schema["$defs"] diff --git a/projects/privacy-guard/tests/service/test_grpc_integration.py b/projects/privacy-guard/tests/service/test_grpc_integration.py index b67a20c..4113c1e 100644 --- a/projects/privacy-guard/tests/service/test_grpc_integration.py +++ b/projects/privacy-guard/tests/service/test_grpc_integration.py @@ -45,7 +45,7 @@ def _config( "entities": [ { "name": "email", - "patterns": [ + "rules": [ { "pattern": pattern, "confidence": "high", diff --git a/projects/privacy-guard/tests/service/test_servicer.py b/projects/privacy-guard/tests/service/test_servicer.py index e48b9ce..205c7aa 100644 --- a/projects/privacy-guard/tests/service/test_servicer.py +++ b/projects/privacy-guard/tests/service/test_servicer.py @@ -46,12 +46,12 @@ def _values( action: str = "replace", *, - patterns: list[dict[str, object]] | None = None, + rules: list[dict[str, object]] | None = None, stage_count: int = 1, stage_name: str | None = None, ) -> dict[str, object]: - if patterns is None: - patterns = [ + if rules is None: + rules = [ { "pattern": r"[a-z]+@[a-z]+\.[a-z]+", "confidence": "high", @@ -64,7 +64,7 @@ def _values( "entities": [ { "name": "email", - "patterns": patterns, + "rules": rules, } ] }, @@ -286,7 +286,7 @@ def test_limit_deny_explains_recovery_options() -> None: assert result.reason == LIMIT_REASON assert "Check Privacy Guard logs for the limit kind" in result.reason assert "Reduce the request or replacement size" in result.reason - assert "simplify the configured stages and patterns" in result.reason + assert "simplify the configured stages and rules" in result.reason assert "--timeout-seconds or PrivacyGuardServer(timeout_seconds=...)" in ( result.reason ) @@ -717,11 +717,11 @@ def test_compiled_cache_eviction_does_not_invalidate_active_processor( ) processor_values = _values( "detect", - patterns=[{"pattern": "aaa", "confidence": "high"}], + rules=[{"pattern": "aaa", "confidence": "high"}], ) validation_values = _values( "detect", - patterns=[{"pattern": "bbb", "confidence": "high"}], + rules=[{"pattern": "bbb", "confidence": "high"}], ) try: diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py index 4b91784..d20538b 100644 --- a/projects/privacy-guard/tests/test_config.py +++ b/projects/privacy-guard/tests/test_config.py @@ -44,7 +44,7 @@ def _config( "entities": [ { "name": "email", - "patterns": [ + "rules": [ { "pattern": r"\buser@example\.com\b", "confidence": "high", @@ -86,7 +86,7 @@ def test_known_discriminator_constructs_the_exact_engine_config() -> None: assert type(stage.config) is RegexEngineConfig assert type(stage.config.pattern_catalog) is RegexPatternCatalog - assert stage.config.pattern_catalog.entities[0].patterns[0].name is None + assert stage.config.pattern_catalog.entities[0].rules[0].name is None assert stage.diagnostic_name(1) == "regex[1]" @@ -206,7 +206,7 @@ def test_catalog_file_change_produces_a_different_validated_config( registry = _registry() first = registry.validate_config(values) - catalog["entities"][0]["patterns"][0]["confidence"] = "low" + catalog["entities"][0]["rules"][0]["confidence"] = "low" catalog_path.write_text(yaml.safe_dump(catalog), encoding="utf-8") second = registry.validate_config(values) @@ -226,7 +226,7 @@ def test_parsed_catalog_cache_evicts_least_recently_used_file_by_weight( catalog = _config()["entity_processing"]["stages"][0]["config"][ "pattern_catalog" ] - catalog["entities"][0]["patterns"][0]["pattern"] = f"pattern-{suffix}" + catalog["entities"][0]["rules"][0]["pattern"] = f"pattern-{suffix}" contents = yaml.safe_dump(catalog) path.write_text(contents, encoding="utf-8") sizes.append(len(contents.encode("utf-8"))) @@ -268,7 +268,7 @@ def test_parsed_catalog_cache_skips_oversized_valid_file( regex_module._clear_parsed_pattern_catalog_cache() monkeypatch.chdir(tmp_path) catalog = _config()["entity_processing"]["stages"][0]["config"]["pattern_catalog"] - catalog["entities"][0]["patterns"][0]["pattern"] = "sensitive-pattern" + catalog["entities"][0]["rules"][0]["pattern"] = "sensitive-pattern" contents = yaml.safe_dump(catalog) path = Path("sensitive-oversized.yaml") path.write_text(contents, encoding="utf-8") @@ -335,17 +335,17 @@ def test_equivalent_catalogs_reuse_compiled_regex_rules( def record_compile_rule( entity: regex_module.RegexEntity, - pattern: regex_module.RegexPattern, + rule: regex_module.RegexRule, catalog_index: int, - entity_pattern_index: int, + entity_rule_index: int, ) -> regex_module._CompiledRule: nonlocal compile_calls compile_calls += 1 return original_compile_rule( entity, - pattern, + rule, catalog_index, - entity_pattern_index, + entity_rule_index, ) monkeypatch.setattr(regex_module, "_compile_rule", record_compile_rule) @@ -357,7 +357,7 @@ def record_compile_rule( changed_values = deepcopy(_config()) changed_values["entity_processing"]["stages"][0]["config"]["pattern_catalog"][ "entities" - ][0]["patterns"][0]["pattern"] = "changed" + ][0]["rules"][0]["pattern"] = "changed" changed = registry.validate_config(changed_values) registry.create_engine(changed.entity_processing.stages[0].config) @@ -488,12 +488,12 @@ def test_catalog_file_rejects_a_fifo_without_blocking(tmp_path: Path) -> None: @pytest.mark.parametrize( "contents", [ - "entities:\n - name: first\n name: duplicate\n patterns: []\n", + "entities:\n - name: first\n name: duplicate\n rules: []\n", ( "entities:\n" " - &shared\n" " name: first\n" - " patterns:\n" + " rules:\n" " - pattern: x\n" " confidence: high\n" " - *shared\n" @@ -626,12 +626,12 @@ def test_explicit_stage_name_cannot_collide_with_a_derived_name() -> None: _registry().validate_config(values) -def test_regex_pattern_names_are_optional_but_supplied_names_are_unique() -> None: +def test_regex_rule_names_are_optional_but_supplied_names_are_unique() -> None: values = _config() - patterns = values["entity_processing"]["stages"][0]["config"]["pattern_catalog"][ + rules = values["entity_processing"]["stages"][0]["config"]["pattern_catalog"][ "entities" - ][0]["patterns"] - patterns.extend( + ][0]["rules"] + rules.extend( [ {"pattern": "second", "confidence": "low"}, {"name": "named", "pattern": "third", "confidence": "medium"}, @@ -641,10 +641,10 @@ def test_regex_pattern_names_are_optional_but_supplied_names_are_unique() -> Non regex_config = config.entity_processing.stages[0].config assert isinstance(regex_config, RegexEngineConfig) - parsed_patterns = regex_config.pattern_catalog.entities[0].patterns - assert [pattern.name for pattern in parsed_patterns] == [None, None, "named"] + parsed_rules = regex_config.pattern_catalog.entities[0].rules + assert [rule.name for rule in parsed_rules] == [None, None, "named"] - patterns.append({"name": "named", "pattern": "duplicate", "confidence": "high"}) + rules.append({"name": "named", "pattern": "duplicate", "confidence": "high"}) with pytest.raises(PrivacyGuardError): _registry().validate_config(values) @@ -656,7 +656,7 @@ def test_validated_config_equality_covers_concrete_expanded_config() -> None: changed_values = _config() changed_values["entity_processing"]["stages"][0]["config"]["pattern_catalog"][ "entities" - ][0]["patterns"][0]["confidence"] = "low" + ][0]["rules"][0]["confidence"] = "low" changed = registry.validate_config(changed_values) assert first == equivalent diff --git a/projects/privacy-guard/tests/test_request_processor.py b/projects/privacy-guard/tests/test_request_processor.py index a147282..be5a22c 100644 --- a/projects/privacy-guard/tests/test_request_processor.py +++ b/projects/privacy-guard/tests/test_request_processor.py @@ -37,7 +37,7 @@ def _values( "entities": [ { "name": "person", - "patterns": [ + "rules": [ { "pattern": "Alice", "confidence": "high", @@ -62,7 +62,7 @@ def _values( "entities": [ { "name": "marker", - "patterns": [ + "rules": [ { "pattern": "person", "confidence": "medium", diff --git a/projects/privacy-guard/tests/test_timeout.py b/projects/privacy-guard/tests/test_timeout.py index 7860375..778e97b 100644 --- a/projects/privacy-guard/tests/test_timeout.py +++ b/projects/privacy-guard/tests/test_timeout.py @@ -27,7 +27,7 @@ def test_expired_timeout_raises_typed_signal() -> None: assert str(captured.value) == ( "Privacy Guard processing timed out. Reduce the request size or simplify " - "the configured stages and patterns, or increase the processing timeout " + "the configured stages and rules, or increase the processing timeout " "to at most 30 seconds, then retry." ) From a9ca19ce1edbe96c35cdc76dfb2270cbdcfe2977 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 16:56:44 +0000 Subject: [PATCH 80/82] feat(privacy-guard): configure OpenShell gateway --- projects/privacy-guard/AGENTS.md | 5 +- projects/privacy-guard/README.md | 17 + .../examples/custom-engine/README.md | 16 +- .../examples/custom-engine/gateway.toml | 11 - .../examples/regex-engine/README.md | 16 +- .../examples/regex-engine/gateway.toml | 11 - .../privacy-guard/src/privacy_guard/cli.py | 136 +++++++- .../src/privacy_guard/gateway_config.py | 294 ++++++++++++++++++ .../tests/examples/test_custom_engine.py | 17 +- .../tests/examples/test_regex_engine.py | 17 +- projects/privacy-guard/tests/test_cli.py | 105 +++++++ .../tests/test_gateway_config.py | 194 ++++++++++++ 12 files changed, 771 insertions(+), 68 deletions(-) delete mode 100644 projects/privacy-guard/examples/custom-engine/gateway.toml delete mode 100644 projects/privacy-guard/examples/regex-engine/gateway.toml create mode 100644 projects/privacy-guard/src/privacy_guard/gateway_config.py create mode 100644 projects/privacy-guard/tests/test_gateway_config.py diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md index 226cbe7..c48302a 100644 --- a/projects/privacy-guard/AGENTS.md +++ b/projects/privacy-guard/AGENTS.md @@ -33,7 +33,10 @@ Run focused tests while working and `make check` before handoff. - `src/privacy_guard/engines/`: engine contract, registry, and built-in implementations - `src/privacy_guard/config.py`: policy action and ordered stage configuration - `src/privacy_guard/request_processor.py`: stage execution and policy disposition -- `src/privacy_guard/cli.py`: command parsing, discovery, configuration-schema output, and server adapter +- `src/privacy_guard/cli.py`: command parsing, discovery, gateway setup, + configuration-schema output, and server adapter +- `src/privacy_guard/gateway_config.py`: safe OpenShell gateway TOML + registration updates - `src/privacy_guard/logging.py`: package-scoped standard-library logging configuration - `src/privacy_guard/base.py`: package-wide strict immutable domain-model base - `src/privacy_guard/string_validators.py`: shared string validators and field types diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 4622f4b..efb98dd 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -229,11 +229,28 @@ environment. ```bash uv run privacy-guard engines uv run privacy-guard configuration-schema +uv run privacy-guard configure-gateway --host-ip YOUR_HOST_IPV4 uv run privacy-guard serve \ --listen 127.0.0.1:50051 \ --timeout-seconds 5 ``` +Replace `YOUR_HOST_IPV4` with the non-loopback IPv4 address that the OpenShell +gateway and sandbox supervisors use to reach this host. `configure-gateway` +adds or updates a registration named `privacy-guard` in the path set by +OpenShell's `OPENSHELL_GATEWAY_CONFIG` override, or OpenShell's standard +per-user gateway config location when that variable is unset: +`$XDG_CONFIG_HOME/openshell/gateway.toml` (normally +`~/.config/openshell/gateway.toml`). Privacy Guard does not guess which host +interface is correct. Use `--name` when the policy references a different +registration name, `--port` when Privacy Guard does not listen on 50051, and +`--config` for a nonstandard gateway TOML. OpenShell middleware registration +names must use 1–128 ASCII bytes containing only letters, digits, `.`, `_`, `-`, +or `/`; the `openshell/` prefix is reserved. The command preserves unrelated +settings and updates the named registration when it already exists. Start +Privacy Guard before restarting the gateway, because OpenShell connects to +registered middleware during startup. + Entity behavior is supplied by OpenShell policy config, not server startup flags. Deployment startup owns only installed engine implementations and operator resources such as model profiles, endpoints, clients, and credentials. diff --git a/projects/privacy-guard/examples/custom-engine/README.md b/projects/privacy-guard/examples/custom-engine/README.md index 7d9d538..e64f860 100644 --- a/projects/privacy-guard/examples/custom-engine/README.md +++ b/projects/privacy-guard/examples/custom-engine/README.md @@ -120,18 +120,20 @@ generate the local gateway configuration: ```bash YOUR_HOST_IP=YOUR_HOST_IPV4 -if [ "$YOUR_HOST_IP" = "YOUR_HOST_IPV4" ] || [ "$YOUR_HOST_IP" = "127.0.0.1" ]; then - echo "Set YOUR_HOST_IP to a non-loopback IPv4 address" -else - sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml - grep grpc_endpoint gateway.local.toml -fi +uv run privacy-guard configure-gateway \ + --host-ip "$YOUR_HOST_IP" \ + --name privacy-guard-custom-engine \ + --config gateway.local.toml ``` Replace `YOUR_HOST_IPV4` with the address you selected. Do not use `127.0.0.1`, a VPN address, or `host.openshell.internal`: the foreground gateway process and the sandbox supervisor must both be able to resolve and reach the -configured endpoint. +configured endpoint. This walkthrough passes `--config` to keep its generated +file local and disposable. Without that option, the command uses OpenShell's +`OPENSHELL_GATEWAY_CONFIG` override when set, otherwise its standard per-user +gateway config location at `$XDG_CONFIG_HOME/openshell/gateway.toml`, normally +`~/.config/openshell/gateway.toml`. ## Restart the local gateway with middleware enabled diff --git a/projects/privacy-guard/examples/custom-engine/gateway.toml b/projects/privacy-guard/examples/custom-engine/gateway.toml deleted file mode 100644 index c1bef4e..0000000 --- a/projects/privacy-guard/examples/custom-engine/gateway.toml +++ /dev/null @@ -1,11 +0,0 @@ -# OpenShell gateway configuration for the custom-engine example. -# Pass this file directly to openshell-gateway after replacing the host IP. - -[openshell] -version = 1 - -[[openshell.supervisor.middleware]] -name = "privacy-guard-custom-engine" -grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051" -max_body_bytes = 4194304 -timeout = "5s" diff --git a/projects/privacy-guard/examples/regex-engine/README.md b/projects/privacy-guard/examples/regex-engine/README.md index 01cad7e..255de36 100644 --- a/projects/privacy-guard/examples/regex-engine/README.md +++ b/projects/privacy-guard/examples/regex-engine/README.md @@ -96,18 +96,20 @@ In terminal 2, from this example directory, assign the selected address: ```bash YOUR_HOST_IP=YOUR_HOST_IPV4 -if [ "$YOUR_HOST_IP" = "YOUR_HOST_IPV4" ] || [ "$YOUR_HOST_IP" = "127.0.0.1" ]; then - echo "Set YOUR_HOST_IP to a non-loopback IPv4 address" -else - sed "s/REPLACE_WITH_HOST_IP/$YOUR_HOST_IP/" gateway.toml > gateway.local.toml - grep grpc_endpoint gateway.local.toml -fi +uv run privacy-guard configure-gateway \ + --host-ip "$YOUR_HOST_IP" \ + --name privacy-guard-regex \ + --config gateway.local.toml ``` Replace `YOUR_HOST_IPV4` with the address you selected. Do not use `127.0.0.1`, a VPN address, or `host.openshell.internal`: the foreground gateway process and the sandbox supervisor must both be able to resolve and reach the -configured endpoint. +configured endpoint. This walkthrough passes `--config` to keep its generated +file local and disposable. Without that option, the command uses OpenShell's +`OPENSHELL_GATEWAY_CONFIG` override when set, otherwise its standard per-user +gateway config location at `$XDG_CONFIG_HOME/openshell/gateway.toml`, normally +`~/.config/openshell/gateway.toml`. ## Restart the local gateway with middleware enabled diff --git a/projects/privacy-guard/examples/regex-engine/gateway.toml b/projects/privacy-guard/examples/regex-engine/gateway.toml deleted file mode 100644 index 2458578..0000000 --- a/projects/privacy-guard/examples/regex-engine/gateway.toml +++ /dev/null @@ -1,11 +0,0 @@ -# OpenShell gateway configuration for the RegexEngine example. -# Generate gateway.local.toml with the host address reachable by supervisors. - -[openshell] -version = 1 - -[[openshell.supervisor.middleware]] -name = "privacy-guard-regex" -grpc_endpoint = "http://REPLACE_WITH_HOST_IP:50051" -max_body_bytes = 4194304 -timeout = "5s" diff --git a/projects/privacy-guard/src/privacy_guard/cli.py b/projects/privacy-guard/src/privacy_guard/cli.py index 1122d98..c0d35f2 100644 --- a/projects/privacy-guard/src/privacy_guard/cli.py +++ b/projects/privacy-guard/src/privacy_guard/cli.py @@ -3,8 +3,10 @@ from __future__ import annotations import importlib +import ipaddress import json from dataclasses import dataclass +from pathlib import Path from typing import Annotated import typer @@ -13,13 +15,24 @@ from privacy_guard.engines import EntityProcessingStrategy from privacy_guard.engines.registry import EngineRegistry, create_builtin_registry from privacy_guard.errors import PrivacyGuardError +from privacy_guard.gateway_config import ( + MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES, + GatewayConfigError, + GatewayConfigUpdate, + default_gateway_config_path, + update_gateway_config, + validate_middleware_name, +) from privacy_guard.logging import LoggingConfig, configure_logging, get_logger from privacy_guard.service.server import DEFAULT_LISTEN_ADDRESS, PrivacyGuardServer from privacy_guard.timeout import validate_timeout_seconds app = typer.Typer( name="privacy-guard", - help="Run Privacy Guard and inspect installed entity-processing engines.", + help=( + "Run Privacy Guard, configure a local OpenShell gateway, and inspect " + "installed entity-processing engines." + ), no_args_is_help=True, add_completion=False, ) @@ -32,19 +45,28 @@ def configure_cli( str | None, typer.Option( help=( - "Python module and callable that return a finalized engine registry, " - "formatted as module:factory." + "Load engines from a trusted Python callable, formatted as " + "module:factory. The callable must return a finalized EngineRegistry." ), ), ] = None, debug: Annotated[ bool, - typer.Option(help="Enable content-safe processing diagnostics."), + typer.Option( + "--debug", + help=( + "Log content-safe diagnostic details for startup and request handling." + ), + ), ] = False, debug_log_content: Annotated[ bool, typer.Option( - help="DANGEROUS: log complete input and processed text.", + "--debug-log-content", + help=( + "DANGEROUS: log complete request and processed text, which may " + "contain secrets or personal data." + ), ), ] = False, ) -> None: @@ -68,7 +90,12 @@ def serve( context: typer.Context, listen: Annotated[ str, - typer.Option(help="Address on which the middleware server listens."), + typer.Option( + help=( + "Host and port on which Privacy Guard listens, formatted as " + "host:port. Use 0.0.0.0 when sandbox supervisors must reach it." + ), + ), ] = DEFAULT_LISTEN_ADDRESS, timeout_seconds: Annotated[ float, @@ -80,7 +107,7 @@ def serve( ), ] = DEFAULT_TIMEOUT_SECONDS, ) -> None: - """Run the middleware server with the selected engine inventory.""" + """Run Privacy Guard until the process receives a shutdown signal.""" options = _command_options(context) try: validated_timeout_seconds = validate_timeout_seconds(timeout_seconds) @@ -100,9 +127,100 @@ def serve( raise typer.Exit(code=1) from None +@app.command("configure-gateway") +def configure_gateway( + host_ip: Annotated[ + str, + typer.Option( + help=( + "Non-loopback IPv4 address of this host that both the OpenShell " + "gateway and sandbox supervisors can reach." + ), + ), + ], + config: Annotated[ + Path | None, + typer.Option( + help=( + "Gateway TOML to update. Defaults to " + "`$OPENSHELL_GATEWAY_CONFIG` when set, otherwise `gateway.toml` " + "under `$XDG_CONFIG_HOME/openshell`." + ), + ), + ] = None, + name: Annotated[ + str, + typer.Option( + help=( + "Gateway registration name referenced by the policy's middleware " + "field. OpenShell allows " + f"1-{MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES} ASCII bytes." + ), + ), + ] = "privacy-guard", + port: Annotated[ + int, + typer.Option( + min=1, + max=65535, + help=( + "Privacy Guard port. Use the same port in `privacy-guard serve " + "--listen`." + ), + ), + ] = 50051, +) -> None: + """Add or update Privacy Guard in an OpenShell gateway TOML file.""" + try: + address = ipaddress.IPv4Address(host_ip) + except ipaddress.AddressValueError: + raise typer.BadParameter( + "Pass one IPv4 address, for example --host-ip 192.168.1.20.", + param_hint="--host-ip", + ) from None + if address.is_loopback or address.is_unspecified: + raise typer.BadParameter( + "Pass a non-loopback host IPv4 address reachable by sandbox " + "supervisors; do not use 127.0.0.1 or 0.0.0.0.", + param_hint="--host-ip", + ) + try: + validated_name = validate_middleware_name(name) + except GatewayConfigError as error: + raise typer.BadParameter( + str(error), + param_hint="--name", + ) from None + + config_path = config or default_gateway_config_path() + try: + result = update_gateway_config( + config_path, + middleware_name=validated_name, + host_ip=str(address), + port=port, + ) + except GatewayConfigError as error: + typer.echo(f"Could not configure the OpenShell gateway: {error}", err=True) + raise typer.Exit(code=1) from None + + action = { + GatewayConfigUpdate.CREATED: "Created", + GatewayConfigUpdate.ADDED: "Added the registration to", + GatewayConfigUpdate.UPDATED: "Updated", + GatewayConfigUpdate.UNCHANGED: "No changes needed in", + }[result] + typer.echo(f"{action} {config_path}") + typer.echo(f"Registered {validated_name} at http://{address}:{port}") + typer.echo( + "Next: start Privacy Guard, then restart the OpenShell gateway so it " + "loads this registration." + ) + + @app.command("configuration-schema") def configuration_schema(context: typer.Context) -> None: - """Print the exact finalized policy JSON Schema.""" + """Print the policy configuration JSON Schema for the installed engines.""" typer.echo( json.dumps( _command_options(context).registry.configuration_json_schema(), @@ -115,7 +233,7 @@ def configuration_schema(context: typer.Context) -> None: @app.command("engines") def engines(context: typer.Context) -> None: - """List installed engines and every supported processing strategy.""" + """List installed engines, supported strategies, and their behavior.""" for description in _command_options(context).registry.describe_engines(): strategies = ",".join( strategy.value diff --git a/projects/privacy-guard/src/privacy_guard/gateway_config.py b/projects/privacy-guard/src/privacy_guard/gateway_config.py new file mode 100644 index 0000000..8ad54d6 --- /dev/null +++ b/projects/privacy-guard/src/privacy_guard/gateway_config.py @@ -0,0 +1,294 @@ +"""OpenShell gateway configuration updates for the command-line application.""" + +from __future__ import annotations + +import os +import re +import stat +import tempfile +import tomllib +from enum import Enum +from pathlib import Path + + +class GatewayConfigUpdate(Enum): + """Result of writing one Privacy Guard middleware registration.""" + + CREATED = "created" + ADDED = "added" + UPDATED = "updated" + UNCHANGED = "unchanged" + + +class GatewayConfigError(ValueError): + """A safe, actionable gateway configuration update error.""" + + +# Mirrors OpenShell's stable-identifier byte limit for external middleware +# registrations. +MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES = 128 + + +def default_gateway_config_path() -> Path: + """Resolve OpenShell's config override or standard per-user gateway path.""" + # Mirrors the OpenShell gateway's `--config` environment override and XDG + # fallback. Package-specific deployments can still pass --config explicitly. + configured_path = os.environ.get("OPENSHELL_GATEWAY_CONFIG") + if configured_path: + return Path(configured_path) + config_home = os.environ.get("XDG_CONFIG_HOME") + if config_home: + return Path(config_home) / "openshell" / "gateway.toml" + return Path.home() / ".config" / "openshell" / "gateway.toml" + + +def update_gateway_config( + path: Path, + *, + middleware_name: str, + host_ip: str, + port: int, +) -> GatewayConfigUpdate: + """Add or update one named Privacy Guard middleware registration.""" + validate_middleware_name(middleware_name) + endpoint = f"http://{host_ip}:{port}" + try: + original = path.read_text(encoding="utf-8") + except FileNotFoundError: + updated = _new_gateway_config( + middleware_name=middleware_name, + endpoint=endpoint, + ) + _write_atomically(path, updated) + return GatewayConfigUpdate.CREATED + except (OSError, UnicodeError) as error: + raise GatewayConfigError( + f"Could not read {path}. Check that the file is readable UTF-8 TOML." + ) from error + + if not original.strip(): + updated = _new_gateway_config( + middleware_name=middleware_name, + endpoint=endpoint, + ) + _write_atomically(path, updated) + return GatewayConfigUpdate.CREATED + + values = _load_gateway_config(original, path) + middleware = _middleware_entries(values, path) + matching_indexes = [ + index + for index, entry in enumerate(middleware) + if entry.get("name") == middleware_name + ] + if len(matching_indexes) > 1: + raise GatewayConfigError( + f"{path} contains multiple middleware registrations named " + f"{middleware_name!r}. Remove the duplicate entries, then retry." + ) + + blocks = list(_MIDDLEWARE_BLOCK_PATTERN.finditer(original)) + if len(blocks) != len(middleware): + raise GatewayConfigError( + f"Could not safely locate every middleware registration in {path}. " + "Format the file as standard TOML tables, then retry." + ) + + if matching_indexes: + block = blocks[matching_indexes[0]] + replacement = _update_middleware_block( + block.group(0), + endpoint=endpoint, + ) + updated = original[: block.start()] + replacement + original[block.end() :] + result = GatewayConfigUpdate.UPDATED + else: + updated = _append_middleware_block( + original, + middleware_name=middleware_name, + endpoint=endpoint, + ) + result = GatewayConfigUpdate.ADDED + + _load_gateway_config(updated, path) + if updated == original: + return GatewayConfigUpdate.UNCHANGED + _write_atomically(path, updated) + return result + + +def validate_middleware_name(name: str) -> str: + """Validate OpenShell's external middleware registration-name contract.""" + if ( + not name + or len(name.encode("utf-8")) > MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES + or any(character not in _MIDDLEWARE_NAME_CHARACTERS for character in name) + ): + raise GatewayConfigError( + "Middleware names must use " + f"1-{MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES} ASCII bytes containing " + "only letters, digits, '.', '_', '-', or '/'." + ) + if name.startswith("openshell/"): + raise GatewayConfigError( + "Operator-owned middleware names cannot start with 'openshell/'." + ) + return name + + +def _new_gateway_config(*, middleware_name: str, endpoint: str) -> str: + return "[openshell]\nversion = 1\n\n" + _middleware_block( + middleware_name=middleware_name, + endpoint=endpoint, + ) + + +def _load_gateway_config(contents: str, path: Path) -> dict[str, object]: + try: + values = tomllib.loads(contents) + except tomllib.TOMLDecodeError as error: + raise GatewayConfigError( + f"{path} is not valid TOML. Fix the reported TOML syntax, then retry." + ) from error + openshell = values.get("openshell") + if not isinstance(openshell, dict) or openshell.get("version") != 1: + raise GatewayConfigError( + f"{path} must contain [openshell] with version = 1. " + "Fix the gateway config version, then retry." + ) + return values + + +def _middleware_entries( + values: dict[str, object], + path: Path, +) -> list[dict[str, object]]: + openshell = values["openshell"] + if not isinstance(openshell, dict): + raise AssertionError("validated OpenShell table is unavailable") + supervisor = openshell.get("supervisor") + if supervisor is None: + return [] + if not isinstance(supervisor, dict): + raise GatewayConfigError( + f"{path} has an invalid [openshell.supervisor] value. " + "Replace it with a TOML table, then retry." + ) + middleware = supervisor.get("middleware") + if middleware is None: + return [] + if not isinstance(middleware, list): + raise GatewayConfigError( + f"{path} has an invalid openshell.supervisor.middleware value. " + "Use [[openshell.supervisor.middleware]] tables, then retry." + ) + entries: list[dict[str, object]] = [] + for entry in middleware: + if not isinstance(entry, dict) or not all( + isinstance(key, str) for key in entry + ): + raise GatewayConfigError( + f"{path} has an invalid openshell.supervisor.middleware value. " + "Use [[openshell.supervisor.middleware]] tables, then retry." + ) + entries.append({str(key): value for key, value in entry.items()}) + return entries + + +def _append_middleware_block( + contents: str, + *, + middleware_name: str, + endpoint: str, +) -> str: + return ( + contents.rstrip() + + "\n\n" + + _middleware_block( + middleware_name=middleware_name, + endpoint=endpoint, + ) + ) + + +def _middleware_block(*, middleware_name: str, endpoint: str) -> str: + return ( + "[[openshell.supervisor.middleware]]\n" + f'name = "{middleware_name}"\n' + f'grpc_endpoint = "{endpoint}"\n' + "max_body_bytes = 4194304\n" + 'timeout = "5s"\n' + ) + + +def _update_middleware_block(block: str, *, endpoint: str) -> str: + updated = _replace_or_append_assignment( + block, + key="grpc_endpoint", + value=f'"{endpoint}"', + ) + updated = _replace_or_append_assignment( + updated, + key="max_body_bytes", + value="4194304", + ) + return _replace_or_append_assignment( + updated, + key="timeout", + value='"5s"', + ) + + +def _replace_or_append_assignment(block: str, *, key: str, value: str) -> str: + pattern = re.compile( + rf"^(?P[ \t]*){re.escape(key)}[ \t]*=.*?$", + flags=re.MULTILINE, + ) + if pattern.search(block): + return pattern.sub( + lambda match: f"{match.group('indent')}{key} = {value}", + block, + count=1, + ) + return block.rstrip() + f"\n{key} = {value}\n" + + +def _write_atomically(path: Path, contents: str) -> None: + try: + path.parent.mkdir(parents=True, exist_ok=True) + mode = stat.S_IMODE(path.stat().st_mode) if path.exists() else 0o600 + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + delete=False, + ) as temporary: + temporary.write(contents) + temporary_path = Path(temporary.name) + temporary_path.chmod(mode) + temporary_path.replace(path) + except OSError as error: + raise GatewayConfigError( + f"Could not write {path}. Check that its directory is writable, then retry." + ) from error + + +_MIDDLEWARE_BLOCK_PATTERN = re.compile( + r"(?ms)^[ \t]*\[\[openshell\.supervisor\.middleware\]\][^\n]*\n" + r".*?(?=^[ \t]*\[\[?[A-Za-z0-9_-]|\Z)" +) + +_MIDDLEWARE_NAME_CHARACTERS = frozenset( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/" +) + + +__all__ = [ + "GatewayConfigError", + "GatewayConfigUpdate", + "MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES", + "default_gateway_config_path", + "update_gateway_config", + "validate_middleware_name", +] diff --git a/projects/privacy-guard/tests/examples/test_custom_engine.py b/projects/privacy-guard/tests/examples/test_custom_engine.py index 195e7ef..706b7e2 100644 --- a/projects/privacy-guard/tests/examples/test_custom_engine.py +++ b/projects/privacy-guard/tests/examples/test_custom_engine.py @@ -6,7 +6,6 @@ import os import subprocess import sys -import tomllib from pathlib import Path import yaml @@ -112,7 +111,6 @@ def test_openshell_walkthrough_uses_the_custom_registry_and_current_policy() -> config = yaml.safe_load( (EXAMPLE_DIRECTORY / "privacy-guard-config.yaml").read_text() ) - gateway = tomllib.loads((EXAMPLE_DIRECTORY / "gateway.toml").read_text()) readme = (EXAMPLE_DIRECTORY / "README.md").read_text() implementation = (EXAMPLE_DIRECTORY / "custom_engine.py").read_text() @@ -124,21 +122,18 @@ def test_openshell_walkthrough_uses_the_custom_registry_and_current_policy() -> middleware_config = policy["network_middlewares"]["privacy_guard_detect"] assert middleware_config["middleware"] == "privacy-guard-custom-engine" assert middleware_config["config"] == config - middleware = gateway["openshell"]["supervisor"]["middleware"] - assert middleware == [ - { - "name": "privacy-guard-custom-engine", - "grpc_endpoint": "http://REPLACE_WITH_HOST_IP:50051", - "max_body_bytes": 4_194_304, - "timeout": "5s", - } - ] stage_config = config["entity_processing"]["stages"][0]["config"] assert stage_config["engine"] == "keyword-tool" assert config["on_detection"]["action"] == "detect" assert "--registry-factory custom_engine:create_registry" in readme assert "cd projects/privacy-guard/examples/custom-engine" in readme assert 'export PYTHONPATH="$PWD${PYTHONPATH:+:$PYTHONPATH}"' in readme + assert "uv run privacy-guard configure-gateway" in readme + assert '--host-ip "$YOUR_HOST_IP"' in readme + assert "--name privacy-guard-custom-engine" in readme + assert "--config gateway.local.toml" in readme + assert 'sed "s/REPLACE_WITH_HOST_IP/' not in readme + assert not (EXAMPLE_DIRECTORY / "gateway.toml").exists() assert "openshell gateway select openshell" in readme assert "openshell gateway add" not in readme assert "OpenShell `v0.0.90`" in readme diff --git a/projects/privacy-guard/tests/examples/test_regex_engine.py b/projects/privacy-guard/tests/examples/test_regex_engine.py index 328249e..96aafc9 100644 --- a/projects/privacy-guard/tests/examples/test_regex_engine.py +++ b/projects/privacy-guard/tests/examples/test_regex_engine.py @@ -6,7 +6,6 @@ import json import subprocess import sys -import tomllib from pathlib import Path import pytest @@ -87,21 +86,11 @@ def test_regex_walkthrough_uses_current_policy_and_gateway_schema() -> None: (EXAMPLE_DIRECTORY / "privacy-guard-config.yaml").read_text() ) catalog = yaml.safe_load((EXAMPLE_DIRECTORY / "patterns.yaml").read_text()) - gateway = tomllib.loads((EXAMPLE_DIRECTORY / "gateway.toml").read_text()) readme = (EXAMPLE_DIRECTORY / "README.md").read_text() assert isinstance(policy, dict) assert isinstance(config, dict) assert isinstance(catalog, dict) - middleware = gateway["openshell"]["supervisor"]["middleware"] - assert middleware == [ - { - "name": "privacy-guard-regex", - "grpc_endpoint": "http://REPLACE_WITH_HOST_IP:50051", - "max_body_bytes": 4_194_304, - "timeout": "5s", - } - ] middleware_config = policy["network_middlewares"]["privacy_guard_replace"] assert middleware_config["middleware"] == "privacy-guard-regex" assert middleware_config["config"] == config @@ -111,6 +100,12 @@ def test_regex_walkthrough_uses_current_policy_and_gateway_schema() -> None: assert stage_config["pattern_catalog"] == "patterns.yaml" RegexPatternCatalog.model_validate(catalog) assert "uv run privacy-guard serve --listen 0.0.0.0:50051" in readme + assert "uv run privacy-guard configure-gateway" in readme + assert '--host-ip "$YOUR_HOST_IP"' in readme + assert "--name privacy-guard-regex" in readme + assert "--config gateway.local.toml" in readme + assert 'sed "s/REPLACE_WITH_HOST_IP/' not in readme + assert not (EXAMPLE_DIRECTORY / "gateway.toml").exists() assert "openshell gateway select openshell" in readme assert "openshell gateway add" not in readme assert "OpenShell `v0.0.90`" in readme diff --git a/projects/privacy-guard/tests/test_cli.py b/projects/privacy-guard/tests/test_cli.py index 5565a1d..e22ef88 100644 --- a/projects/privacy-guard/tests/test_cli.py +++ b/projects/privacy-guard/tests/test_cli.py @@ -6,6 +6,7 @@ import re from collections.abc import Iterator from importlib.metadata import entry_points +from pathlib import Path from types import SimpleNamespace import pytest @@ -32,6 +33,7 @@ def test_cli_help_exposes_server_and_discovery_commands() -> None: output = _plain_output(result) assert "serve" in output assert "configuration-schema" in output + assert "configure-gateway" in output assert "engines" in output assert "--debug" in output assert "--debug-log-content" in output @@ -41,6 +43,109 @@ def test_cli_help_exposes_server_and_discovery_commands() -> None: assert "--scanner-name" not in output +def test_cli_configure_gateway_help_requires_an_explicit_host_ip() -> None: + result = CliRunner().invoke( + app, + ["configure-gateway", "--help"], + terminal_width=240, + ) + + assert result.exit_code == 0 + output = _normalized_output(result) + assert "--host-ip" in output + assert "required" in output.lower() + assert "Non-loopback IPv4" in output + assert "$OPENSHELL_GATEWAY_CONFIG" in output + assert "$XDG_CONFIG_HOME/openshell" in output + assert "1-128 ASCII bytes" in output + assert "restart the OpenShell gateway" not in output + + +def test_cli_configure_gateway_updates_the_default_xdg_config( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + + result = CliRunner().invoke( + app, + ["configure-gateway", "--host-ip", "192.168.1.20"], + ) + + assert result.exit_code == 0 + config_path = tmp_path / "openshell" / "gateway.toml" + assert config_path.exists() + output = _plain_output(result) + assert f"Created {config_path}" in output + assert "Registered privacy-guard at http://192.168.1.20:50051" in output + assert "start Privacy Guard, then restart the OpenShell gateway" in output + + +@pytest.mark.parametrize("host_ip", ["host.openshell.internal", "127.0.0.1", "0.0.0.0"]) +def test_cli_configure_gateway_rejects_unusable_host_ip(host_ip: str) -> None: + result = CliRunner().invoke( + app, + ["configure-gateway", "--host-ip", host_ip], + terminal_width=240, + ) + + assert result.exit_code == 2 + output = _normalized_output(result) + assert "--host-ip" in output + assert "IPv4 address" in output + + +@pytest.mark.parametrize( + "name", + [ + "a" * 129, + "privacy guard", + "openshell/privacy-guard", + ], +) +def test_cli_configure_gateway_rejects_invalid_registration_name( + name: str, +) -> None: + result = CliRunner().invoke( + app, + [ + "configure-gateway", + "--host-ip", + "192.168.1.20", + "--name", + name, + ], + terminal_width=240, + ) + + assert result.exit_code == 2 + assert "--name" in _normalized_output(result) + + +def test_cli_configure_gateway_reports_invalid_existing_config( + tmp_path: Path, +) -> None: + path = tmp_path / "gateway.toml" + path.write_text("not valid TOML") + + result = CliRunner().invoke( + app, + [ + "configure-gateway", + "--host-ip", + "192.168.1.20", + "--config", + str(path), + ], + ) + + assert result.exit_code == 1 + output = _plain_output(result) + assert "Could not configure the OpenShell gateway" in output + assert "not valid TOML" in output + assert path.read_text() == "not valid TOML" + + def test_console_script_targets_the_cli_module() -> None: console_script = next( entry_point diff --git a/projects/privacy-guard/tests/test_gateway_config.py b/projects/privacy-guard/tests/test_gateway_config.py new file mode 100644 index 0000000..1632dfc --- /dev/null +++ b/projects/privacy-guard/tests/test_gateway_config.py @@ -0,0 +1,194 @@ +"""OpenShell gateway configuration update tests.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +import pytest + +from privacy_guard.gateway_config import ( + MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES, + GatewayConfigError, + GatewayConfigUpdate, + default_gateway_config_path, + update_gateway_config, + validate_middleware_name, +) + + +def test_default_gateway_config_path_uses_xdg_config_home( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.delenv("OPENSHELL_GATEWAY_CONFIG", raising=False) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + + assert default_gateway_config_path() == tmp_path / "openshell" / "gateway.toml" + + +def test_default_gateway_config_path_uses_home_config_without_xdg( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.delenv("OPENSHELL_GATEWAY_CONFIG", raising=False) + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + + assert ( + default_gateway_config_path() + == tmp_path / ".config" / "openshell" / "gateway.toml" + ) + + +def test_default_gateway_config_path_honors_openshell_override( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + configured_path = tmp_path / "custom.toml" + monkeypatch.setenv("OPENSHELL_GATEWAY_CONFIG", str(configured_path)) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "ignored")) + + assert default_gateway_config_path() == configured_path + + +def test_middleware_name_validation_matches_openshell_constraints() -> None: + longest_name = "a" * MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES + + assert validate_middleware_name(longest_name) == longest_name + + for invalid_name in ( + "", + "a" * (MAX_MIDDLEWARE_REGISTRATION_NAME_BYTES + 1), + "privacy guard", + "priväcy-guard", + "openshell/privacy-guard", + ): + with pytest.raises(GatewayConfigError): + validate_middleware_name(invalid_name) + + +def test_update_gateway_config_creates_minimal_default_config( + tmp_path: Path, +) -> None: + path = tmp_path / "openshell" / "gateway.toml" + + result = update_gateway_config( + path, + middleware_name="privacy-guard", + host_ip="192.168.1.20", + port=50051, + ) + + assert result is GatewayConfigUpdate.CREATED + assert path.stat().st_mode & 0o777 == 0o600 + assert tomllib.loads(path.read_text()) == { + "openshell": { + "version": 1, + "supervisor": { + "middleware": [ + { + "name": "privacy-guard", + "grpc_endpoint": "http://192.168.1.20:50051", + "max_body_bytes": 4_194_304, + "timeout": "5s", + } + ] + }, + } + } + + +def test_update_gateway_config_appends_without_rewriting_existing_settings( + tmp_path: Path, +) -> None: + path = tmp_path / "gateway.toml" + original = ( + "# Keep this operator comment.\n" + "[openshell]\n" + "version = 1\n\n" + "[openshell.gateway]\n" + 'compute_drivers = ["docker"]\n' + ) + path.write_text(original) + + result = update_gateway_config( + path, + middleware_name="privacy-guard", + host_ip="10.0.0.12", + port=50052, + ) + + assert result is GatewayConfigUpdate.ADDED + assert path.read_text().startswith(original.rstrip() + "\n\n") + + +def test_update_gateway_config_updates_only_the_named_registration( + tmp_path: Path, +) -> None: + path = tmp_path / "gateway.toml" + path.write_text( + "[openshell]\n" + "version = 1\n\n" + "[[openshell.supervisor.middleware]]\n" + 'name = "other-service"\n' + 'grpc_endpoint = "http://10.0.0.2:9000"\n' + "max_body_bytes = 1000\n" + 'timeout = "1s"\n\n' + "[[openshell.supervisor.middleware]]\n" + 'name = "privacy-guard"\n' + "# Keep this registration comment.\n" + 'grpc_endpoint = "http://10.0.0.3:50051"\n' + "max_body_bytes = 2048\n" + 'timeout = "2s"\n' + ) + + result = update_gateway_config( + path, + middleware_name="privacy-guard", + host_ip="10.0.0.4", + port=50053, + ) + + assert result is GatewayConfigUpdate.UPDATED + contents = path.read_text() + assert 'grpc_endpoint = "http://10.0.0.2:9000"' in contents + assert "# Keep this registration comment." in contents + assert 'grpc_endpoint = "http://10.0.0.4:50053"' in contents + assert "max_body_bytes = 4194304" in contents + assert 'timeout = "5s"' in contents + + repeated = update_gateway_config( + path, + middleware_name="privacy-guard", + host_ip="10.0.0.4", + port=50053, + ) + + assert repeated is GatewayConfigUpdate.UNCHANGED + + +@pytest.mark.parametrize( + "contents", + [ + "[openshell\n", + "[other]\nversion = 1\n", + "[openshell]\nversion = 2\n", + ], +) +def test_update_gateway_config_rejects_invalid_existing_config( + tmp_path: Path, + contents: str, +) -> None: + path = tmp_path / "gateway.toml" + path.write_text(contents) + + with pytest.raises(GatewayConfigError): + update_gateway_config( + path, + middleware_name="privacy-guard", + host_ip="192.168.1.20", + port=50051, + ) + + assert path.read_text() == contents From a05914b8af600d63d7b95a8eec75a8028998167f Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 17:34:16 +0000 Subject: [PATCH 81/82] refactor(privacy-guard): simplify regex catalog loading --- .../architecture/configuration.md | 9 +- .../architecture/safety-and-limits.md | 17 ++- .../architecture/service-boundary.md | 11 +- .../src/privacy_guard/constants.py | 3 +- .../src/privacy_guard/engines/regex.py | 91 +------------- .../src/privacy_guard/request_processor.py | 5 +- projects/privacy-guard/tests/test_config.py | 114 ------------------ 7 files changed, 26 insertions(+), 224 deletions(-) diff --git a/docs/documentation/privacy-guard/architecture/configuration.md b/docs/documentation/privacy-guard/architecture/configuration.md index 52e9061..2aadd71 100644 --- a/docs/documentation/privacy-guard/architecture/configuration.md +++ b/docs/documentation/privacy-guard/architecture/configuration.md @@ -85,10 +85,11 @@ without aliases, duplicate keys, or unsafe tags. File-backed and inline inputs normalize to the same `RegexPatternCatalog`. The complete validated configuration contains the structured catalog rather -than its source path. File metadata keys a bounded parser cache; the normalized -immutable catalog keys a bounded compiled-rule cache. A content change produces -a different validated configuration and causes the next evaluation to prepare -an active-policy replacement. +than its source path. File-backed catalogs are read and validated on each policy +validation. The normalized immutable catalog keys a bounded compiled-rule +cache, so equivalent rules do not need to be recompiled. A content change +produces a different validated configuration and causes the next evaluation to +prepare an active-policy replacement. Privacy Guard maintains the catalog schema and safety limits but does not ship an authoritative pattern set. Repository catalogs are examples to copy and diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md index 6664017..3b22fec 100644 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ b/docs/documentation/privacy-guard/architecture/safety-and-limits.md @@ -253,21 +253,20 @@ During an update, the fully prepared candidate may coexist with the active processor. Evaluations that already captured the prior processor may keep it alive until they finish. A failed candidate leaves active state unchanged. -Regex's supporting caches remain independently bounded: +Regex's supporting compiled-rule cache remains bounded: | Cache | Weight budget | Entry cap | Entry weight | | --- | ---: | ---: | --- | -| Parsed Regex file catalogs | 8 MiB | 64 | source file bytes | | Compiled Regex rules | 32 MiB | 128 | canonical catalog bytes plus 4 KiB per rule | The compiled-rule allowance accounts for retained backend state that is much -larger than short pattern text. Cache budgets are independent. One entry that -exceeds a Regex cache budget remains valid and usable for the current operation -but is not retained there. This does not invalidate an active or candidate -processor holding those rules. The active policy is not rejected by cache -admission; policy and schema limits bound its shape, while custom-engine -prepared memory remains an operator responsibility. Regex cache skip and -eviction records contain only cache names and weights, plus eviction counts. +larger than short pattern text. One entry that exceeds the budget remains valid +and usable for the current operation but is not retained there. This does not +invalidate an active or candidate processor holding those rules. The active +policy is not rejected by cache admission; policy and schema limits bound its +shape, while custom-engine prepared memory remains an operator responsibility. +Regex cache skip and eviction records contain only the cache name and weights, +plus eviction counts. ## Changing a limit diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index 3c260f1..01ca64f 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -66,13 +66,12 @@ run in the bounded worker pool. Validation still occurs for every evaluation. Equivalent normalized Regex catalogs may reuse an entry in the independent compiled-rule cache during validation and engine construction. -Regex maintains two independent LRU caches. Parsed file catalogs retain at most -64 entries and 8 MiB of source files. Compiled rules retain at most 128 catalogs -and 32 MiB; each entry weighs its canonical catalog bytes plus 4 KiB per rule. -Evicting a compiled-rule entry only removes that cache's reference. A processor -that already uses those rules remains valid. +Regex's compiled-rule LRU retains at most 128 catalogs and 32 MiB. Each entry +weighs its canonical catalog bytes plus 4 KiB per rule. Evicting an entry only +removes the cache's reference. A processor that already uses those rules remains +valid. -An otherwise valid Regex cache entry larger than its byte budget is built and +An otherwise valid compiled-rule entry larger than the byte budget is built and used for the current operation but is not retained in that cache. The active processor itself has no LRU admission budget: it is deliberate process state retained until replacement or shutdown. Regex cache eviction or a skipped entry diff --git a/projects/privacy-guard/src/privacy_guard/constants.py b/projects/privacy-guard/src/privacy_guard/constants.py index 36043ae..4fbda0c 100644 --- a/projects/privacy-guard/src/privacy_guard/constants.py +++ b/projects/privacy-guard/src/privacy_guard/constants.py @@ -49,8 +49,7 @@ MAX_REGEX_CATALOG_FILE_BYTES = 16 * 1024 * 1024 MAX_REGEX_CATALOG_PATH_BYTES = 1024 -# Prepared-state cache budgets. Entry-count caps remain secondary guards. -MAX_REGEX_PARSED_CATALOG_CACHE_BYTES = 8 * 1024 * 1024 +# Prepared-state cache budget. The entry-count cap remains a secondary guard. MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES = 32 * 1024 * 1024 REGEX_COMPILED_RULE_WEIGHT_BYTES = 4 * 1024 diff --git a/projects/privacy-guard/src/privacy_guard/engines/regex.py b/projects/privacy-guard/src/privacy_guard/engines/regex.py index fc1f7d9..4b0323b 100644 --- a/projects/privacy-guard/src/privacy_guard/engines/regex.py +++ b/projects/privacy-guard/src/privacy_guard/engines/regex.py @@ -31,7 +31,6 @@ MAX_REGEX_COMPILED_CACHE_WEIGHT_BYTES, MAX_REGEX_ENTITIES_PER_CATALOG, MAX_REGEX_NAME_BYTES, - MAX_REGEX_PARSED_CATALOG_CACHE_BYTES, MAX_REGEX_PATTERN_BYTES, MAX_REGEX_RULES_PER_CATALOG, REGEX_COMPILED_RULE_WEIGHT_BYTES, @@ -408,12 +407,7 @@ def _load_pattern_catalog_file(value: str) -> RegexPatternCatalog: raise ValueError return _read_pattern_catalog_file( descriptor, - path_text, - metadata.st_dev, - metadata.st_ino, metadata.st_size, - metadata.st_mtime_ns, - metadata.st_ctime_ns, ) except ( OSError, @@ -444,79 +438,16 @@ def _open_pattern_catalog_file(relative_path: Path) -> int: def _read_pattern_catalog_file( descriptor: int, - path: str, - device: int, - inode: int, - size: int, - modified_at_ns: int, - changed_at_ns: int, + expected_size: int, ) -> RegexPatternCatalog: - cache_key = ( - path, - device, - inode, - size, - modified_at_ns, - changed_at_ns, - ) - with _PATTERN_CATALOG_CACHE_LOCK: - cached = _PATTERN_CATALOG_CACHE.get(cache_key) - if cached is not None: - _PATTERN_CATALOG_CACHE.move_to_end(cache_key) - return cached[0] - contents = _read_bounded_file(descriptor) - if len(contents) != size or len(contents) > MAX_REGEX_CATALOG_FILE_BYTES: - raise ValueError - final_metadata = os.fstat(descriptor) - if ( - final_metadata.st_dev, - final_metadata.st_ino, - final_metadata.st_size, - final_metadata.st_mtime_ns, - final_metadata.st_ctime_ns, - ) != (device, inode, size, modified_at_ns, changed_at_ns): + if len(contents) != expected_size or len(contents) > MAX_REGEX_CATALOG_FILE_BYTES: raise ValueError values = yaml.load( contents.decode("utf-8", errors="strict"), Loader=_StrictCatalogLoader, ) - catalog = RegexPatternCatalog.model_validate(values) - if size > MAX_REGEX_PARSED_CATALOG_CACHE_BYTES: - _LOGGER.debug( - "privacy_guard_cache_skip cache=regex_parsed_catalog " - "weight_bytes=%d budget_bytes=%d", - size, - MAX_REGEX_PARSED_CATALOG_CACHE_BYTES, - ) - return catalog - evicted_entries = 0 - evicted_weight_bytes = 0 - with _PATTERN_CATALOG_CACHE_LOCK: - cached = _PATTERN_CATALOG_CACHE.get(cache_key) - if cached is not None: - _PATTERN_CATALOG_CACHE.move_to_end(cache_key) - return cached[0] - _PATTERN_CATALOG_CACHE[cache_key] = (catalog, size) - global _PATTERN_CATALOG_CACHE_WEIGHT_BYTES - _PATTERN_CATALOG_CACHE_WEIGHT_BYTES += size - while ( - len(_PATTERN_CATALOG_CACHE) > _MAX_CACHED_PATTERN_CATALOGS - or _PATTERN_CATALOG_CACHE_WEIGHT_BYTES - > MAX_REGEX_PARSED_CATALOG_CACHE_BYTES - ): - _, (_, evicted_weight) = _PATTERN_CATALOG_CACHE.popitem(last=False) - _PATTERN_CATALOG_CACHE_WEIGHT_BYTES -= evicted_weight - evicted_entries += 1 - evicted_weight_bytes += evicted_weight - if evicted_entries: - _LOGGER.debug( - "privacy_guard_cache_eviction cache=regex_parsed_catalog " - "entries=%d weight_bytes=%d", - evicted_entries, - evicted_weight_bytes, - ) - return catalog + return RegexPatternCatalog.model_validate(values) def _read_bounded_file(descriptor: int) -> bytes: @@ -625,13 +556,6 @@ def _clear_compiled_pattern_cache() -> None: _COMPILED_PATTERN_CACHE_WEIGHT_BYTES = 0 -def _clear_parsed_pattern_catalog_cache() -> None: - global _PATTERN_CATALOG_CACHE_WEIGHT_BYTES - with _PATTERN_CATALOG_CACHE_LOCK: - _PATTERN_CATALOG_CACHE.clear() - _PATTERN_CATALOG_CACHE_WEIGHT_BYTES = 0 - - def _compile_rule( entity: RegexEntity, rule: RegexRule, @@ -780,14 +704,9 @@ def _rendered_template_size(template: str, entity: str) -> int: ConfidenceLevel.HIGH: 2, } _RULE_METADATA_KEY = "rule" -_MAX_CACHED_PATTERN_CATALOGS = 64 -_PATTERN_CATALOG_CACHE: OrderedDict[ - tuple[str, int, int, int, int, int], - tuple[RegexPatternCatalog, int], -] = OrderedDict() -_PATTERN_CATALOG_CACHE_WEIGHT_BYTES = 0 -_PATTERN_CATALOG_CACHE_LOCK = RLock() _MAX_CACHED_COMPILED_CATALOGS = 128 +# Python dict preserves insertion order, but this LRU must move cache hits to the +# newest position and efficiently evict the oldest entry. _COMPILED_PATTERN_CACHE: OrderedDict[ RegexPatternCatalog, tuple[tuple[_CompiledRule, ...], int], diff --git a/projects/privacy-guard/src/privacy_guard/request_processor.py b/projects/privacy-guard/src/privacy_guard/request_processor.py index 78af758..459f6fb 100644 --- a/projects/privacy-guard/src/privacy_guard/request_processor.py +++ b/projects/privacy-guard/src/privacy_guard/request_processor.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections import OrderedDict from collections.abc import Sequence from enum import StrEnum @@ -183,10 +182,10 @@ def process(self, text: str) -> RequestProcessingResult: def _aggregate_detections( stage_results: Sequence[tuple[str, TextProcessingResult]], ) -> tuple[EntityDetectionSummary, ...]: - groups: OrderedDict[ + groups: dict[ tuple[str, str, ConfidenceLevel | None], int, - ] = OrderedDict() + ] = {} for source, result in stage_results: for detection in result.detections: key = (source, detection.entity, detection.confidence) diff --git a/projects/privacy-guard/tests/test_config.py b/projects/privacy-guard/tests/test_config.py index d20538b..58b936d 100644 --- a/projects/privacy-guard/tests/test_config.py +++ b/projects/privacy-guard/tests/test_config.py @@ -1,6 +1,5 @@ from __future__ import annotations -import logging import os import subprocess import sys @@ -213,119 +212,6 @@ def test_catalog_file_change_produces_a_different_validated_config( assert first != second -def test_parsed_catalog_cache_evicts_least_recently_used_file_by_weight( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - regex_module._clear_parsed_pattern_catalog_cache() - monkeypatch.chdir(tmp_path) - paths = tuple(Path(f"sensitive-{suffix}.yaml") for suffix in "abc") - sizes: list[int] = [] - for suffix, path in zip("abc", paths, strict=True): - catalog = _config()["entity_processing"]["stages"][0]["config"][ - "pattern_catalog" - ] - catalog["entities"][0]["rules"][0]["pattern"] = f"pattern-{suffix}" - contents = yaml.safe_dump(catalog) - path.write_text(contents, encoding="utf-8") - sizes.append(len(contents.encode("utf-8"))) - assert len(set(sizes)) == 1 - monkeypatch.setattr( - regex_module, - "MAX_REGEX_PARSED_CATALOG_CACHE_BYTES", - sizes[0] * 2, - ) - - try: - with caplog.at_level(logging.DEBUG, logger="privacy_guard.engines.regex"): - first = regex_module._load_pattern_catalog_file(str(paths[0])) - regex_module._load_pattern_catalog_file(str(paths[1])) - assert regex_module._load_pattern_catalog_file(str(paths[0])) is first - regex_module._load_pattern_catalog_file(str(paths[2])) - - assert tuple( - cache_key[0] for cache_key in regex_module._PATTERN_CATALOG_CACHE - ) == (str(paths[0]), str(paths[2])) - assert regex_module._PATTERN_CATALOG_CACHE_WEIGHT_BYTES == sizes[0] * 2 - assert regex_module._PATTERN_CATALOG_CACHE_WEIGHT_BYTES == sum( - entry[1] for entry in regex_module._PATTERN_CATALOG_CACHE.values() - ) - assert ( - "privacy_guard_cache_eviction cache=regex_parsed_catalog entries=1" - in caplog.text - ) - assert "sensitive-" not in caplog.text - finally: - regex_module._clear_parsed_pattern_catalog_cache() - - -def test_parsed_catalog_cache_skips_oversized_valid_file( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - regex_module._clear_parsed_pattern_catalog_cache() - monkeypatch.chdir(tmp_path) - catalog = _config()["entity_processing"]["stages"][0]["config"]["pattern_catalog"] - catalog["entities"][0]["rules"][0]["pattern"] = "sensitive-pattern" - contents = yaml.safe_dump(catalog) - path = Path("sensitive-oversized.yaml") - path.write_text(contents, encoding="utf-8") - size = len(contents.encode("utf-8")) - monkeypatch.setattr( - regex_module, - "MAX_REGEX_PARSED_CATALOG_CACHE_BYTES", - size - 1, - ) - - try: - with caplog.at_level(logging.DEBUG, logger="privacy_guard.engines.regex"): - first = regex_module._load_pattern_catalog_file(str(path)) - second = regex_module._load_pattern_catalog_file(str(path)) - - assert first is not second - assert regex_module._PATTERN_CATALOG_CACHE == {} - assert regex_module._PATTERN_CATALOG_CACHE_WEIGHT_BYTES == 0 - assert ( - caplog.text.count("privacy_guard_cache_skip cache=regex_parsed_catalog") - == 2 - ) - assert "sensitive-oversized" not in caplog.text - assert "sensitive-pattern" not in caplog.text - finally: - regex_module._clear_parsed_pattern_catalog_cache() - - -def test_parsed_catalog_failure_preserves_existing_weight( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - regex_module._clear_parsed_pattern_catalog_cache() - monkeypatch.chdir(tmp_path) - valid_catalog = _config()["entity_processing"]["stages"][0]["config"][ - "pattern_catalog" - ] - Path("valid.yaml").write_text( - yaml.safe_dump(valid_catalog), - encoding="utf-8", - ) - Path("invalid.yaml").write_text("entities: []\n", encoding="utf-8") - - try: - regex_module._load_pattern_catalog_file("valid.yaml") - retained_keys = tuple(regex_module._PATTERN_CATALOG_CACHE) - retained_weight = regex_module._PATTERN_CATALOG_CACHE_WEIGHT_BYTES - - with pytest.raises(ValueError, match="catalog file is invalid"): - regex_module._load_pattern_catalog_file("invalid.yaml") - - assert tuple(regex_module._PATTERN_CATALOG_CACHE) == retained_keys - assert regex_module._PATTERN_CATALOG_CACHE_WEIGHT_BYTES == retained_weight - finally: - regex_module._clear_parsed_pattern_catalog_cache() - - def test_equivalent_catalogs_reuse_compiled_regex_rules( monkeypatch: pytest.MonkeyPatch, ) -> None: From 127c48a5ceadf6445bc2d0701b71befaa858f0a7 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 27 Jul 2026 17:59:46 +0000 Subject: [PATCH 82/82] ci(docs): require manual production publish --- .github/workflows/docs.yml | 4 ++-- docs/development/index.md | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4200fdd..2a40601 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -59,7 +59,7 @@ jobs: - name: Upload production documentation if: >- github.ref == 'refs/heads/main' && - (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@v7 with: name: production-docs-${{ github.run_id }} @@ -71,7 +71,7 @@ jobs: name: Deploy documentation if: >- github.ref == 'refs/heads/main' && - (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + github.event_name == 'workflow_dispatch' needs: validate runs-on: ubuntu-latest permissions: diff --git a/docs/development/index.md b/docs/development/index.md index 7871d11..0d29668 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -109,9 +109,12 @@ Dependabot pull requests validate with read-only credentials but do not publish previews on the production documentation origin. The `gh-pages` branch stores the composite production site and active previews; -GitHub Pages remains configured with **GitHub Actions** as its publishing source. -Pushes and manual workflow runs from `main` update the production site while -preserving active previews, then deploy the complete branch through the official -Pages artifact workflow. The first production deployment creates `gh-pages` -automatically. To roll back to a revision before preview support, leave the Pages -source set to **GitHub Actions** and rerun the restored documentation workflow. +GitHub Pages remains configured with **GitHub Actions** as its publishing +source. Pushes to `main` validate documentation without publishing it. When the +current documentation is ready for production, manually run the **Docs** +workflow from `main`; that run updates the production site while preserving +active previews, then deploys the complete branch through the official Pages +artifact workflow. The first production deployment creates `gh-pages` +automatically. To roll back to a revision before preview support, leave the +Pages source set to **GitHub Actions** and rerun the restored documentation +workflow.