diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 91d56ef..ef587cf 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -99,14 +99,18 @@ jobs: 'scripts/build-docs.sh', 'scripts/publish-agent-markdown.py', 'scripts/render-dev-notes.py', + 'scripts/stage-privacy-guard-docs.py', 'tests/test_agent_markdown.py', 'tests/test_docs_404.py', 'tests/test_render_dev_notes.py', + 'tests/test_stage_privacy_guard_docs.py', 'zensical.toml', ]); const docsChanged = files.some( ({ filename }) => filename.startsWith('docs/') || - filename.startsWith('overrides/') || exactInputs.has(filename), + filename.startsWith('overrides/') || + filename.startsWith('projects/privacy-guard/docs/') || + exactInputs.has(filename), ); operation = docsChanged ? 'deploy' : 'remove'; } diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index dcc1ce2..6f54a22 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -47,14 +47,18 @@ jobs: 'scripts/build-docs.sh', 'scripts/publish-agent-markdown.py', 'scripts/render-dev-notes.py', + 'scripts/stage-privacy-guard-docs.py', 'tests/test_agent_markdown.py', 'tests/test_docs_404.py', 'tests/test_render_dev_notes.py', + 'tests/test_stage_privacy_guard_docs.py', 'zensical.toml', ]); const docsChanged = files.some( ({ filename }) => filename.startsWith('docs/') || - filename.startsWith('overrides/') || exactInputs.has(filename), + filename.startsWith('overrides/') || + filename.startsWith('projects/privacy-guard/docs/') || + exactInputs.has(filename), ); core.setOutput('operation', docsChanged ? 'deploy' : 'remove'); diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2a40601..2f98ced 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -59,7 +59,8 @@ jobs: - name: Upload production documentation if: >- github.ref == 'refs/heads/main' && - github.event_name == 'workflow_dispatch' + (github.event_name == 'push' || + github.event_name == 'workflow_dispatch') uses: actions/upload-artifact@v7 with: name: production-docs-${{ github.run_id }} @@ -71,7 +72,8 @@ jobs: name: Deploy documentation if: >- github.ref == 'refs/heads/main' && - github.event_name == 'workflow_dispatch' + (github.event_name == 'push' || + github.event_name == 'workflow_dispatch') needs: validate runs-on: ubuntu-latest permissions: diff --git a/.gitignore b/.gitignore index 7d964c6..ee6c962 100644 --- a/.gitignore +++ b/.gitignore @@ -121,6 +121,7 @@ lib/ # Static site and docs output public/ site/ +docs/documentation/privacy-guard/ .docusaurus/ .vitepress/cache/ .vitepress/dist/ diff --git a/AGENTS.md b/AGENTS.md index 03de54f..85d4fe9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,8 @@ documentation for work that uses OpenShell as its runtime. ## Work routing - Put self-contained implementations and experiments in `projects//`. -- Put durable user-facing guides and references in `docs/documentation/`. +- Put project-specific user guides and references in `projects//docs/`. +- Put cross-project user-facing documentation in `docs/documentation/`. - Put Dev Notes (human-written technical notes) `docs/dev-notes/`. - Put agent-facing repository maintenance workflows in `docs/development/`. @@ -32,6 +33,9 @@ read `docs/development/index.md`. `.env.example`. - Do not hand-edit generated Dev Notes cards, bylines, or navigation; use `python3 scripts/render-dev-notes.py`. +- Do not hand-edit generated project-documentation mirrors under + `docs/documentation/`; use the project's canonical `docs/` tree and its + documented staging script. - When adding a project or workflow with distinct conventions, add a nested `AGENTS.md` in that directory instead of expanding this root file. diff --git a/docs/development/index.md b/docs/development/index.md index 0d29668..ceef7e5 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -5,35 +5,39 @@ description: Agent instructions for maintaining the OpenShell Research documenta # Documentation site development -Follow these instructions for changes under `docs/`, `zensical.toml`, the Dev -Notes renderer, or the documentation workflow. Run commands from the repository -root. Use Python 3.10 or newer. +Follow these instructions for changes under `docs/`, a project's `docs/` tree, +`zensical.toml`, the Dev Notes renderer, or the documentation workflow. Run +commands from the repository root. Use Python 3.10 or newer. ## Content routing -- Put durable software knowledge—installation, usage, reproducibility, and known - limitations—under `docs/documentation/`. +- Put project-specific software knowledge—installation, usage, reproducibility, + and known limitations—under `projects//docs/`. The clean build stages + configured project documentation under `docs/documentation/` for publishing. +- Put cross-project software documentation directly under + `docs/documentation/`. - Put dated experiments, benchmarks, releases, use cases, and engineering updates under `docs/dev-notes/`. -- When a Dev Note introduces reusable software, add its durable guide under - `docs/documentation/`, link it to the originating Dev Note, and add it to - `zensical.toml`. +- When a Dev Note introduces reusable software, add its durable guide to the + owning project, link it to the originating Dev Note, and add its staged site + path to `zensical.toml`. ## Agent-readable Markdown -Every canonical content page under `docs/dev-notes/posts/` and -`docs/documentation/` must declare `agent_markdown: true` in its front matter. -The clean site build copies those sources byte-for-byte into `site/` at the same -path relative to `docs/`, and each rendered page links to its same-origin -Markdown source for people and agents. The generated copies under `site/` must -not be edited. - -Every page in the two canonical content directories is published, including -the Documentation index. Keep presentation-only landing pages such as the -homepage and Dev Notes card index, redirect-only pages, obsolete or orphan -project pages, internal development documentation, and the 404 page outside -those directories and do not add the marker to them. Those pages are not -canonical content for agent consumption. +Every canonical content page under `docs/dev-notes/posts/`, +`docs/documentation/`, and a staged project documentation tree must declare +`agent_markdown: true` in its front matter. The clean site build first stages +project documentation, then copies all published Markdown sources byte-for-byte +into `site/` at their site paths. Each rendered page links to its same-origin +Markdown source for people and agents. Generated copies under +`docs/documentation/` and `site/` must not be edited. + +Every page in a published content tree is included, including the Documentation +index. Keep presentation-only landing pages such as the homepage and Dev Notes +card index, redirect-only pages, obsolete or orphan project pages, internal +development documentation, and the 404 page outside those trees and do not add +the marker to them. Those pages are not canonical content for agent +consumption. ## Dev Notes @@ -88,8 +92,10 @@ scripts/build-docs.sh ``` `scripts/build-docs.sh` recreates `.venv-docs`, installs the pinned toolchain, -renders Dev Notes metadata, and runs `zensical build --clean --strict`. Do not -report success unless it completes without issues. +stages canonical Privacy Guard documentation from +`projects/privacy-guard/docs/`, renders Dev Notes metadata, and runs +`zensical build --clean --strict`. Do not report success unless it completes +without issues. For documentation-site changes, serve the complete built artifact before handing the task back: @@ -110,11 +116,11 @@ previews on the production documentation origin. The `gh-pages` branch stores the composite production site and active previews; GitHub Pages remains configured with **GitHub Actions** as its publishing -source. Pushes to `main` validate documentation without publishing it. When the -current documentation is ready for production, manually run the **Docs** -workflow from `main`; that run updates the production site while preserving -active previews, then deploys the complete branch through the official Pages -artifact workflow. The first production deployment creates `gh-pages` -automatically. To roll back to a revision before preview support, leave the -Pages source set to **GitHub Actions** and rerun the restored documentation -workflow. +source. Every successful push to `main` updates the production site while +preserving active previews, then deploys the complete branch through the +official Pages artifact workflow. Merging a pull request therefore publishes +its documentation after the **Docs** workflow passes. A manual **Docs** +workflow dispatch from `main` can republish the current revision without a new +commit. The first production deployment creates `gh-pages` automatically. To +roll back to a revision before preview support, leave the Pages source set to +**GitHub Actions** and rerun the restored documentation workflow. diff --git a/docs/documentation/index.md b/docs/documentation/index.md index 5579bdd..8f2d1e5 100644 --- a/docs/documentation/index.md +++ b/docs/documentation/index.md @@ -9,6 +9,4 @@ agent_markdown: true Technical documentation and references for installing, using, and extending OpenShell Research projects. -- [Privacy Guard architecture](privacy-guard/architecture/index.md): component - boundaries, request flow, extension interfaces, concurrency, failures, and - limits +- [Privacy Guard](privacy-guard/index.md): middleware for protecting sensitive data in OpenShell. diff --git a/docs/documentation/privacy-guard/architecture/configuration.md b/docs/documentation/privacy-guard/architecture/configuration.md deleted file mode 100644 index 2aadd71..0000000 --- a/docs/documentation/privacy-guard/architecture/configuration.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -title: Configuration and text boundary -description: Runtime configuration ownership and the single-text processing contract. -agent_markdown: true ---- - -# Configuration and text boundary - -Privacy Guard processes the complete request body as one text string. Its -runtime configuration is supplied by OpenShell for each evaluation and -normalized into strict structured models before processing. - -## Bytes and text - -The service owns the transport boundary: - -- it validates the advertised 4 MiB request-body limit -- it allows an empty body without invoking an engine -- it decodes each non-empty body as strict UTF-8 -- it passes exactly one `str` to `RequestProcessor` -- detect and block return no body mutation -- replace UTF-8 encodes the final processed text - -Because detect and block return no mutation, OpenShell retains the exact -original bytes. Replace returns the final text even when it happens to equal -the input. - -Headers and media types do not change this behavior. Privacy Guard does not -parse JSON, select nested values, create document regions, or reconstruct -structured payloads. - -## Policy configuration - -The OpenShell policy owns: - -- ordered entity-processing stages -- each stage's exact engine configuration -- entity definitions and detection settings -- engine-specific replacement recipes -- the final detect, block, or replace action - -For example: - -```yaml -entity_processing: - stages: - - name: identifiers - config: - engine: regex - pattern_catalog: - entities: - - name: customer-id - rules: - - name: prefixed-eight-digit-id - pattern: '\bCUST-[0-9]{8}\b' - confidence: high - replacement: - strategy: template - template: "[{entity}]" -on_detection: - action: replace -``` - -`config.engine` is the Pydantic discriminator. `EngineRegistry.finalize()` -builds the complete policy model from the exact config type registered for each -engine. Engine-specific fields therefore validate and serialize without a -generic mapping layer. - -Deployment startup owns operational resources rather than privacy behavior: -installed engine implementations, approved model profiles, clients, endpoints, -credentials, and data-egress constraints. - -## Regex catalogs - -`RegexEngineConfig.pattern_catalog` accepts either a structured -`RegexPatternCatalog` or a relative `.yaml` or `.yml` path: - -```yaml -pattern_catalog: patterns.yaml -``` - -Paths resolve beneath Privacy Guard's working directory. Absolute paths, -traversal, and symlinks are rejected. Catalog files must be bounded UTF-8 YAML -without aliases, duplicate keys, or unsafe tags. - -File-backed and inline inputs normalize to the same `RegexPatternCatalog`. -The complete validated configuration contains the structured catalog rather -than its source path. File-backed catalogs are read and validated on each policy -validation. The normalized immutable catalog keys a bounded compiled-rule -cache, so equivalent rules do not need to be recompiled. A content change -produces a different validated configuration and causes the next evaluation to -prepare an active-policy replacement. - -Privacy Guard maintains the catalog schema and safety limits but does not ship -an authoritative pattern set. Repository catalogs are examples to copy and -adapt, not presets or runtime defaults. - -`ValidateConfig` proves that each Regex pattern has valid syntax and rejects -patterns that immediately match empty input. It cannot prove that every -context-dependent branch consumes text. If a boundary, lookaround, or -alternation produces a zero-width match only when triggering request text is -observed, evaluation rejects the policy as `config_invalid` rather than treating -the result as an internal engine failure. Lookarounds and boundaries remain -valid when the complete match consumes text. - -## Current transport constraint - -The copied OpenShell protocol carries policy configuration in a -per-evaluation `google.protobuf.Struct` limited to 64 KiB. An inline catalog is -bounded by that transport. A file-backed configuration carries only its bounded -relative path through the protocol and loads the deployment-mounted catalog in -the middleware process. - -The service rejects an encoded configuration above that limit before protobuf -conversion or policy validation. A policy may contain at most ten ordered -entity-processing stages. The service validates the complete bounded -configuration and compares it with the single active policy. Equal validated -configurations reuse the active processor; a different valid configuration pays -full engine and processor preparation before atomic replacement. Reuse avoids -repeated engine initialization but does not increase either limit. - -`Struct` carries every number as a double. At this transport boundary only, -Privacy Guard converts finite integral values in the safe integer range -`-(2^53 - 1)` through `2^53 - 1` to Python integers before strict policy -validation. Non-integral and out-of-range values remain floats, so integer -fields reject them. Direct Python registry validation remains strict and does -not perform this transport normalization. Custom-engine integer settings must -therefore remain within the safe range when supplied through OpenShell policy. - -A future self-contained transport for larger expanded catalogs requires an -upstream OpenShell contract for preparing configuration and referring to it -during evaluation. Privacy Guard must not create a private protocol fork. - -## Policy identity - -Privacy Guard compares the complete immutable validated configuration, -including every concrete engine field, nested replacement variant, and expanded -Regex catalog. Equal configurations reuse the active processor. A different -configuration is an update candidate and replaces the active policy only after -complete preparation succeeds. - -The current protocol carries neither an explicit update operation nor a policy -version. Privacy Guard therefore cannot distinguish an intentional user update -from any other changed per-evaluation configuration. OpenShell must send one -consistent policy stream to a process; interleaved old and new configurations -can repeatedly replace one another. A future protocol can make preparation and -activation explicit and let evaluations refer to a versioned policy. - -[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md deleted file mode 100644 index d92d8c9..0000000 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ /dev/null @@ -1,334 +0,0 @@ ---- -title: Entity-processing engines -description: Engine configuration, lifecycle, output contract, and extension rules. -agent_markdown: true ---- - -# Entity-processing engines - -An `EntityProcessingEngine` detects sensitive entities in one text string and -may replace them when requested. It has no access to the user-facing policy -action, request metadata, gRPC, or protobuf messages. - -Each configured stage owns one engine instance. The same instance may serve -concurrent requests, while separate stages may use the same implementation -with different configurations. - -## Engine interface - -An engine: - -1. declares a concrete configuration type and, when needed, a resource type - through generics -2. declares its non-empty set of `supported_strategies` -3. receives validated configuration and operator-owned resources through the - base constructor -4. optionally derives immutable reusable state in `_initialize()` -5. implements `_run(text, strategy, timeout)` -6. returns one `TextProcessingResult` - -The public `run()` method validates input, strategy support, the timeout, and -the complete output contract. Custom engines implement `_run()` and do not -override `run()` or define `__init__`. `_initialize()` is optional, and -`@override` is not required. The base constructor and public wrapper are final -lifecycle methods. Registration rejects direct or inherited overrides and -directs custom engines to `_initialize()` and `_run()` instead. - -## Configuration - -Every concrete config subclasses `EngineConfig` and declares exactly one -literal `engine` discriminator: - -```python -class AcmeEngineConfig(EngineConfig): - engine: Literal["acme-pii"] = "acme-pii" - replacement: AcmeReplacement | None = None -``` - -The object under `EntityProcessingStage.config` is this exact concrete model. -It is validated and serialized as a member of the registry-built Pydantic -discriminated union, then passed unchanged to the engine constructor. - -Custom integer fields remain strict for direct Python callers. OpenShell policy -uses protobuf `Struct`, whose numeric representation is a double, so the service -normalizes finite integral values only within the safe range `-(2^53 - 1)` -through `2^53 - 1`. Values outside that range and non-integral values remain -floats and fail strict integer fields. Nested models and lists follow the same -transport rule. - -`EngineConfig` is a nominal, strict base model. It does not prescribe a -`replacement` field or any other algorithm-specific setting. An engine that -needs a replacement recipe declares that field on its concrete config. Engines -with multiple replacement algorithms may define a nested discriminated union, -using fields such as `replacement.strategy`. An engine whose underlying tool -has intrinsic replacement behavior may support `REPLACE` without declaring a -replacement field at all. - -Configuration contains privacy behavior. An optional `EngineResources` object -contains operator-owned runtime dependencies such as initialized model clients, -SDK adapters, endpoints, credential providers, or approved model profiles. -Resources are registered by the operator and never serialized into policy. - -`EngineResources` is a nominal contract. A concrete resource bundle subclasses -it, is typed as the engine's second generic argument, and must: - -- contain operational dependencies rather than policy behavior -- retain no request text or mutable per-request state -- expose only dependencies that are safe for concurrent engine calls -- avoid relying on construction or mutation during request processing - -For example: - -```python -from dataclasses import dataclass - -from privacy_guard.engines import EngineResources - - -@dataclass(frozen=True) -class AcmeResources(EngineResources): - client: AcmeClient - - -class AcmeEngine(EntityProcessingEngine[AcmeEngineConfig, AcmeResources]): - ... -``` - -Resources are optional. A resource-free engine omits the second generic -argument and receives `None` from the base class: - -```python -class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig]): - ... -``` - -## Invocation strategy - -```python -class EntityProcessingStrategy(StrEnum): - DETECT = "detect" - REPLACE = "replace" -``` - -An engine declares exactly the invocation strategies it supports. A -detection-only engine declares: - -```python -supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) -``` - -A replacement-only engine declares: - -```python -supported_strategies = frozenset({EntityProcessingStrategy.REPLACE}) -``` - -An engine that exposes both operations includes both enum values. Supporting -`REPLACE` does not imply that the engine exposes `DETECT`, even when replacement -requires internal detection. Blocking does not appear here: the processor runs -engines with `DETECT` and applies the block disposition afterward. - -The registry calls `validate_run_config()` with the strategy derived from the -policy action. The base implementation verifies strategy support. An engine -may add technique-specific requirements through `_validate_run_config()`, such -as requiring a template only when `REPLACE` is requested. The engine receives -the strategy, never the user-facing `PolicyAction`. - -## Result contract - -`TextProcessingResult` contains: - -| Field | Meaning | -| --- | --- | -| `text` | Authoritative text returned by this stage | -| `detections` | Bounded tuple of all `EntityDetection` occurrences | - -Each `EntityDetection` contains: - -| Field | Meaning | -| --- | --- | -| `entity` | Bounded entity label | -| `start` | Inclusive Unicode code-point offset in the stage input | -| `end` | Exclusive Unicode code-point offset in the stage input | -| `confidence` | Optional `low`, `medium`, or `high` certainty | -| `metadata` | Optional bounded engine-specific attribution retained inside the processing boundary | - -A detection span is non-empty and must fall within the stage input. The public -wrapper independently enforces stage detection and output-size limits for every -returned result. Engines that produce detections lazily should use -`TextProcessingResult.from_detections()`; this optional convenience stops -consuming the iterable as soon as the stage limit is exceeded, but it is not -the enforcement boundary. - -For `DETECT`, output text must exactly equal input text. For `REPLACE`, a -successful return is the engine's authoritative completed result. Text may not -change without at least one detection. Engines must raise on native partial -failure instead of returning partial output. - -Privacy Guard preserves categorical confidence as supplied by the engine and -does not compare confidence across engines. - -## Custom engine example - -```python -import re -from typing import Literal - -from pydantic import Field - -from privacy_guard.engines import ( - EngineConfig, - EntityDetection, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, -) -from privacy_guard.timeout import Timeout - - -class KeywordEngineConfig(EngineConfig): - engine: Literal["keyword"] = "keyword" - keyword: str = Field(min_length=1) - - -class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig]): - supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - matches = re.finditer(re.escape(self.config.keyword), text) - return TextProcessingResult.from_detections( - text=text, - detections=( - EntityDetection( - entity="keyword", - start=match.start(), - end=match.end(), - ) - for match in matches - ), - ) -``` - -Register the implementation and, when required, its resources before finalizing -the registry: - -```python -registry = EngineRegistry(include_builtin_engines=True) -registry.register(KeywordEngine) -registry = registry.finalize() -``` - -The opt-in constructor argument registers the engines shipped by Privacy Guard -before application-specific engines. Omit it when the registry should contain -only explicitly registered implementations. - -Finalization freezes registration and constructs the exact policy config type, -JSON Schema, and engine discovery metadata. - -The complete example at -`projects/privacy-guard/examples/custom-engine/README.md` keeps the engine, -configuration, and registry factory in one Python file and runs that registry -through discovery, schema generation, serving, and an OpenShell policy. - -## Timeout and concurrency - -One `Timeout` is created for the processor run and passed through every stage. -The public engine wrapper checks it immediately before and after `_run()`, so -ordinary in-memory implementations do not repeat those checks. An engine must -not create a fresh per-stage duration. - -When a delegated API accepts a timeout, pass the remaining duration: - -```python -remaining = timeout.remaining_seconds() -``` - -When that API raises Python's `TimeoutError` on expiration, use the shared -context manager to translate it into Privacy Guard's domain error: - -```python -with timeout.enforce(): - result = client.process(text, timeout=timeout.remaining_seconds()) -``` - -Unique long-running loops may also call `timeout.raise_if_expired()`. -Operations that cannot be interrupted must be documented and bounded -independently. - -Engine configuration, derived state, and injected resources are shared across -concurrent requests. Keep request text, detections, and counters local to -`_run()`. Resources and engines must be concurrency-safe. - -## RegexEngine - -`RegexEngine` owns both regular-expression detection and deterministic template -replacement. Its `RegexPatternCatalog` contains structured entities and ordered -rules. Each rule combines one regex pattern with its optional diagnostic name, -confidence, and flags. Privacy Guard maintains the schema and limits but no -authoritative pattern set. - -Important properties: - -- rules compile once during configuration validation and initialization -- a non-capturing wrapper plus a private trailing named marker preserves - numeric backreferences and proves the configured match completed -- user-defined named groups and inline flags are rejected -- `ignore_case`, `multiline`, `dot_all`, and `ascii` are explicit fields -- detection retains overlapping matches within and across rules -- each backend search receives the shared remaining timeout -- rule names are optional; unnamed rules receive deterministic - diagnostic identities without changing serialized configuration -- catalogs may be supplied inline or through a bounded relative YAML path and - normalize to the same structured configuration -- replacement resolves overlaps by categorical confidence, span length, - offsets, entity, and rule identity -- templates allow literal text and `{entity}` only -- replacement size is projected before output allocation - -The third-party `regex` backend is used because it can interrupt an individual -search when the timeout expires. - -## Tool-specific custom engines - -Tool integrations belong in custom engines until they have a complete, -production-backed implementation. Privacy Guard does not ship placeholder -engine types or runtime protocols that merely resemble a third-party tool. - -The first NeMo Anonymizer integration will be a custom engine. Its configuration -and replacement types should preserve Anonymizer's native concepts, while the -engine itself owns all translation to and from the actual Anonymizer SDK. - -## State - -Cross-request entity memory is intentionally out of scope. The engine API has -no state argument, session identifier, persistent replacement map, or -placeholder storage interface. - -Future memory support requires separate decisions about tenant isolation, -retention, deletion, consistency, and policy ownership. - -## Testing an engine - -Test engines without gRPC: - -- valid and invalid exact configuration and resource types -- detection-only immutability -- replacement behavior and native partial failure -- Unicode offsets and invalid spans -- categorical confidence -- output and detection limits -- timeout propagation and expiration -- deterministic ordering where relevant -- concurrent calls over immutable initialized state -- content-safe errors that omit input and engine secrets - -Processor tests should cover only ordered multi-stage behavior, shared timeout, -policy disposition, aggregate limits, and stage-qualified findings. - -[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/index.md b/docs/documentation/privacy-guard/architecture/index.md deleted file mode 100644 index 2e8d932..0000000 --- a/docs/documentation/privacy-guard/architecture/index.md +++ /dev/null @@ -1,171 +0,0 @@ ---- -title: Privacy Guard architecture -description: System structure, component boundaries, and request flow for Privacy Guard. -agent_markdown: true ---- - -# Privacy Guard architecture - -Privacy Guard is OpenShell middleware that detects and optionally replaces -sensitive entities in provider-bound HTTP request text before OpenShell adds -provider credentials. - -This architecture is a clean break from the earlier implementation. A processor -run receives one UTF-8 text value and invokes an ordered pipeline of -entity-processing engines. Structured-body parsing and compatibility with the -previous extension and policy APIs are intentionally out of scope. - -Privacy Guard can allow the original body, allow a replacement body, or deny -the request. It never sends the provider request itself. - -## Where Privacy Guard runs - -```text -Sandbox process - | - | provider-bound HTTP request - v -OpenShell supervisor - | - | gRPC: SupervisorMiddleware - v -Privacy Guard - | - | allow original body, allow replacement body, or deny - v -OpenShell supervisor - | - | credentials added only after middleware allows - v -Provider -``` - -Privacy Guard implements the OpenShell-owned `SupervisorMiddleware` gRPC -service and advertises one HTTP-request binding in the pre-credentials phase. -The checked-in protocol and generated bindings are canonical copies from -OpenShell and must not be edited locally. - -## Component boundaries - -Source paths on these pages are relative to -`projects/privacy-guard/src/privacy_guard/`. - -- `service/` owns gRPC, protobuf conversion, UTF-8 decoding and encoding, - bounded worker scheduling, active-processor preparation and replacement, and - finding serialization. Outside generated `bindings/`, no other package - imports gRPC or generated bindings. -- `cli.py` owns command parsing, registry-factory loading, engine discovery, - configuration-schema output, logging options, and the adapter that starts - the programmatic server. Top-level `logging.py` provides the shared, - standard-library logging configuration used by the CLI and available to - programmatic deployments. -- `request_processor.py` runs configured stages over one text value, shares one - timeout across them, aggregates detections, and applies the user-facing - policy action. It does not import gRPC or implement an engine's algorithms. -- `engines/` defines the custom-engine contract, registers engine - implementations and operator-owned resources, builds the exact Pydantic - discriminated union, and contains the built-in Regex implementation. Each - engine owns its detection and replacement algorithms. -- `config.py` defines ordered stages and the required policy action. -- top-level `base.py` defines the package-wide strict immutable domain-model - base. -- `string_validators.py` defines shared string validators and field types. - -The OpenShell policy is the single source of privacy behavior: stage order, -each stage's exact engine configuration, entity definitions, replacement -recipes, and the final `detect`, `block`, or `replace` action. Deployment -configuration registers implementations and injects operational resources such -as model profiles, endpoints, clients, and credentials. - -## Request flow - -```text -HttpRequestEvaluation protobuf - | - v -PrivacyGuardMiddleware - validates phase and body size - validates and normalizes policy configuration - reuses the matching active RequestProcessor or prepares and activates - a replacement - decodes a non-empty body as strict UTF-8 - | - v -RequestProcessor.process(text) - derives DETECT or REPLACE engine strategy - creates one shared Timeout - | - v -stage 1 engine.run(current text) - | - v -stage 2 engine.run(stage 1 text) - | - ... - | - v -RequestProcessor - aggregates stage-qualified detections - applies detect, block, or replace - | - v -RequestProcessingResult - | - v -PrivacyGuardMiddleware - serializes bounded findings - encodes replacement text only for replace - | - v -HttpRequestResult protobuf -``` - -The processor passes only `EntityProcessingStrategy.DETECT` or -`EntityProcessingStrategy.REPLACE` to engines. Blocking is a request -disposition and never crosses the engine boundary. - -The processor is synchronous. The service runs it in a dedicated thread pool -so engine work does not block the gRPC event loop. - -## Core data types - -| Type | Meaning | -| --- | --- | -| `PrivacyGuardConfig` | Ordered entity-processing stages and the required action on detection | -| `EntityProcessingStage` | One configured engine invocation with an optional diagnostic name | -| `EngineConfig` | Nominal strict base for an engine's exact policy configuration | -| `EntityProcessingStrategy` | Per-run engine selection: detect or replace | -| `EntityDetection` | One occurrence with stage-input offsets and optional confidence | -| `TextProcessingResult` | One engine's authoritative output text and detections | -| `Timeout` | One monotonic deadline shared across all stages | -| `EntityDetectionSummary` | Bounded stage/entity/confidence aggregate for audit output | -| `RequestProcessingResult` | Allow or deny decision, detection summaries, and replacement text when requested | - -Pydantic domain models are strict, frozen, reject unknown fields, hide rejected -input from validation errors, and suppress sensitive fields from normal -representations. - -## Deliberate omissions - -- Cross-request entity memory is not part of v0. -- Engines do not receive transport metadata or the user-facing policy action. -- There is no parallel execution-plan model; preparation constructs a - `RequestProcessor` directly. -- There is no generic replacement field or replacement-strategy enum. Each - engine owns any replacement settings appropriate to its underlying - algorithm. -- Regex catalogs may be supplied inline or as bounded relative YAML paths - beneath Privacy Guard's working directory. - -## Read next - -- [Request lifecycle](request-lifecycle.md) explains configuration resolution, - ordered execution, actions, and output behavior. -- [Entity-processing engines](entity-processing-engines.md) defines the - extension contract and built-in engines. -- [Configuration and text boundary](configuration.md) covers the one-text - contract, configuration ownership, and current catalog limits. -- [Service boundary](service-boundary.md) covers gRPC adaptation, policy - activation, and concurrency. -- [Safety and limits](safety-and-limits.md) records failure behavior and - resource bounds. diff --git a/docs/documentation/privacy-guard/architecture/request-lifecycle.md b/docs/documentation/privacy-guard/architecture/request-lifecycle.md deleted file mode 100644 index 9a0cda9..0000000 --- a/docs/documentation/privacy-guard/architecture/request-lifecycle.md +++ /dev/null @@ -1,178 +0,0 @@ ---- -title: Request lifecycle -description: How Privacy Guard prepares policy, runs ordered engines, and applies one request action. -agent_markdown: true ---- - -# Request lifecycle - -`RequestProcessor` owns the complete protobuf-free flow for one text value. It -runs configured entity-processing stages in order, aggregates detections, and -applies the policy action. - -The gRPC service owns the surrounding bytes/text and configuration transport -boundaries. - -## Policy configuration - -The policy supplies: - -```yaml -entity_processing: - stages: - - name: optional-diagnostic-name - config: - engine: regex - pattern_catalog: - entities: - - name: email - rules: - - pattern: '...' - confidence: high - replacement: - strategy: template - template: "[{entity}]" -on_detection: - action: detect -``` - -`entity_processing` groups the ordered stages and owns their non-empty-list and -unique-diagnostic-name validation. V0 defines only `stages`. - -Each `EntityProcessingStage` contains: - -- optional `name`, used only as bounded diagnostic provenance -- required `config`, which is the exact concrete configuration model owned by - the selected engine - -`config.engine` is a Pydantic discriminator. After every implementation is -registered, `EngineRegistry.finalize()` constructs a real discriminated union -of their concrete config models. Engine-specific fields and nested replacement -variants therefore validate, serialize, and appear in JSON Schema without a -generic mapping or translation layer. - -When a stage name is omitted, Privacy Guard derives a deterministic one-based -label such as `regex[1]`. All resulting diagnostic names must be unique. - -## Configuration resolution and preparation - -For each evaluation under the current OpenShell protocol, the service: - -1. converts the protobuf `Struct` to a mapping -2. validates it through the finalized registry-backed Pydantic model -3. validates each concrete config against its registered implementation and - injected resources -4. validates the action/replacement compatibility -5. reuses the active `RequestProcessor` when its complete immutable validated - configuration is equal -6. otherwise serializes preparation, constructs every configured engine and a - candidate processor, and atomically activates the candidate only after full - success - -The first evaluation establishes the active policy and pays its preparation -cost. A failed validation or preparation leaves an existing active processor -unchanged and fails the triggering evaluation. `ValidateConfig` performs -validation without constructing engines or changing the active policy. - -There is no separate execution-plan abstraction. The validated stage order -already contains the necessary policy structure, and the active processor -privately retains the corresponding ordered engine instances. - -## Text input - -The service validates the pre-credentials phase and the request body byte -limit before processing. It still validates configuration and resolves the -active processor for an empty body, then immediately allows that body without -invoking an engine. - -A non-empty body must decode as strict UTF-8. The decoded `str` is the only -request input passed to `RequestProcessor`; headers, content type, request ID, -target, and protobuf messages do not cross that boundary. - -The processor validates the encoded UTF-8 byte bound before running the -pipeline. - -## Ordered stage execution - -The processor derives one invocation strategy for the whole pipeline: - -| Policy action | Engine strategy | -| --- | --- | -| `detect` | `DETECT` | -| `block` | `DETECT` | -| `replace` | `REPLACE` | - -`PolicyAction` is never passed to an engine. - -The processor then: - -1. creates one monotonic `Timeout` -2. calls each stage exactly once in policy order -3. passes the current text, invocation strategy, and shared timeout to - `engine.run()` -4. validates intermediate UTF-8 byte and detection limits -5. passes the returned text to the next stage -6. checks the same timeout after the final result - -In detect and block mode, the public engine contract requires returned text to -equal that stage's input. In replace mode, each later stage sees the preceding -stage's processed text. - -Detection offsets always refer to the input revision seen by the producing -stage. Privacy Guard does not reinterpret earlier offsets after a later stage -changes the text. - -If a stage times out, exceeds an execution limit, or fails, its partial text and -detections are discarded. No later stage runs. - -## Detection aggregation - -After all stages succeed, the processor aggregates detections by: - -```text -source stage + entity + confidence representation -``` - -It does not deduplicate across stages. Two stages may have inspected different -text revisions, and confidence values from different tools are not assumed to -be calibrated. - -The aggregate `EntityDetectionSummary` intentionally omits matched text, -surrounding text, offsets, patterns, and raw engine metadata. - -## Applying the policy action - -The processor owns the final disposition: - -| Action | No detections | One or more detections | -| --- | --- | --- | -| `detect` | Allow original body | Allow original body and report detection summaries | -| `block` | Allow original body | Deny with `privacy_guard_blocked` and report detection summaries | -| `replace` | Allow final processed text | Allow final processed text and report detection summaries | - -For `replace`, configuration validation requires every stage to advertise -replacement support and to include its valid engine-specific replacement -recipe. A recipe may remain configured but dormant when the action is changed -to detect or block. - -Replacement behavior belongs to each engine. For example, `RegexEngine` -selects deterministic non-overlapping matches and renders its constrained -template. A custom engine backed by another tool owns that tool's native -replacement operation. `RequestProcessor` does not reproduce either algorithm. - -## Output - -`RequestProcessor.process()` returns a `RequestProcessingResult`: - -- detect and block-without-detections return `ALLOW` without replacement text -- block-with-detections returns `DENY`, detection summaries, and - `privacy_guard_blocked` -- replace returns `ALLOW`, the final text, and detection summaries -- timeout or execution-limit exhaustion returns `DENY` with - `privacy_guard_limit_exceeded` and no partial summaries or replacement - -The service leaves the original request bytes untouched for detect and block. -For replace it UTF-8 encodes the final text and sets `has_body=true`, including -when the final text happens to equal the input. - -[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md deleted file mode 100644 index 3b22fec..0000000 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ /dev/null @@ -1,286 +0,0 @@ ---- -title: Safety and limits -description: Validation ownership, bounded entity processing, failure behavior, and logging rules. -agent_markdown: true ---- - -# Safety and limits - -Privacy Guard bounds request text, engine output, detections, regex execution, -transport output, and concurrency. Limits are part of the architecture, not -optional tuning defaults. - -Package-wide values live in `constants.py`. - -## Validation ownership - -Each layer validates facts it can establish: - -| Layer | Validates | -| --- | --- | -| Policy and engine config models | Strict types, exact fields, discriminators, local field rules, catalog bounds, and regex validity | -| `EngineRegistry` | Registration contracts, exact config and resource types, pure resource-backed config checks, and action/replacement compatibility | -| `EntityProcessingEngine.run()` | Text and invocation types, supported strategy, timeout, result model, spans, detection-only immutability, mutation attribution, and stage output bounds | -| Concrete engine | Algorithm-specific execution, replacement completeness, backend timeout propagation, and native failure normalization | -| `RequestProcessor` | Request text bounds, shared timeout, intermediate output, request-wide detection count, aggregation, and policy disposition | -| Service | Phase, transport body size, strict UTF-8, protobuf conversion, replacement encoding, finding representation, and response limits | - -Keep request-wide policy decisions in the processor and algorithm-specific -behavior in the concrete engine. The service adapts transport but does not -reimplement either. - -## Text, processing, and transport limits - -| Limit | Constant | Value | Owner | -| --- | --- | ---: | --- | -| Incoming request body | `MAX_BODY_BYTES` | 4 MiB | Service | -| Input or intermediate UTF-8 bytes | `MAX_BODY_BYTES` | 4 MiB | Processor | -| Engine output UTF-8 bytes | `MAX_BODY_BYTES` | 4 MiB | Engine wrapper | -| Encoded replacement body | `MAX_BODY_BYTES` | 4 MiB | Service | -| gRPC receive message | `MAX_RECEIVE_MESSAGE_BYTES` | 5 MiB | gRPC server | -| Detections returned by one stage | `MAX_DETECTIONS_PER_STAGE` | 256 | Engine wrapper | -| Detections across one request | `MAX_DETECTIONS_PER_REQUEST` | 4,096 | Processor | -| Default shared timeout | `DEFAULT_TIMEOUT_SECONDS` | 1 second | Server and processor | -| Maximum configured timeout | `MAX_TIMEOUT_SECONDS` | 30 seconds | Server, processor, and `Timeout` | -| Active processor workers | `MAX_CONCURRENT_PROCESSING` | 4 | Service | -| Concurrent gRPC calls | `MAX_CONCURRENT_RPCS` | 16 | gRPC server | - -The gRPC receive allowance is 1 MiB larger than the request body limit for the -protobuf envelope. The service still enforces the advertised 4 MiB body bound. - -One monotonic `Timeout` is shared by all stages and final result validation. -`RegexEngine` passes the remaining duration into every backend search. -Third-party calls that accept a timeout should receive the same remaining -duration; calls that cannot be interrupted continue to occupy a service worker -until they exit. - -Operators configure the internal bound with `--timeout-seconds` or the -programmatic `PrivacyGuardServer(timeout_seconds=...)` argument. OpenShell's -outer middleware timeout must include additional headroom for worker queueing -and configuration or engine preparation beyond this internal bound. - -## Diagnostic and result limits - -| Limit | Constant | Value | -| --- | --- | ---: | -| Bounded diagnostic string | `MAX_DIAGNOSTIC_TEXT_BYTES` | 1,024 UTF-8 bytes | -| Metadata entries per detection | `MAX_FINDING_METADATA_ENTRIES` | 32 | -| Aggregated protobuf finding groups | `MAX_PROTO_FINDING_GROUPS` | 32 | -| Encoded size per protobuf finding | `MAX_PROTO_FINDING_BYTES` | 4 KiB | - -The diagnostic-string bound applies to stage names, entity names, metadata -keys and values, model-profile names, and other audit-safe identifiers built -from the shared domain field type. These values must be non-empty printable -Unicode without control or bidirectional formatting characters. Untrusted -request IDs use the same content and size rules for logging; an invalid request -ID is represented by a constant placeholder and does not change the evaluation -result. The human-readable log message quotes request IDs and escapes ASCII -spaces so their text cannot introduce another key/value token; structured log -records retain the exact validated ID. Regex entity and supplied rule names -have a stricter ASCII grammar and limit described below. - -The processor aggregates occurrences before the service applies protobuf -limits. If a safe result cannot be represented, the service returns a limit -deny with no partial findings. - -## Regex and catalog limits - -| Limit | Constant | Value | -| --- | --- | ---: | -| Regex entity or supplied rule name | `MAX_REGEX_NAME_BYTES` | 128 ASCII bytes | -| Entities per catalog | `MAX_REGEX_ENTITIES_PER_CATALOG` | 2,000 | -| Rules per catalog | `MAX_REGEX_RULES_PER_CATALOG` | 10,000 | -| Pattern string | `MAX_REGEX_PATTERN_BYTES` | 16 KiB | - -Entity and rule names use -`[A-Za-z_][A-Za-z0-9_-]*`. Rule names are optional; their deterministic -derived diagnostic identities are not serialized back into configuration. - -The per-evaluation config `Struct` is limited to 64 KiB. A catalog may satisfy -Privacy Guard's engine limits yet remain too large for the current upstream -protocol. Internal caching does not raise that transport ceiling. - -## Regex execution safety - -`RegexEngine` validates and compiles the complete catalog before accepting -configuration. It: - -- rejects invalid and empty pattern strings -- rejects patterns that match empty input -- rejects user-defined named groups and inline flags -- verifies its private trailing marker on every match -- treats a context-dependent zero-width match as an atomic configuration - failure -- evaluates patterns independently to retain overlaps -- uses the timeout-capable third-party `regex` backend -- caps detections per stage -- projects exact UTF-8 replacement size before rendering - -No timeout, limit, or pattern failure returns partial stage detections or -mutated text. - -## When a limit is exceeded - -The processor returns a successful deny with -`privacy_guard_limit_exceeded` and no partial findings or replacement when: - -- the shared timeout expires -- an engine exceeds its detection or output limit -- intermediate text exceeds the UTF-8 byte limit -- aggregate request detections exceed the limit - -The service returns the same bounded deny when: - -- findings exceed the protobuf group or encoded-size limit -- replacement text cannot be encoded within the body limit -- a deny reason code is not safely representable - -The processor emits a content-safe `timeout` or `resource` limit kind, and the -service adapter emits `resource` for representation limits. Neither log includes -request content or collaborator error text. The deny reason directs users to -check that log, then reduce the request or replacement size, simplify the -configured stages and rules, or increase `--timeout-seconds` or -`PrivacyGuardServer(timeout_seconds=...)` up to 30 seconds. When increasing the -processing timeout, users must give OpenShell's middleware timeout additional -headroom for worker queueing and configuration or engine preparation. These -options reduce bounded output pressure or provide appropriate time without -weakening the fail-closed behavior. - -Malformed input and engine contract or execution failures instead abort the RPC -with a cataloged error. OpenShell then applies the middleware registration's -failure behavior. - -The distinction is deliberate: - -- an exhausted declared processing bound is a closed, successful deny -- invalid user input is an `INVALID_ARGUMENT` RPC failure -- an engine or middleware defect is an `INTERNAL` RPC failure - -## Error catalog - -Production failures use `PrivacyGuardError` and a stable `ErrorCode`. Each -error spec defines: - -- invalid-input or internal classification -- responsible component -- failed operation -- content-safe summary -- remediation hint - -Successful policy and safety-limit denials use stable reason codes plus -content-safe recovery guidance; they do not add domain fields solely for -diagnostic logging. CLI validation failures name the rejected option and state -the accepted form or next action. Cataloged server-startup failures print one -content-safe message and exit nonzero rather than exposing a traceback. -Internal engine and third-party failures remain concise because the owning -boundary translates them before they reach users. - -Caught extension and third-party exceptions are translated at their trust -boundary. Raw exception messages and chains are not exposed. Configuration -errors do not include rejected pattern strings, paths, credentials, provider -endpoints, or request text. - -Use a new catalog entry for a genuinely new externally observable failure. Do -not return raw Pydantic, regex-backend, engine, or protobuf exceptions. - -## Logging - -Default operational logs include: - -- request ID -- evaluation duration -- allow or deny action -- aggregate finding count -- stable error code -- stage source and invocation strategy at debug level - -They do not include: - -- request or replacement text -- matched or surrounding text -- offsets -- patterns or catalog contents -- headers, targets, or credentials -- model endpoints -- arbitrary exception text - -Sensitive domain fields use `repr=False` to reduce accidental exposure. This -does not replace safe exception handling. - -`--debug` enables content-safe processing diagnostics. -`--debug-log-content` deliberately logs complete input and processed text and -emits a warning; use it only in a controlled development environment. - -## Extension requirements - -Custom engines should: - -- use the base constructor and public `run()` wrapper -- return the exact `TextProcessingResult` contract -- return stable, declared entity identifiers rather than values derived from - request text -- keep request data local to `_run()` -- keep initialized state immutable and make resources concurrency-safe -- pass the shared remaining timeout to delegated APIs when they support it -- fail the stage on native partial failure -- use `TextProcessingResult.from_detections()` to bound lazy detection streams -- add manual bounds only when a low-level collaborator allocates proportional - output before Privacy Guard can consume it -- translate expected collaborator failures to the content-safe engine - exception hierarchy -- avoid logging input or caught exception text - -Privacy Guard keeps framework-controlled fields and identifiers that pass its -shared validation content-safe for findings and operational logs. -Operator-installed Python engine code is trusted deployment code: it can access -request text and must honor the declared identifier and logging contract. The -framework cannot make arbitrary trusted Python code non-exfiltrating, so -operators must review and install custom engines with the same care as other -code in the middleware process. - -Do not add retries, fallback providers, persistent request artifacts, or extra -validation without a concrete failure mode and a clear owning layer. - -## State and retention - -Cross-request entity memory is not implemented. Privacy Guard retains one active -validated policy and its immutable configured engine state, but never request -text, detections, or replacement mappings. The active processor is deliberate -process state, not an LRU entry: it remains until successful replacement or -shutdown. - -During an update, the fully prepared candidate may coexist with the active -processor. Evaluations that already captured the prior processor may keep it -alive until they finish. A failed candidate leaves active state unchanged. - -Regex's supporting compiled-rule cache remains bounded: - -| Cache | Weight budget | Entry cap | Entry weight | -| --- | ---: | ---: | --- | -| Compiled Regex rules | 32 MiB | 128 | canonical catalog bytes plus 4 KiB per rule | - -The compiled-rule allowance accounts for retained backend state that is much -larger than short pattern text. One entry that exceeds the budget remains valid -and usable for the current operation but is not retained there. This does not -invalidate an active or candidate processor holding those rules. The active -policy is not rejected by cache admission; policy and schema limits bound its -shape, while custom-engine prepared memory remains an operator responsibility. -Regex cache skip and eviction records contain only the cache name and weights, -plus eviction counts. - -## Changing a limit - -A limit change may affect policy schema, processor behavior, engine contracts, -protobuf serialization, tests, examples, and the OpenShell middleware -registration. - -Before changing a limit: - -1. identify every layer that enforces or advertises it -2. update exact-boundary tests -3. check the failure result immediately above the boundary -4. run the complete project validation -5. benchmark changes that affect regex compilation or request processing -6. coordinate upstream when the copied OpenShell protocol is involved - -[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md deleted file mode 100644 index 01ca64f..0000000 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ /dev/null @@ -1,346 +0,0 @@ ---- -title: Service boundary -description: gRPC adaptation, policy activation, worker scheduling, and server lifecycle. -agent_markdown: true ---- - -# Service boundary - -Outside the generated `bindings/` package, `service/` is the only layer that -imports gRPC or generated protobuf bindings. It translates between OpenShell's -transport contract and Privacy Guard's one-text domain contract. - -The `.proto` file and generated bindings checked into this project are copied -from OpenShell. Protocol changes must land upstream and be adopted from a new -canonical copy; Privacy Guard must not create a private fork. - -## gRPC methods - -Privacy Guard implements the three methods currently defined by -`SupervisorMiddleware`: - -| RPC | Behavior | -| --- | --- | -| `Describe` | Advertises service identity, one pre-credentials HTTP binding, and the 4 MiB body limit | -| `ValidateConfig` | Validates supplied policy configuration and registered resources | -| `EvaluateHttpRequest` | Validates transport input, resolves the active processor, processes text, and returns a decision | - -`Describe` advertises only -`SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS`. Evaluations at any other phase -are invalid input. - -The current manifest message has no field for the finalized policy schema or -engine discovery metadata. Those are available through the local -`privacy-guard configuration-schema` and `privacy-guard engines` commands. - -## Configuration lifecycle - -The current protocol carries a complete `google.protobuf.Struct` on both -validation and every request evaluation. - -`ValidateConfig`: - -1. rejects an encoded `Struct` larger than 64 KiB before conversion -2. converts the `Struct` to a mapping -3. validates the registry-built discriminated policy model -4. validates each exact engine config against registered resources -5. validates replacement support for a replace action -6. returns `valid=true`, or a content-safe reason - -It does not construct engines, change the active policy, contact model providers, -download resources, or write artifacts. Validation runs in the bounded worker -pool so catalog parsing and engine-specific checks do not block the async gRPC -event loop. - -During evaluation, the service validates and normalizes the same config. It -retains one active pair: the complete immutable validated configuration and its -configured `RequestProcessor`. Equal config reuses the processor. When no -processor is active or config differs, one serialized update path constructs -engines from the exact stage configs and operator-injected resources, constructs -a candidate processor, and atomically replaces the active pair only after -preparation succeeds. Failure preserves the prior active pair and fails the -triggering evaluation. - -Validation, resolution or preparation, strict UTF-8 decoding, and processing all -run in the bounded worker pool. Validation still occurs for every evaluation. -Equivalent normalized Regex catalogs may reuse an entry in the independent -compiled-rule cache during validation and engine construction. - -Regex's compiled-rule LRU retains at most 128 catalogs and 32 MiB. Each entry -weighs its canonical catalog bytes plus 4 KiB per rule. Evicting an entry only -removes the cache's reference. A processor that already uses those rules remains -valid. - -An otherwise valid compiled-rule entry larger than the byte budget is built and -used for the current operation but is not retained in that cache. The active -processor itself has no LRU admission budget: it is deliberate process state -retained until replacement or shutdown. Regex cache eviction or a skipped entry -never invalidates it. Process restart discards the active processor, and the -first later evaluation reconstructs it from supplied configuration. - -Each active processor receives the server's operational processing timeout. -The default is 1 second shared across every configured stage. Operators may set -up to 30 seconds with `privacy-guard serve --timeout-seconds`; programmatic -applications pass the same `timeout_seconds` argument to `PrivacyGuardServer`. -OpenShell's outer middleware `timeout` must include the processing timeout plus -additional headroom for worker queueing, repeated configuration validation, -and occasional candidate preparation so the supervisor does not end the RPC -first. - -## Incoming requests - -For each evaluation, the service: - -1. validates the pre-credentials phase -2. validates the encoded context (4 KiB), configuration (64 KiB), target - (32 KiB), headers (128 entries and 64 KiB), and body (4 MiB) limits -3. acquires one bounded worker slot -4. converts transport-safe integral numbers, validates the maximum ten stages, - and resolves or updates the active policy in that worker -5. allows an empty body without invoking an engine -6. decodes a non-empty body as strict UTF-8 -7. calls `RequestProcessor.process(text)` in the same worker - -Request context, target, headers, middleware name, and protobuf values remain at -the service boundary. The request ID is used only for content-safe operational -logging. - -## Outgoing results - -The processor returns `RequestProcessingResult`. The service maps it to -`HttpRequestResult`: - -| Domain result | Protobuf result | -| --- | --- | -| Detect allow | `DECISION_ALLOW`, `has_body=false` | -| Block allow with no detections | `DECISION_ALLOW`, `has_body=false` | -| Replace allow | `DECISION_ALLOW`, `has_body=true`, final UTF-8 body | -| Policy block | `DECISION_DENY`, `privacy_guard_blocked`, no body | -| Limit deny | `DECISION_DENY`, `privacy_guard_limit_exceeded`, no body or partial findings | - -The service checks the encoded replacement size again before serialization. - -### Findings - -The processor has already aggregated occurrences by source stage, entity, and -confidence. For each `EntityDetectionSummary`, the service emits: - -- `type`: `detected_entity` -- `label`: `entity (source-stage)` -- `confidence`: the categorical value or empty when absent -- `count`: aggregate occurrence count - -The current OpenShell `Finding` message has no dedicated source field. -Stage provenance is therefore secondary text in the bounded label while the -entity remains primary. Matched content, offsets, patterns, and raw engine -metadata never cross the protobuf boundary. - -## Concurrency model - -The gRPC server accepts at most 16 concurrent RPCs. Synchronous processing uses -a dedicated executor and semaphore with four active slots: - -```text -gRPC event loop - | - | acquire processing slot - v -4-slot semaphore - | - v -4-thread executor - | - v -configuration validation -and active-policy resolution - | - v -RequestProcessor.process - | - v -ordered engine pipeline -``` - -This keeps synchronous configuration and engine work off the async event loop -and bounds the number of active operations. - -The active processor, engine instances, and injected resources may be used by -multiple worker threads. They must retain no mutable per-request state and must -be safe for concurrent access. Policy preparation is serialized. A candidate is -fully prepared before the active reference is swapped. An evaluation that -already captured the old processor may finish with it while later evaluations -use the replacement; this is an atomic reference cutover, not a draining -cutover. - -## Cancellation - -Cancelling an async RPC cannot stop Python code already running in its worker -thread. The service shields the worker bridge and releases the semaphore slot -only after that worker actually finishes. - -Cancellation therefore cannot create more than four active synchronous -operations. -An engine should pass the remaining shared timeout to any delegated API that -supports bounded execution. A non-preemptible call continues to occupy its slot -until it exits. - -During shutdown, the server stops gRPC and waits for active executor work. - -## Error mapping - -`PrivacyGuardError` classifies each cataloged failure as invalid input or -internal failure: - -| Error kind | gRPC status | -| --- | --- | -| `invalid_input` | `INVALID_ARGUMENT` | -| `internal` | `INTERNAL` | - -This mapping applies to evaluation. `ValidateConfig` instead returns -`valid=false` with a content-safe reason. - -Unexpected failures become `unexpected_service_failure`. The service does not -return caught collaborator messages or exception chains. - -A gRPC failure is distinct from a successful policy deny. OpenShell applies the -middleware registration's failure behavior when an RPC fails. A policy deny is -a successful RPC result that explicitly stops the request. - -## Programmatic server lifecycle - -`PrivacyGuardServer` is the high-level programmatic API. It owns: - -```text -EngineRegistry - -> PrivacyGuardMiddleware - -> gRPC server -``` - -The built-in registry includes `RegexEngine`. Operators register custom engines -and resource-backed tool integrations before registry finalization, then pass -that registry to `PrivacyGuardServer`. - -The registry is an explicit application-scoped dependency, not a global -singleton. `PrivacyGuardServer` and `PrivacyGuardMiddleware` reject unfinalized -registries. A deployment creates and finalizes one registry during startup; -the middleware constructs configured stage engines and the active processor -from that registry. -Different middleware applications in the same process may intentionally use -different engine inventories or runtime resources. - -Synchronous applications use: - -```python -from privacy_guard.engines.registry import create_builtin_registry -from privacy_guard.service import PrivacyGuardServer - -server = PrivacyGuardServer( - create_builtin_registry(), - timeout_seconds=5, -) -server.serve_sync("127.0.0.1:50051") -``` - -Async applications call `await server.serve_async(address)` instead. The -explicit suffixes make the blocking behavior visible at each call site. Both -entry points use the same server instance and lifecycle. - -The listen address uses `host:port` or bracketed IPv6 `[address]:port` form, -with an explicit port from 1 through 65535. Privacy Guard rejects zero and -out-of-range numeric ports before calling gRPC, then verifies that gRPC bound -the requested port before starting the service. - -The server: - -1. creates an unstarted async gRPC server with receive and concurrency limits -2. binds the configured address -3. starts and waits for termination -4. stops gRPC -5. closes the middleware executor - -A bind or asynchronous gRPC startup `RuntimeError` becomes the stable -`server_bind_failed` error. Its catalog operation is the broader `server.start` -phase so the rendered failure does not mislabel a post-bind startup error. - -The server module has no command framework, module-import-string, discovery, -schema-rendering, or command-logging responsibilities. - -## Command-line application - -`privacy_guard.cli` owns the Typer application. The `--registry-factory` option -identifies a Python function in `module:function` form. At startup, the CLI -imports the module, calls the function once, and uses the returned finalized -`EngineRegistry` for the selected command: - -```bash -privacy-guard --registry-factory my_engines:create_registry engines -privacy-guard --registry-factory my_engines:create_registry configuration-schema -privacy-guard --registry-factory my_engines:create_registry serve -``` - -This is a deployment setting, not part of an OpenShell policy. The person -starting Privacy Guard chooses the function, and importing its module executes -Python code, so it must refer to code that the deployer controls. A policy -cannot select a factory or cause Privacy Guard to import a module. - -Applications that start `PrivacyGuardServer` from Python do not need this CLI -option. They create the registry normally and pass it to the server: - -```python -registry = create_registry() -PrivacyGuardServer(registry, timeout_seconds=5).serve_sync() -``` - -The CLI exposes: - -```bash -privacy-guard engines -privacy-guard configuration-schema -privacy-guard serve --listen 127.0.0.1:50051 --timeout-seconds 5 -``` - -Entity behavior comes from policy configuration, not server startup flags. -Engine implementations and operational resources come from the selected -application registry. The processing timeout is an operational server bound, -not policy-controlled behavior. - -## Upstream protocol work - -The intended large-catalog and discovery experience requires coordinated -OpenShell changes for: - -- explicit preparation and activation that accepts expanded configuration - larger than the current 64 KiB evaluation limit -- versioned policy references on evaluations -- finalized policy schema and engine discovery in the manifest -- a dedicated finding source field - -These are protocol evolution items, not local implementation hooks. Until they -land upstream, Privacy Guard continues to validate the per-evaluation config, -treat a changed valid config as an active-policy update, expose discovery -through the CLI, and render stage provenance inside the finding label. - -The protocol cannot distinguish an intentional update from any other changed -configuration. Each process must receive one consistent policy stream. -Interleaved old and new configurations can repeatedly switch the active policy. - -## Testing the boundary - -Service tests should cover: - -- protobuf/domain translation -- manifest fields supported by the current protocol -- pure policy validation -- phase, body-size, and UTF-8 validation -- first activation, equal-config reuse, serialized replacement, failed-update - preservation, and concurrent cutover -- detect, replacement, block, and limit serialization -- finding encoding and limits -- gRPC status mapping -- worker-slot behavior under cancellation -- startup, bind failure, and shutdown - -Engine algorithms and ordered policy semantics belong in engine and processor -tests unless transport translation is essential to the case. - -[Back to the architecture overview](index.md) diff --git a/docs/stylesheets/documentation.css b/docs/stylesheets/documentation.css new file mode 100644 index 0000000..3348bcc --- /dev/null +++ b/docs/stylesheets/documentation.css @@ -0,0 +1,21 @@ +/* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ +/* SPDX-License-Identifier: Apache-2.0 */ + +/* + * Keep documentation diagrams subordinate to the surrounding explanation. + * Small screens retain the full available width so labels remain legible. + */ +.md-typeset img[src*="assets/diagrams/"] { + display: block; + width: 88%; + max-width: 52rem; + height: auto; + margin-right: auto; + margin-left: auto; +} + +@media screen and (max-width: 48rem) { + .md-typeset img[src*="assets/diagrams/"] { + width: 100%; + } +} diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md index c48302a..d84a913 100644 --- a/projects/privacy-guard/AGENTS.md +++ b/projects/privacy-guard/AGENTS.md @@ -42,8 +42,8 @@ Run focused tests while working and `make check` before handoff. - `src/privacy_guard/string_validators.py`: shared string validators and field types - `src/privacy_guard/service/`: gRPC lifecycle and protobuf adapter - `src/privacy_guard/bindings/`: generated protobuf files; never hand-edit -- [`docs/architecture/`](docs/architecture/index.md): symlink to the canonical - site sources under `../../docs/documentation/privacy-guard/architecture/` +- `docs/`: canonical user and architecture documentation; the repository docs + build stages this tree at the public Privacy Guard documentation route - `tests/`: tests that mirror source boundaries - `examples/`: copyable policy-authoring examples diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index efb98dd..d435c61 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -1,18 +1,64 @@ # Privacy Guard -Privacy Guard is an OpenShell supervisor middleware that detects and optionally -replaces sensitive entities in provider-bound request text before credentials -are attached. +Privacy Guard is an OpenShell supervisor middleware that detects, blocks, or +replaces configured entities in provider-bound HTTP request bodies before +OpenShell attaches provider credentials. -This release is a clean-break redesign. It does not preserve the former -`Scanner`, `FormatHandler`, JSON traversal, `observe`, `redact`, startup catalog, -or scanner-profile APIs. A processor run accepts one UTF-8 text body and runs -the policy's entity-processing stages in order. +It processes the complete request body as UTF-8 text through an ordered +pipeline of entity-processing engines. -## Policy experience +> **Experimental:** Privacy Guard is a proof of concept. It reduces exposure on +> provider-bound network requests that OpenShell routes through the middleware; +> it does not guarantee that sensitive data cannot leak. -The OpenShell policy owns entity behavior: ordered stages, each engine's exact -configuration, and the final `detect`, `block`, or `replace` action. +Privacy Guard does not intercept data before a harness writes it to disk. +Prompts, tool output, transcripts, and session histories may therefore retain +raw sensitive values even when the provider-bound request is later replaced or +blocked. Use harness persistence controls and appropriate storage isolation, +retention, and cleanup in addition to Privacy Guard. + +## What it does + +| Policy action | Behavior | +| --- | --- | +| `detect` | Allow the original body and report bounded findings | +| `block` | Deny requests containing configured entities | +| `replace` | Allow the final body returned by replacement-capable engines | + +Findings contain entity, stage, confidence, and count. Framework-controlled +fields and the built-in `RegexEngine` do not add matched text, surrounding +text, offsets, patterns, headers, or credentials. Custom engines must use +stable entity identifiers that are not derived from request text. + +## Developer start + +Requirements: + +- Python 3.11 or newer +- `uv` 0.11 or newer + +From this directory: + +```bash +uv sync --locked +uv run privacy-guard engines +uv run privacy-guard configuration-schema +``` + +Start the built-in `RegexEngine` service locally: + +```bash +uv run privacy-guard serve \ + --listen 127.0.0.1:50051 +``` + +Use `0.0.0.0` when OpenShell sandbox supervisors outside the host network +namespace must reach the service. The development server uses plaintext gRPC; +restrict the port to trusted host and sandbox networks. + +## Policy configuration + +Privacy behavior comes from the OpenShell policy: ```yaml entity_processing: @@ -33,148 +79,54 @@ on_detection: action: replace ``` -`entity_processing.stages` is ordered. In replace mode, each stage receives the -preceding stage's processed text. Detect and block run the same engines with the -detection-only strategy, so replacement recipes may remain configured but -dormant. - -`pattern_catalog` accepts either the structured catalog or a relative path to a -complete `.yaml` or `.yml` catalog. File-backed catalogs are resolved beneath -Privacy Guard's working directory and normalized to the same -`RegexPatternCatalog` model as inline input. Absolute paths, traversal, symlinks, -unsafe YAML tags, aliases, duplicate keys, invalid UTF-8, and oversized files -are rejected. The -[RegexEngine end-to-end example](https://github.com/NVIDIA/OpenShell-Research/blob/main/projects/privacy-guard/examples/regex-engine/README.md) -passes its catalog as `patterns.yaml`. - -## Architecture - -```text -OpenShell HttpRequestEvaluation - -> finalized Pydantic policy union (config.engine discriminator) - -> reuse the matching active RequestProcessor, or fully prepare and - atomically activate a replacement - -> strict UTF-8 decode - -> RequestProcessor.process(one text string) - -> stage 1 engine.run(current text) - -> stage 2 engine.run(stage 1 text) - -> ... - -> policy action: detect, block, or replace - -> safe aggregated entity findings - -> OpenShell HttpRequestResult -``` +`entity_processing.stages` runs in order. In replace mode, each stage receives +the text returned by the preceding stage. Detect and block invoke engines with +the detection-only strategy. -The policy action never crosses the engine boundary. Engines receive only -`EntityProcessingStrategy.DETECT` or `EntityProcessingStrategy.REPLACE`. -Blocking is a request-level disposition owned by `RequestProcessor`. +`RegexEngine` accepts an inline catalog or a relative `.yaml` or `.yml` path: -The copied `proto/supervisor_middleware.proto` and generated bindings are owned -by OpenShell. Update them only through the repository's middleware-kit workflow; -never hand-edit them. Today's protocol carries a `google.protobuf.Struct` -configuration on each evaluation. Privacy Guard validates it every time and -retains one active configured processor. The first evaluation prepares that -processor. A different validated configuration is fully prepared and atomically -replaces it only after preparation succeeds; failure leaves the active processor -unchanged. - -The protocol has no explicit update or policy-version marker, so any changed -per-evaluation configuration is treated as an update. Each Privacy Guard process -must therefore receive one consistent policy stream. Large-catalog preparation -RPCs, versioned policy references, manifest schema fields, and a dedicated -finding-source field require a coordinated change in the canonical OpenShell -protocol rather than a private proto fork. - -## Built-in engines - -### RegexEngine - -`RegexEngine` compiles configured rules once and supports overlapping -detection and deterministic template replacement. It preserves numeric -backreferences by wrapping each rule's pattern in a non-capturing group followed -by a private named marker. Rule names are optional diagnostic identities; -`pattern` is the only field containing the regex string. Equivalent -normalized catalogs reuse a bounded compiled-rule cache across validation and -engine construction. - -The third-party `regex` backend provides enforceable per-search timeouts. -Explicit `ignore_case`, `multiline`, `dot_all`, and `ascii` flags are supported; -inline flags and user-defined named groups are rejected to protect the wrapper -contract. - -Privacy Guard owns the catalog schema but maintains no authoritative patterns. - -## Custom engines - -Custom engines are a first-class extension point. Authors declare one typed -config, optional typed resources, `supported_strategies`, and `_run`. They do not -write `__init__`; `_initialize` is optional, and `@override` is not required. -The public wrapper validates strategy support, input, timeout, spans, output -size, mutation behavior, and result cardinality for every returned result. -`TextProcessingResult.from_detections()` is an optional safe materializer for -lazy detection streams, not the enforcement boundary. - -Resource-backed engines define an `EngineResources` subclass containing their -operator-owned runtime dependencies. Resource bundles may contain initialized -tool clients, SDK adapters, models, endpoints, or credential providers, but -must contain no policy behavior or per-request state and must be safe for -concurrent use. Resource-free engines omit the second -`EntityProcessingEngine` generic argument entirely. - -The first NeMo Anonymizer integration will be implemented as a custom engine, -not as a built-in or placeholder abstraction in Privacy Guard. - -Application startup registers engines and operator-owned resources, then -returns one finalized registry: - -```python -from privacy_guard.engines.registry import EngineRegistry - - -def create_registry() -> EngineRegistry: - registry = EngineRegistry(include_builtin_engines=True) - registry.register(AcmeEngine, resources=AcmeResources(client=client)) - return registry.finalize() +```yaml +pattern_catalog: patterns.yaml ``` -Set `include_builtin_engines=True` when a custom deployment should extend the -standard engine inventory. Leave it at the default `False` for an intentionally -isolated registry. +Relative paths resolve beneath Privacy Guard's working directory. Absolute +paths, traversal, symlinks, unsafe YAML tags, aliases, duplicate keys, invalid +UTF-8, and oversized catalogs are rejected. -Pass that factory to every CLI operation so discovery, schema generation, and -the running server use the same engine inventory. The module must be installed -or otherwise present on Python's import path: +## CLI ```bash -uv run privacy-guard --registry-factory my_engines:create_registry engines -uv run privacy-guard --registry-factory my_engines:create_registry configuration-schema -uv run privacy-guard --registry-factory my_engines:create_registry serve +uv run privacy-guard engines +uv run privacy-guard configuration-schema +uv run privacy-guard configure-gateway --host-ip YOUR_HOST_IPV4 +uv run privacy-guard serve \ + --listen 0.0.0.0:50051 \ + --timeout-seconds 4 ``` -The -[custom engine end-to-end example](https://github.com/NVIDIA/OpenShell-Research/blob/main/projects/privacy-guard/examples/custom-engine/README.md) -contains one Python file with a typed detection config, engine implementation, -and registry factory, plus its OpenShell policy and walkthrough. It also shows -the explicit `PYTHONPATH` setup needed when the factory is a standalone local -module rather than an installed package. +`configure-gateway` adds or updates a Privacy Guard registration in the +OpenShell gateway TOML. Its registration name must match the policy's +`middleware` field. Restart the gateway after changing registrations. -The registry is application-scoped, not a process-global singleton. A -`PrivacyGuardServer` requires an explicit finalized registry. The finalized -registry builds a Pydantic discriminated union containing the exact config type -of every registered engine, so `stage.config` round-trips without dropping -engine-specific or replacement-variant fields. +The processing timeout is one bound shared by every stage. It defaults to 1 +second and cannot exceed 30 seconds. `configure-gateway` writes a five-second +OpenShell middleware timeout, so use a shorter processing timeout or edit the +registration to provide more headroom. Rerunning `configure-gateway` restores +the five-second value. -The base installation has an explicit built-in registry containing -`RegexEngine`: +Use a trusted registry factory for custom engines: -```python -from privacy_guard.engines.registry import create_builtin_registry +```bash +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + engines -registry = create_builtin_registry() +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + serve ``` -## Programmatic server - -The server is a library API independent of the command-line application: +## Python server API ```python from privacy_guard.engines.registry import create_builtin_registry @@ -187,117 +139,40 @@ server = PrivacyGuardServer( server.serve_sync("127.0.0.1:50051") ``` -`serve_sync()` is the blocking entry point. Async applications use the -explicitly named asynchronous counterpart: +Async applications use: ```python await server.serve_async("127.0.0.1:50051") ``` -Listen addresses use `host:port` or bracketed IPv6 `[address]:port` form. The -port must be an explicit integer from 1 through 65535. - -Custom applications pass their own finalized registry to -`PrivacyGuardServer`. The server owns the middleware adapter, bounded gRPC -transport, worker executor, and shutdown lifecycle; it does not load Python -modules, select engines, generate schemas, or parse command-line options. - -Privacy Guard emits records through Python's standard `privacy_guard` logger -and does not configure logging when imported. Applications may use their own -standard-library logging setup or opt into Privacy Guard's concise console -format: - -```python -from privacy_guard.logging import ColorMode, LoggingConfig, configure_logging - -configure_logging(LoggingConfig(level="DEBUG", color_mode=ColorMode.AUTO)) -``` +## Documentation -Repeated calls replace only the handler installed by `configure_logging()`; -application-owned handlers are preserved. The CLI uses this same configuration -path. The default format includes the date, time, padded level, logger name, -and message. It adds level-aware ANSI colors when writing to an interactive -terminal and stays plain when redirected. Set `color_mode=ColorMode.ALWAYS` or -`color_mode=ColorMode.NEVER` for an explicit choice instead of the default -`ColorMode.AUTO`. Its default level is `INFO`, and `--debug` enables -content-safe `DEBUG` diagnostics. `--debug-log-content` additionally logs -complete request and processed text and should be enabled only in a controlled -environment. +- [Overview and end-to-end quickstart](docs/index.md) +- [Configure policies](docs/configuration.md) +- [Run and operate Privacy Guard](docs/operations.md) +- [Use RegexEngine](docs/engines/regex.md) +- [Add a custom engine](docs/engines/custom.md) +- [System architecture](docs/architecture/index.md) +- [Limits and failure behavior](docs/reference/limits-and-failures.md) -## CLI +## Runnable examples -```bash -uv run privacy-guard engines -uv run privacy-guard configuration-schema -uv run privacy-guard configure-gateway --host-ip YOUR_HOST_IPV4 -uv run privacy-guard serve \ - --listen 127.0.0.1:50051 \ - --timeout-seconds 5 -``` +- [`examples/regex-engine`](examples/regex-engine/README.md): detect and replace + email addresses and customer IDs with the built-in engine. +- [`examples/custom-engine`](examples/custom-engine/README.md): implement, + register, and run a typed custom engine. -Replace `YOUR_HOST_IPV4` with the non-loopback IPv4 address that the OpenShell -gateway and sandbox supervisors use to reach this host. `configure-gateway` -adds or updates a registration named `privacy-guard` in the path set by -OpenShell's `OPENSHELL_GATEWAY_CONFIG` override, or OpenShell's standard -per-user gateway config location when that variable is unset: -`$XDG_CONFIG_HOME/openshell/gateway.toml` (normally -`~/.config/openshell/gateway.toml`). Privacy Guard does not guess which host -interface is correct. Use `--name` when the policy references a different -registration name, `--port` when Privacy Guard does not listen on 50051, and -`--config` for a nonstandard gateway TOML. OpenShell middleware registration -names must use 1–128 ASCII bytes containing only letters, digits, `.`, `_`, `-`, -or `/`; the `openshell/` prefix is reserved. The command preserves unrelated -settings and updates the named registration when it already exists. Start -Privacy Guard before restarting the gateway, because OpenShell connects to -registered middleware during startup. - -Entity behavior is supplied by OpenShell policy config, not server startup -flags. Deployment startup owns only installed engine implementations and -operator resources such as model profiles, endpoints, clients, and credentials. -The processing timeout is an operational bound shared by every stage; it -defaults to 1 second and may be increased to at most 30 seconds. When increasing -it, configure OpenShell's middleware `timeout` with additional headroom for -worker queueing and configuration or engine preparation so the supervisor does -not end the RPC first. Use `--registry-factory module:factory` for a custom -engine installation. -Registry factories execute operator Python code; load only trusted modules. -The `privacy_guard.cli` module owns the executable application and adapts its -`serve` command to the programmatic server API. - -## Safety and limits - -- Input and replacement bodies are limited to 4 MiB. -- One monotonic `Timeout` is shared across every stage and result validation. -- Regex searches receive the remaining timeout and fail atomically. -- Intermediate text and detection cardinality are bounded. -- Detect and block never return a body mutation; replace returns final text. -- Findings expose entity, bounded confidence, count, and stage provenance. - Framework-controlled fields and compliant custom engines never expose matched - or surrounding text, offsets, patterns, or raw tool metadata. Custom engines - must return stable, declared identifiers rather than request-derived values. -- Engine instances and injected resources must be safe for concurrent requests. -- Cross-request entity memory is intentionally out of scope. - -## Updating the OpenShell protocol - -Privacy Guard uses -[`openshell-middleware-kit`](https://github.com/NVIDIA/OpenShell-Research/blob/main/projects/openshell-middleware-kit/README.md) -to keep its copied protocol and generated Python bindings aligned with an -OpenShell release. -Install the repository's local `omkit`, then update: +## Logging -```bash -uv tool install --force ../openshell-middleware-kit -omkit update --openshell-version v0.0.90 -``` +`--debug` enables content-safe diagnostic records. -The updater replaces only the copied protocol, generated bindings, lockfile, and -`.openshell-middleware-manifest.json` from a validated temporary copy. Review -those generated changes and run `make check`. +`--debug-log-content` logs complete input and processed text. Use it only in a +controlled development environment. -## Development validation +Imported applications can configure the standard `privacy_guard` logger +themselves or use `privacy_guard.logging.configure_logging()`. -The project Makefile exposes the normal workflow: +## Development ```bash make help @@ -307,6 +182,10 @@ make check make check-py311 ``` -`make check` delegates to `scripts/check.sh`, the authoritative local and CI -gate. It runs tests, formatting, lint, `ty`, an import smoke check, and a -dependency audit. +`make check` runs tests, formatting, lint, type checking, an import smoke check, +and a dependency audit. + +The copied `proto/supervisor_middleware.proto` and generated bindings are owned +by OpenShell. Update them through the repository's +[`openshell-middleware-kit`](../openshell-middleware-kit/README.md), then run +`make check`. diff --git a/projects/privacy-guard/analysis/README.md b/projects/privacy-guard/analysis/README.md index ddeebc6..0e6d176 100644 --- a/projects/privacy-guard/analysis/README.md +++ b/projects/privacy-guard/analysis/README.md @@ -13,10 +13,10 @@ Run from `projects/privacy-guard/`: uv run python analysis/render_latency_plot.py ``` -The command writes: +The command writes the documentation asset: ```text -analysis/privacy-guard-latency-vs-prompt-size.svg +docs/assets/analysis/privacy-guard-latency-vs-prompt-size.svg ``` Verify that the committed figure matches the data and renderer: diff --git a/projects/privacy-guard/analysis/render_latency_plot.py b/projects/privacy-guard/analysis/render_latency_plot.py index d3bff43..da4ff2b 100644 --- a/projects/privacy-guard/analysis/render_latency_plot.py +++ b/projects/privacy-guard/analysis/render_latency_plot.py @@ -11,7 +11,13 @@ _ANALYSIS_DIR = Path(__file__).resolve().parent _DEFAULT_DATA = _ANALYSIS_DIR / "privacy-guard-latency.csv" -_DEFAULT_OUTPUT = _ANALYSIS_DIR / "privacy-guard-latency-vs-prompt-size.svg" +_DEFAULT_OUTPUT = ( + _ANALYSIS_DIR.parent + / "docs" + / "assets" + / "analysis" + / "privacy-guard-latency-vs-prompt-size.svg" +) _WIDTH = 1200 _HEIGHT = 720 diff --git a/projects/privacy-guard/docs/architecture b/projects/privacy-guard/docs/architecture deleted file mode 120000 index d63f714..0000000 --- a/projects/privacy-guard/docs/architecture +++ /dev/null @@ -1 +0,0 @@ -../../../docs/documentation/privacy-guard/architecture \ No newline at end of file diff --git a/projects/privacy-guard/docs/architecture/index.md b/projects/privacy-guard/docs/architecture/index.md new file mode 100644 index 0000000..ec1ff73 --- /dev/null +++ b/projects/privacy-guard/docs/architecture/index.md @@ -0,0 +1,141 @@ +--- +title: System architecture +description: Privacy Guard components, boundaries, state, and request-processing structure. +agent_markdown: true +--- + +# System architecture + +Privacy Guard is a pre-credentials OpenShell middleware. It translates an +OpenShell HTTP-request evaluation into one text-processing run and returns an +allow, replacement, or deny result. + +This is an experimental network-middleware boundary. It does not mediate +harness filesystem writes, so raw prompts and session histories may be stored +before a request reaches Privacy Guard. + +## System boundary + +![Privacy Guard component architecture. The service layer translates +OpenShell gRPC messages, the request processor applies policy, and registered +engines detect or replace entities without depending on the transport +layer.](../assets/diagrams/component-architecture.svg) + +OpenShell owns request routing and provider credential attachment. Privacy +Guard does not send provider requests. + +## Components + +Source paths below are relative to +`projects/privacy-guard/src/privacy_guard/`. + +| Component | Responsibility | +| --- | --- | +| `cli.py` | CLI commands, registry-factory loading, engine discovery, schema output, and server startup | +| `gateway_config.py` | Safe updates to OpenShell gateway middleware registrations | +| `service/` | gRPC lifecycle, protobuf conversion, transport validation, active configuration, worker scheduling, and result serialization | +| `request_processor.py` | Ordered stage execution, shared timeout, aggregation, and policy action | +| `engines/base.py` | Engine lifecycle and result-contract enforcement | +| `engines/registry.py` | Engine and resource registration, policy schema construction, discovery, and engine creation | +| `engines/regex.py` | Regex catalog validation, matching, overlap handling, caching, and replacement | +| `config.py` | Policy stages and action models | +| `base.py` | Shared strict immutable domain-model base | +| `timeout.py` | Monotonic shared request deadline | +| `errors.py` | Stable content-safe error catalog | +| `logging.py` | Standard-library logging configuration and content-safe records | + +Only `service/` imports gRPC and generated protobuf bindings. Engines and the +request processor use domain models and can be tested without the transport +layer. + +## Configuration and runtime resources + +Privacy behavior comes from the OpenShell policy: + +- ordered stages +- concrete engine configuration +- entity definitions +- replacement recipes +- final `detect`, `block`, or `replace` action + +Operational resources come from the running application: + +- installed engine implementations +- clients and SDK adapters +- endpoints and credential providers +- approved models or profiles +- processing timeout + +`EngineRegistry.finalize()` constructs one Pydantic discriminated union from +the exact config type registered for each engine. `stage.config.engine` selects +the union member. + +## Text boundary + +The service validates request bytes and decodes a non-empty body as strict +UTF-8. The request processor receives exactly one `str`. + +Headers, target, request ID, content type, and protobuf values do not cross the +processor boundary. Privacy Guard does not parse JSON or create structured +regions. + +Detect and block leave the original bytes untouched. Replace UTF-8 encodes the +final text and returns it as the new body. + +## Processing pipeline + +![Privacy Guard processing pipeline from validated policy and input text +through strategy selection, one shared timeout, ordered engine stages, +validation, aggregation, and the final policy +decision.](../assets/diagrams/processing-pipeline.svg) + +Engines receive `DETECT` or `REPLACE`, not the user-facing policy action. +Blocking is applied after detection by `RequestProcessor`. + +## State + +Privacy Guard retains: + +- one active validated policy +- one configured `RequestProcessor` +- immutable configured engine instances +- operator-injected resources +- a bounded compiled Regex catalog cache + +It does not retain request text, detections, replacement mappings, or +cross-request entity memory. + +Equal validated configuration reuses the active processor. A changed +configuration is fully prepared and atomically activated. Failed preparation +does not replace the active processor. + +## Concurrency + +The gRPC server accepts at most 16 concurrent RPCs. Synchronous configuration +and processing use a four-slot worker pool. + +Engine instances and resources can be used by multiple worker threads. They +must keep per-request state local to the call and support concurrent access. +Policy preparation is serialized; evaluations already using the previous +processor may finish while later evaluations use the replacement. + +## Result boundary + +The processor returns only: + +- allow or deny +- final replacement text when applicable +- stage-qualified detection summaries +- a stable deny reason when applicable + +Framework-controlled detection summaries omit matched text, context, offsets, +patterns, and raw engine metadata. Custom engines must keep entity identifiers +stable and independent of request text. + +## Read next + +- [Request lifecycle](request-lifecycle.md) +- [Service boundary](service-boundary.md) +- [Configure policies](../configuration.md) +- [Add a custom engine](../engines/custom.md) +- [Limits and failure behavior](../reference/limits-and-failures.md) diff --git a/projects/privacy-guard/docs/architecture/request-lifecycle.md b/projects/privacy-guard/docs/architecture/request-lifecycle.md new file mode 100644 index 0000000..16fd498 --- /dev/null +++ b/projects/privacy-guard/docs/architecture/request-lifecycle.md @@ -0,0 +1,174 @@ +--- +title: Request lifecycle +description: How Privacy Guard validates configuration, runs stages, and returns an OpenShell request decision. +agent_markdown: true +--- + +# Request lifecycle + +Each OpenShell evaluation carries request metadata, body bytes, and complete +Privacy Guard configuration. The service validates the transport, resolves the +active processor, processes one text value, and serializes the result. + +## Lifecycle summary + +![Privacy Guard request lifecycle. The service validates the transport and +policy, resolves the active processor, handles an empty body or decodes UTF-8 +text, runs the ordered engine stages, then returns either a gRPC failure or a +successful allow, replacement, or deny +result.](../assets/diagrams/request-lifecycle.svg) + +## 1. Validate the transport + +The service checks: + +- pre-credentials middleware phase +- request context size +- policy configuration size +- target size +- header count and size +- request body size + +Input at another middleware phase or outside a transport bound fails the RPC as +invalid input. + +## 2. Validate configuration + +The service converts the protobuf `Struct` to a mapping and validates it through +the finalized registry-backed Pydantic model. + +Validation includes: + +- exact fields and strict types +- known engine discriminator +- stage count and unique stage names +- engine-specific config +- registered resource compatibility +- action and replacement-strategy compatibility +- catalog shape and Regex pattern compilation + +`ValidateConfig` performs these checks without constructing engines or changing +the active processor. + +## 3. Resolve the active processor + +The service compares the complete immutable validated configuration with the +active one: + +| State | Behavior | +| --- | --- | +| Equal configuration | Reuse the active processor | +| No active processor | Construct every stage engine and activate the processor | +| Changed configuration | Prepare a complete candidate and atomically replace the active processor | +| Validation or preparation failure | Keep the active processor unchanged and fail the triggering evaluation | + +Only one preparation path runs at a time. Configuration validation still runs +for every evaluation. + +## 4. Decode the request body + +An empty body is allowed without invoking an engine after configuration +validation and processor resolution. + +A non-empty body must decode as strict UTF-8. The decoded `str` is the only +request input passed to `RequestProcessor`. Headers, target, content type, +request ID, and protobuf objects remain in the service layer. + +## 5. Select the engine strategy + +The processor derives one strategy for the complete stage pipeline: + +| Policy action | Engine strategy | +| --- | --- | +| `detect` | `DETECT` | +| `block` | `DETECT` | +| `replace` | `REPLACE` | + +Engines never receive `PolicyAction`. + +## 6. Run stages in order + +The processor: + +1. validates the input UTF-8 size +2. creates one monotonic `Timeout` +3. calls each stage once in policy order +4. passes the current text, strategy, and shared timeout +5. validates each returned result and intermediate size +6. passes returned text to the next stage +7. checks the timeout after final result validation + +In `DETECT`, each engine must return its input text unchanged. In `REPLACE`, a +later stage receives the preceding stage's output. + +If a stage times out, exceeds a limit, or fails, its partial output is discarded +and later stages do not run. + +## 7. Validate engine output + +The public engine wrapper checks: + +- returned model type +- supported strategy +- non-empty spans within the stage input +- per-stage detection count +- output UTF-8 size +- unchanged text in `DETECT` +- at least one detection when text changes in `REPLACE` + +Detection offsets refer to the stage input. Privacy Guard does not remap earlier +offsets after later replacement stages change the text. + +## 8. Aggregate detections + +After all stages succeed, the processor aggregates occurrences by: + +```text +source stage + entity + confidence +``` + +It does not deduplicate across stages. Stages may inspect different text +revisions, and confidence values from different engines are not assumed to be +calibrated. + +Framework-controlled summary fields exclude matched text, surrounding text, +offsets, patterns, and engine metadata. Engine-provided entity identifiers +cross the boundary, so custom engines must not derive them from request text. + +## 9. Apply the policy action + +| Action | No detections | One or more detections | +| --- | --- | --- | +| `detect` | Allow original body | Allow original body and report summaries | +| `block` | Allow original body | Deny with `privacy_guard_blocked` and report summaries | +| `replace` | Allow final text | Allow final text and report summaries | + +A timeout or processing-limit failure returns a deny with +`privacy_guard_limit_exceeded` and no partial summaries or replacement. + +## 10. Serialize the result + +The service maps the domain result to OpenShell: + +| Domain result | OpenShell result | +| --- | --- | +| Detect allow | `DECISION_ALLOW`, `has_body=false` | +| Block with no detections | `DECISION_ALLOW`, `has_body=false` | +| Replace allow | `DECISION_ALLOW`, `has_body=true`, final UTF-8 body | +| Policy block | `DECISION_DENY`, `privacy_guard_blocked` | +| Limit deny | `DECISION_DENY`, `privacy_guard_limit_exceeded` | + +For detect and block, OpenShell keeps the exact original bytes. For replace, +Privacy Guard returns the final encoded text even when it equals the input. + +## Failure outcomes + +| Outcome | Transport status | Request effect | +| --- | --- | --- | +| Policy block | Successful RPC | Deny | +| Processing limit | Successful RPC | Deny | +| Invalid request or configuration | `INVALID_ARGUMENT` | OpenShell applies middleware `on_error` | +| Internal engine or service error | `INTERNAL` | OpenShell applies middleware `on_error` | + +See [Limits and failure behavior](../reference/limits-and-failures.md) for the +complete error and bound reference. diff --git a/projects/privacy-guard/docs/architecture/service-boundary.md b/projects/privacy-guard/docs/architecture/service-boundary.md new file mode 100644 index 0000000..94ab08f --- /dev/null +++ b/projects/privacy-guard/docs/architecture/service-boundary.md @@ -0,0 +1,178 @@ +--- +title: Service boundary +description: Privacy Guard gRPC methods, protobuf translation, active policy management, and worker scheduling. +agent_markdown: true +--- + +# Service boundary + +The `service/` package adapts OpenShell's `SupervisorMiddleware` gRPC contract +to Privacy Guard's protobuf-free domain model. No other handwritten package +imports gRPC or generated bindings. + +The checked-in protocol and generated bindings are canonical OpenShell copies. + +## gRPC methods + +| RPC | Behavior | +| --- | --- | +| `Describe` | Advertise service identity, the pre-credentials HTTP binding, and the 4 MiB body limit | +| `ValidateConfig` | Validate policy configuration against the active engine registry and resources | +| `EvaluateHttpRequest` | Validate transport input, resolve the configured processor, process text, and return a decision | + +`Describe` advertises only +`SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS`. + +Engine discovery and the finalized policy schema are available through: + +```bash +privacy-guard engines +privacy-guard configuration-schema +``` + +## Configuration transport + +OpenShell supplies the complete policy config as `google.protobuf.Struct` for +validation and evaluation. The service: + +1. rejects an encoded config larger than 64 KiB +2. converts the `Struct` to a mapping +3. normalizes finite integral doubles in the safe integer range +4. validates the registry-built policy model +5. validates engine config against registered resources +6. checks replacement support for `replace` + +`Struct` represents every number as a double. The service converts finite +integral values only within `-(2^53 - 1)` through `2^53 - 1` before strict +model validation. Other values remain floats. + +## Active policy + +The service retains one pair: + +```text +validated PrivacyGuardConfig + configured RequestProcessor +``` + +Equal validated configuration reuses this pair. A changed configuration enters +a serialized preparation path: + +1. construct each configured engine from its exact config and registered + resources +2. construct the candidate processor +3. activate the complete candidate atomically + +Failed preparation leaves the active pair unchanged. An evaluation that already +captured the previous processor can finish with it while later evaluations use +the replacement. + +## Request adaptation + +For each evaluation, the service: + +1. validates the phase +2. validates context, config, target, headers, and body limits +3. acquires a processing slot +4. validates config and resolves the processor in a worker +5. allows an empty body without engine invocation +6. decodes a non-empty body as strict UTF-8 +7. runs `RequestProcessor.process(text)` in the same worker + +Request metadata stays in the service layer. The request ID is used only for +content-safe logging. + +## Result adaptation + +The service maps `RequestProcessingResult` to `HttpRequestResult` and validates +the encoded replacement and finding representation before returning. + +### Findings + +Each aggregated detection summary becomes one OpenShell finding: + +| Finding field | Value | +| --- | --- | +| `type` | `detected_entity` | +| `label` | `entity (source-stage)` | +| `confidence` | `low`, `medium`, `high`, or empty | +| `count` | Number of aggregated occurrences | + +The protobuf has no fields for matched content, context, offsets, patterns, or +raw engine metadata. Engine-provided entity identifiers and configured stage +names do cross in `label`, so custom engines must keep identifiers independent +of request text. + +If the complete result cannot fit OpenShell's finding bounds, the service +returns `privacy_guard_limit_exceeded` without partial findings. + +## Worker scheduling + +The async gRPC server accepts at most 16 concurrent RPCs. Synchronous config +validation, processor preparation, UTF-8 decoding, and engine processing run in +a dedicated four-thread executor guarded by a four-slot semaphore. + +This prevents synchronous engine work from blocking the gRPC event loop and +bounds active processing. + +## Cancellation + +Cancelling the async RPC does not stop Python code already running in a worker +thread. The service keeps the worker bridge alive and releases the semaphore +slot only when that worker exits. + +Engines should pass the shared remaining timeout to interruptible collaborators. +A non-interruptible call continues to occupy its worker slot. + +Server shutdown stops gRPC, waits for active executor work, and closes the +middleware executor. + +## Error mapping + +| Privacy Guard error kind | gRPC status | +| --- | --- | +| Invalid input | `INVALID_ARGUMENT` | +| Internal failure | `INTERNAL` | + +`ValidateConfig` returns `valid=false` with a content-safe reason instead of a +gRPC error for invalid policy configuration. + +Unexpected failures map to the stable `unexpected_service_failure` error. +Collaborator exception messages and chains are not returned. + +A gRPC failure is different from a successful policy deny: + +- OpenShell applies middleware `on_error` after a gRPC failure. +- OpenShell follows the explicit deny after `privacy_guard_blocked` or + `privacy_guard_limit_exceeded`. + +## Server API + +`PrivacyGuardServer` owns the finalized registry, middleware adapter, gRPC +server, and shutdown lifecycle: + +```python +from privacy_guard.engines.registry import create_builtin_registry +from privacy_guard.service import PrivacyGuardServer + +server = PrivacyGuardServer( + create_builtin_registry(), + timeout_seconds=5, +) +server.serve_sync("127.0.0.1:50051") +``` + +Async applications use `await server.serve_async(address)`. + +The server rejects an unfinalized registry, invalid listen address, or port +outside 1 through 65535. Bind and startup failures use the stable +`server_bind_failed` error. + +See [Run and operate Privacy Guard](../operations.md) for CLI startup and +gateway registration. + +## Related pages + +- [System architecture](index.md) +- [Request lifecycle](request-lifecycle.md) +- [Add a custom engine](../engines/custom.md) +- [Limits and failure behavior](../reference/limits-and-failures.md) diff --git a/projects/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg b/projects/privacy-guard/docs/assets/analysis/privacy-guard-latency-vs-prompt-size.svg similarity index 100% rename from projects/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg rename to projects/privacy-guard/docs/assets/analysis/privacy-guard-latency-vs-prompt-size.svg diff --git a/projects/privacy-guard/docs/assets/diagrams/component-architecture.svg b/projects/privacy-guard/docs/assets/diagrams/component-architecture.svg new file mode 100644 index 0000000..e101301 --- /dev/null +++ b/projects/privacy-guard/docs/assets/diagrams/component-architecture.svg @@ -0,0 +1,80 @@ + + Privacy Guard component architecture + Four architecture layers show transport and startup adapters, typed configuration and registry, request processing, and entity-processing engines. Downward arrows show configuration flowing into the request processor and the processor invoking concrete engines through their shared wrapper. + + + + + + + + + TRANSPORT AND STARTUP + + CLI and gateway config + cli.py · gateway_config.py + + OpenShell gRPC adapter + service/ · protobuf validation · result serialization + + + TYPED POLICY AND RUNTIME INVENTORY + + Policy configuration + config.py · ordered stages · final action + + Engine registry + config union · resources · engine construction + + + REQUEST PROCESSING + + RequestProcessor + stage order · one timeout · aggregation · allow or deny + + + ENTITY PROCESSING + + Engine wrapper · contract and bounds + + RegexEngine + built-in implementation + + Custom engines + trusted integrations + + + validate config + + validated policy + + construct engines + run stages + + + diff --git a/projects/privacy-guard/docs/assets/diagrams/processing-pipeline.svg b/projects/privacy-guard/docs/assets/diagrams/processing-pipeline.svg new file mode 100644 index 0000000..05d7818 --- /dev/null +++ b/projects/privacy-guard/docs/assets/diagrams/processing-pipeline.svg @@ -0,0 +1,94 @@ + + Privacy Guard processing pipeline + Validated policy and input text produce one detect or replace strategy and shared timeout. Ordered engine stages run and validate their output. Privacy Guard aggregates detections and applies the final detect, block, or replace action. + + + + + + + + + 1 + Input + Validated policy + UTF-8 text + + + + + 2 + Invocation controls + DETECT or REPLACE + one Timeout + + + + + 3 + Stage 1 + engine.run(current text) + + + + + 4 + Validate result + Text · spans · limits · timeout + + + text + + + 5 + Next stage + Repeat in configured order + + + + + 6 + Aggregate + Stage + entity + confidence + + + + + 7 + Apply the policy action + + + detect + allow original + + block + deny on detection + + replace + allow final text + + + repeat until final stage + diff --git a/projects/privacy-guard/docs/assets/diagrams/request-lifecycle.svg b/projects/privacy-guard/docs/assets/diagrams/request-lifecycle.svg new file mode 100644 index 0000000..340cb18 --- /dev/null +++ b/projects/privacy-guard/docs/assets/diagrams/request-lifecycle.svg @@ -0,0 +1,108 @@ + + Privacy Guard request lifecycle + The request lifecycle has four phases: validate the OpenShell transport, validate and activate configuration, decode and process text, and serialize the result. Empty bodies bypass engine processing. Invalid input fails the RPC, while policy and limit decisions return successful allow or deny results. + + + + + + + + + 1 · TRANSPORT + + Receive + HttpRequestEvaluation + pre-credentials phase + + + Validate bounds + context · config + target · headers · body + phase + + + + + 2 · CONFIGURATION + + Validate policy + typed union · engines + resources · action + + + Resolve processor + reuse equal config + or prepare + activate + + + + + 3 · PROCESSING + + Decode text + strict UTF-8 + empty body bypass + + + Run pipeline + ordered stages + one shared timeout + aggregate findings + + + + + 4 · RESULT + + Apply + detect + block + replace + + + Serialize + allow or deny + body + findings + + + RPC FAILURE + Invalid input or internal failure + INVALID_ARGUMENT or INTERNAL + OpenShell applies middleware on_error + + + + + SUCCESSFUL MIDDLEWARE RESULT + Allow, policy deny, or limit deny + privacy_guard_blocked + privacy_guard_limit_exceeded + + diff --git a/projects/privacy-guard/docs/assets/diagrams/request-path.svg b/projects/privacy-guard/docs/assets/diagrams/request-path.svg new file mode 100644 index 0000000..d19a5a8 --- /dev/null +++ b/projects/privacy-guard/docs/assets/diagrams/request-path.svg @@ -0,0 +1,74 @@ + + Privacy Guard request path + A provider-bound request travels from a sandbox application through the OpenShell supervisor to Privacy Guard. Privacy Guard returns allow, replacement, or deny. OpenShell attaches credentials only after an allow result and then sends the request to the provider. + + + + + + + + + + + Sandbox application + Creates a provider-bound HTTP request + + + request + + + + + OpenShell supervisor + Routes the request before credentials are attached + + + body + policy over gRPC + + + + PRE-CREDENTIALS + + + + Privacy Guard + Detects entities and applies the configured action + + + allow original, replacement, or deny + + + + + OpenShell supervisor + Stops denied requests; attaches credentials after allow + + + authorized request + + + + + Provider + diff --git a/projects/privacy-guard/docs/configuration.md b/projects/privacy-guard/docs/configuration.md new file mode 100644 index 0000000..357769a --- /dev/null +++ b/projects/privacy-guard/docs/configuration.md @@ -0,0 +1,281 @@ +--- +title: Configure policies +description: Configure Privacy Guard stages, actions, catalogs, and OpenShell middleware routing. +agent_markdown: true +--- + +# Configure policies + +Privacy Guard configuration is embedded in an OpenShell +`network_middlewares` entry. The policy determines: + +- which provider endpoints use Privacy Guard +- the order of entity-processing stages +- each engine's exact configuration +- whether detections are reported, blocked, or replaced +- OpenShell's behavior when the middleware RPC fails + +Privacy Guard validates the complete configuration before processing a request. + +## Complete middleware entry + +```yaml +network_middlewares: + privacy_guard_replace: + name: Replace email addresses and customer IDs + middleware: privacy-guard + order: 0 + config: + entity_processing: + stages: + - name: identifiers + config: + engine: regex + pattern_catalog: + entities: + - name: email + rules: + - name: conventional-email + pattern: '(? stage 1 -> stage 2 -> final replacement text +``` + +Detection offsets belong to the input revision seen by the stage that produced +them. Findings aggregate by stage, entity, and confidence. + +## Detection actions + +Set `on_detection.action` to one of: + +| Action | Engine strategy | No detections | Detections | +| --- | --- | --- | --- | +| `detect` | `DETECT` | Allow original body | Allow original body and report findings | +| `block` | `DETECT` | Allow original body | Deny with `privacy_guard_blocked` | +| `replace` | `REPLACE` | Allow final stage output | Allow final stage output and report findings | + +`replace` requires every configured stage to support replacement and to satisfy +its engine-specific replacement requirements. A replacement recipe may remain +configured when the action is `detect` or `block`; it is not used in those +modes. + +## Common policy recipes + +### Detect without changing the request + +```yaml +entity_processing: + stages: + - name: identifiers + config: + engine: regex + pattern_catalog: patterns.yaml +on_detection: + action: detect +``` + +Use this to observe findings while leaving the provider-bound body unchanged. + +### Block requests containing configured entities + +```yaml +entity_processing: + stages: + - name: restricted-values + config: + engine: regex + pattern_catalog: patterns.yaml +on_detection: + action: block +``` + +The request is denied only when at least one configured entity is detected. + +### Replace entities + +```yaml +entity_processing: + stages: + - name: identifiers + config: + engine: regex + pattern_catalog: patterns.yaml + replacement: + strategy: template + template: "[{entity}]" +on_detection: + action: replace +``` + +`{entity}` is replaced with the catalog entity name. For example, +`user@example.com` becomes `[email]`. + +### Run multiple stages + +```yaml +entity_processing: + stages: + - name: structured-identifiers + config: + engine: regex + pattern_catalog: identifiers.yaml + replacement: + strategy: template + template: "[{entity}]" + - name: organization-model + config: + engine: acme-pii + model_profile: organization-default + replacement: + strategy: native +on_detection: + action: replace +``` + +The `acme-pii` engine and its configuration are examples of a custom +installation. The running registry must contain every engine named by the +policy. + +## Regex catalogs + +`RegexEngine` accepts an inline catalog or a relative YAML path. + +Inline: + +```yaml +pattern_catalog: + entities: + - name: customer-id + rules: + - name: prefixed-eight-digit-id + pattern: '\bCUST-[0-9]{8}\b' + confidence: high +``` + +File-backed: + +```yaml +pattern_catalog: patterns.yaml +``` + +Relative paths resolve beneath Privacy Guard's working directory. The path must +end in `.yaml` or `.yml`. Absolute paths, `..` traversal, and symlinks are +rejected. Start Privacy Guard from the directory that contains the referenced +catalog, or use a path relative to that directory. + +See [RegexEngine](engines/regex.md) for the complete catalog schema. + +## Inspect and validate configuration + +List the engines installed in the selected registry: + +```bash +uv run privacy-guard engines +``` + +Print the exact JSON Schema accepted by that registry: + +```bash +uv run privacy-guard configuration-schema +``` + +For a custom registry, pass the same factory to inspection and serving: + +```bash +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + engines + +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + configuration-schema + +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + serve +``` + +Sandbox creation calls `ValidateConfig`. A successful creation proves that the +middleware registration is reachable and that the supplied config matches the +running registry. + +## Policy and deployment ownership + +Keep privacy behavior in policy: + +- stage order +- entity definitions +- detection settings +- engine-specific replacement recipes +- final action + +Keep operational resources in the Privacy Guard deployment: + +- installed engine implementations +- model clients and SDK adapters +- endpoints and credentials +- approved model profiles +- processing timeout + +A policy cannot select a registry factory or import Python code. + +## Configuration activation + +OpenShell sends the complete configuration on each evaluation. Privacy Guard +validates it and compares the normalized immutable result with the active +configuration: + +- equal configuration reuses the active processor +- changed valid configuration is fully prepared, then atomically activated +- failed validation or preparation leaves the active processor unchanged and + fails the triggering evaluation + +Send one consistent configuration stream to each Privacy Guard process. +Interleaving configurations causes the active processor to switch between them. + +The transport configuration is limited to 64 KiB. File-backed Regex catalogs +carry only their relative path through the transport and are loaded by the +Privacy Guard process. + +## Next steps + +- [RegexEngine](engines/regex.md) +- [Add a custom engine](engines/custom.md) +- [Run and operate Privacy Guard](operations.md) +- [Limits and failure behavior](reference/limits-and-failures.md) diff --git a/projects/privacy-guard/docs/engines/custom.md b/projects/privacy-guard/docs/engines/custom.md new file mode 100644 index 0000000..990d122 --- /dev/null +++ b/projects/privacy-guard/docs/engines/custom.md @@ -0,0 +1,325 @@ +--- +title: Add a custom engine +description: Implement, register, run, and test a typed Privacy Guard entity-processing engine. +agent_markdown: true +--- + +# Add a custom engine + +A custom engine integrates another detector or replacement tool with Privacy +Guard. It receives one text string, an invocation strategy, a shared timeout, +and validated engine-specific configuration. It returns processed text and +bounded detections. + +Custom engine code runs inside the Privacy Guard process and can access request +text. Install only reviewed, trusted implementations. + +## Engine contract + +A custom engine defines: + +1. a concrete `EngineConfig` +2. optional typed `EngineResources` +3. supported invocation strategies +4. optional immutable initialization in `_initialize()` +5. request processing in `_run()` + +Do not override `__init__()` or the public `run()` method. The framework-owned +wrapper validates strategy support, timeouts, detection spans, detection +cardinality, output size, and mutation behavior. + +## Minimal detection engine + +```python +import re +from typing import Literal + +from pydantic import Field + +from privacy_guard.engines import ( + EngineConfig, + EntityDetection, + EntityProcessingEngine, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.timeout import Timeout + + +class KeywordEngineConfig(EngineConfig): + engine: Literal["keyword"] = "keyword" + keyword: str = Field(min_length=1, max_length=256) + + +class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig]): + supported_strategies = frozenset( + {EntityProcessingStrategy.DETECT} + ) + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + matches = re.finditer(re.escape(self.config.keyword), text) + return TextProcessingResult.from_detections( + text=text, + detections=( + EntityDetection( + entity="keyword", + start=match.start(), + end=match.end(), + ) + for match in matches + ), + ) +``` + +`TextProcessingResult.from_detections()` stops consuming a lazy detection +stream when the per-stage limit is exceeded. The public engine wrapper remains +the enforcement boundary. + +## Configuration + +Each config class must: + +- subclass `EngineConfig` +- declare one literal `engine` discriminator +- use strict typed fields for all policy-owned behavior +- reject unknown fields through the shared base model +- keep sensitive values out of normal representations when applicable + +```python +class AcmeEngineConfig(EngineConfig): + engine: Literal["acme-pii"] = "acme-pii" + model_profile: str + replacement: AcmeReplacement | None = None +``` + +Privacy Guard adds the exact config type to the registry-built Pydantic +discriminated union. The policy object is passed unchanged to the engine. + +OpenShell transports numbers through protobuf `Struct`. Integer settings must +fit the safe range `-(2^53 - 1)` through `2^53 - 1`. + +## Operational resources + +Use `EngineResources` for deployment-owned clients, adapters, endpoints, +credential providers, or preloaded models: + +```python +from dataclasses import dataclass + +from privacy_guard.engines import EngineResources + + +@dataclass(frozen=True) +class AcmeResources(EngineResources): + client: AcmeClient + + +class AcmeEngine( + EntityProcessingEngine[AcmeEngineConfig, AcmeResources] +): + ... +``` + +Resources must: + +- contain operational dependencies, not policy behavior +- retain no request text or per-request state +- be safe for concurrent use +- be created before request processing + +A resource-free engine omits the second generic argument. + +## Supported strategies + +Declare the exact operations exposed by the engine: + +```python +supported_strategies = frozenset( + { + EntityProcessingStrategy.DETECT, + EntityProcessingStrategy.REPLACE, + } +) +``` + +`block` is not an engine strategy. The processor invokes `DETECT` and applies +the block decision after successful engine execution. + +Override `_validate_run_config()` when a strategy requires additional +configuration. For example, a replacement engine can require a replacement +recipe only when invoked with `REPLACE`. + +## Result requirements + +Return `TextProcessingResult` with: + +| Field | Requirement | +| --- | --- | +| `text` | Complete authoritative stage output | +| `detections` | Every bounded occurrence produced by the stage | + +Each `EntityDetection` provides: + +| Field | Requirement | +| --- | --- | +| `entity` | Stable declared identifier, never a value derived from request text | +| `start` | Inclusive Unicode code-point offset in stage input | +| `end` | Exclusive non-empty offset in stage input | +| `confidence` | Optional `low`, `medium`, or `high` | +| `metadata` | Optional bounded internal attribution | + +For `DETECT`, returned text must exactly equal input text. For `REPLACE`, text +may change only when the result contains at least one detection. Do not return +partial text or detections after a collaborator failure. + +## Timeouts + +One `Timeout` is shared across the complete stage pipeline. Pass its remaining +duration to APIs that accept a timeout: + +```python +result = client.process( + text, + timeout=timeout.remaining_seconds(), +) +``` + +Translate Python `TimeoutError` with the shared context manager: + +```python +with timeout.enforce(): + result = client.process( + text, + timeout=timeout.remaining_seconds(), + ) +``` + +Long-running local loops may call `timeout.raise_if_expired()`. Document and +bound operations that cannot be interrupted. + +## Concurrency + +One configured engine instance may process requests concurrently. Keep request +text, detections, counters, and temporary objects local to `_run()`. Treat +configuration and derived initialization state as immutable. Ensure injected +clients and resources support concurrent calls. + +## Errors and logging + +Translate expected collaborator failures into Privacy Guard's content-safe +engine exceptions. Do not include: + +- input or replacement text +- matched values or surrounding text +- credentials or endpoints +- raw exception messages +- model or SDK response bodies + +Stable engine and entity identifiers may appear in findings and diagnostic +logs when they satisfy shared validation. + +Use a static, content-safe message when translating an operational failure: + +```python +from privacy_guard.engines import EngineExecutionError + +try: + result = self.resources.client.process( + text, + timeout=timeout.remaining_seconds(), + ) +except AcmeClientError: + raise EngineExecutionError("Acme processing failed") from None +``` + +| Exception | Use in a custom engine | +| --- | --- | +| `EngineConfigurationError` | Strategy-specific configuration is unusable | +| `EngineExecutionError` | A collaborator or runtime operation failed | +| `EngineLimitExceededError` | Engine-owned bounded work or output exceeded its limit | + +The framework raises `EngineContractError` when returned text or detections +violate the engine contract; custom engines should not use it for collaborator +failures. + +## Register the engine + +Create one application-scoped registry factory: + +```python +from privacy_guard.engines.registry import EngineRegistry + + +def create_registry() -> EngineRegistry: + registry = EngineRegistry(include_builtin_engines=True) + registry.register(KeywordEngine) + return registry.finalize() +``` + +Pass resources during registration when required: + +```python +registry.register( + AcmeEngine, + resources=AcmeResources(client=client), +) +``` + +Use `include_builtin_engines=True` to add the built-in `RegexEngine`. Omit it +when the registry should contain only explicitly registered custom engines. + +## Inspect and run the registry + +```bash +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + engines + +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + configuration-schema + +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + serve \ + --listen 0.0.0.0:50051 +``` + +The module must be installed or available on `PYTHONPATH`. The factory is +trusted deployment code and executes for each CLI invocation. + +The complete runnable example is in +[`projects/privacy-guard/examples/custom-engine`](https://github.com/NVIDIA/OpenShell-Research/tree/main/projects/privacy-guard/examples/custom-engine). + +## Verify the integration + +Before deploying a custom engine: + +1. Run its unit tests directly against `run()` for every supported strategy. + Assert the exact returned text, entity identifiers, spans, and confidence. + Include Unicode input when offsets come from another library. +2. Run `engines` and `configuration-schema` with the registry factory. Confirm + that the engine and its policy fields appear. +3. Send a representative request through a running Privacy Guard service with + the deployment policy. Confirm the OpenShell decision, replacement body, and + findings. +4. Force collaborator timeouts and failures. Confirm that the request fails + without partial output and that responses and logs contain no request text, + credentials, or raw collaborator errors. + +Test engine-specific behavior and integrations. Privacy Guard's own suite +covers the shared wrapper contract, processor ordering, request-wide limits, +and gRPC result mapping. + +## Related pages + +- [Configure policies](../configuration.md) +- [Run and operate Privacy Guard](../operations.md) +- [System architecture](../architecture/index.md) +- [Limits and failure behavior](../reference/limits-and-failures.md) diff --git a/projects/privacy-guard/docs/engines/index.md b/projects/privacy-guard/docs/engines/index.md new file mode 100644 index 0000000..a20d970 --- /dev/null +++ b/projects/privacy-guard/docs/engines/index.md @@ -0,0 +1,26 @@ +--- +title: Engines +description: Understand how Privacy Guard engines detect and replace sensitive entities. +agent_markdown: true +--- + +# Engines + +Engines are pluggable processors that inspect request text for configured +entities. Each policy stage selects an engine, supplies its configuration, and +receives detections plus replacement text when replacement is enabled. + +Engines do not decide whether Privacy Guard allows or denies a request. The +request processor runs the configured stages in order, enforces shared safety +bounds, and applies the policy's final action to their combined results. + +Privacy Guard includes two integration paths: + +- [RegexEngine](regex.md) provides deterministic detection and replacement + using a deployment-defined pattern catalog. +- [Custom engines](custom.md) integrate another detector, model, SDK, or + service through Privacy Guard's engine contract and registry. + +Use the regex engine when the sensitive values have stable, testable formats. +Add a custom engine when detection requires semantics or an external system +that regular expressions cannot provide reliably. diff --git a/projects/privacy-guard/docs/engines/regex.md b/projects/privacy-guard/docs/engines/regex.md new file mode 100644 index 0000000..2c40a3a --- /dev/null +++ b/projects/privacy-guard/docs/engines/regex.md @@ -0,0 +1,215 @@ +--- +title: RegexEngine +description: Configure RegexEngine catalogs, matching flags, findings, and deterministic replacement. +agent_markdown: true +--- + +# RegexEngine + +`RegexEngine` is the built-in Privacy Guard engine. It detects every configured +regular-expression match and can replace a deterministic non-overlapping subset +with a constrained template. + +Privacy Guard provides the catalog schema and execution bounds. It does not +provide an authoritative pattern catalog. Define and test patterns for the data +your deployment handles. + +## Engine configuration + +```yaml +engine: regex +pattern_catalog: patterns.yaml +replacement: + strategy: template + template: "[{entity}]" +``` + +| Field | Required | Purpose | +| --- | --- | --- | +| `engine` | Yes | Must be `regex` | +| `pattern_catalog` | Yes | Inline catalog or relative YAML path | +| `replacement` | For `replace` actions | Template replacement configuration | + +## Catalog structure + +```yaml +entities: + - name: email + rules: + - name: conventional-email + pattern: '(? **Experimental:** Privacy Guard is a proof of concept, not a guarantee that +> sensitive data cannot leak. It currently protects only provider-bound network +> requests that OpenShell routes through this middleware. + +Privacy Guard does not intercept prompts, tool output, transcripts, or session +history before a harness writes them to disk. Those files may contain raw +sensitive values even when Privacy Guard later replaces or blocks the network +request. Use harness persistence controls and appropriate storage isolation, +retention, and cleanup in addition to Privacy Guard. + +Privacy Guard processes the complete request body as UTF-8 text. It does not +parse JSON fields, inspect files in the sandbox, modify provider responses, or +send network requests to the provider. + +## How it fits into OpenShell + +![A provider-bound request travels from the sandbox through OpenShell and +Privacy Guard. Privacy Guard returns an allow, replacement, or deny decision +before OpenShell attaches credentials and sends an allowed request to the +provider.](assets/diagrams/request-path.svg) + +Privacy Guard exposes OpenShell's `SupervisorMiddleware` gRPC service. It +registers for the pre-credentials request phase so replacement happens before +the provider receives the request. + +## Quickstart + +This quickstart uses the built-in `RegexEngine` example to replace an email +address and customer ID. It requires: + +- Python 3.11 or newer +- `uv` 0.11 or newer +- OpenShell and `openshell-gateway` `v0.0.90`, the version recorded in the + [middleware manifest](https://github.com/NVIDIA/OpenShell-Research/blob/main/projects/privacy-guard/.openshell-middleware-manifest.json) +- a Docker or Podman backend supported by OpenShell +- Claude Code subscription access for the final provider request; the OpenShell + base sandbox supplies the Claude Code binary + +Confirm the installed command versions: + +```bash +uv --version +openshell --version +openshell-gateway --version +``` + +Run the commands from a checkout of OpenShell Research. + +### 1. Prepare the example + +```bash +cd projects/privacy-guard/examples/regex-engine +uv sync --locked +uv run privacy-guard engines +``` + +The engine list should contain: + +```text +regex detect,replace +``` + +The example files are: + +| File | Purpose | +| --- | --- | +| `patterns.yaml` | Email and customer-ID rules | +| `privacy-guard-config.yaml` | Standalone Privacy Guard policy configuration | +| `policy.yaml` | Complete OpenShell sandbox policy with the same configuration | + +### 2. Start Privacy Guard + +From the example directory: + +```bash +uv run privacy-guard serve --listen 0.0.0.0:50051 +``` + +Keep the process running. The development server uses plaintext gRPC and +receives request bodies. Restrict port 50051 to the host and trusted sandbox +network. + +### 3. Register Privacy Guard with the gateway + +Choose a non-loopback host IPv4 address that both the gateway and sandbox +supervisor can reach: + +```bash +# macOS +ipconfig getifaddr en0 + +# Linux +hostname -I +``` + +Create a local gateway configuration: + +```bash +export PRIVACY_GUARD_HOST_IP=YOUR_HOST_IPV4 + +uv run privacy-guard configure-gateway \ + --host-ip "$PRIVACY_GUARD_HOST_IP" \ + --name privacy-guard-regex \ + --config gateway.local.toml +``` + +Do not use `127.0.0.1`: loopback inside the sandbox supervisor does not refer to +the host. + +Restart the local gateway with `gateway.local.toml`. For a Homebrew +installation: + +```bash +brew services stop openshell +export OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" +openshell-gateway --config "$PWD/gateway.local.toml" +``` + +For a Debian or RPM installation: + +```bash +systemctl --user stop openshell-gateway +export OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/tls" +openshell-gateway --config "$PWD/gateway.local.toml" +``` + +Keep the foreground gateway running. + +### 4. Create a sandbox + +In another terminal, return to the example directory and verify the gateway: + +```bash +openshell gateway select openshell +openshell status +``` + +Create the example sandbox: + +```bash +openshell sandbox create \ + --name privacy-guard-regex \ + --from base \ + --no-auto-providers \ + --policy "$PWD/policy.yaml" \ + -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude +``` + +After authenticating Claude Code, submit: + +```text +Draft a short greeting for user@example.com about customer CUST-12345678. +``` + +The provider-bound body should contain `[email]` and `[customer-id]` instead of +the original values. + +### 5. Verify the result + +From a host terminal: + +```bash +openshell logs privacy-guard-regex -n 100 --source sandbox +``` + +Find the provider request and confirm: + +- `transformed:true` +- an `email (identifiers)` finding +- a `customer-id (identifiers)` finding +- no matched email address or customer ID in the findings + +See [Run and operate Privacy Guard](operations.md) for cleanup, gateway +configuration, logging, timeouts, and troubleshooting. + +## Core concepts + +### Policies control behavior + +The OpenShell policy supplies the ordered engine stages and final action. +Privacy Guard startup selects which engine implementations and operational +resources are installed. + +### Stages run in order + +Each stage receives the current text. In `replace` mode, later stages receive +the text returned by earlier stages. In `detect` and `block` mode, engines must +return the input text unchanged. + +### Findings use stable identifiers + +Framework-controlled fields do not add matched text, surrounding text, offsets, +regex patterns, headers, or request bodies to findings. `RegexEngine` uses +configured entity identifiers. Custom engines must also return stable, +declared entity identifiers that are not derived from request text. + +### Processing is bounded + +Request size, output size, detections, regex execution, concurrency, and +processing time have explicit limits. A limit failure denies the request +without returning partial replacement text or partial findings. + +## Documentation map + +- [Configure policies](configuration.md): stages, actions, catalogs, and policy + recipes. +- [Run and operate Privacy Guard](operations.md): CLI, gateway registration, + server lifecycle, logging, and troubleshooting. +- [RegexEngine](engines/regex.md): pattern catalogs, flags, matching, and + replacement. +- [Add a custom engine](engines/custom.md): engine contract, resources, + registry factories, and tests. +- [System architecture](architecture/index.md): components, trust boundaries, + state, and concurrency. +- [Request lifecycle](architecture/request-lifecycle.md): validation, + activation, processing, and result mapping. +- [Service boundary](architecture/service-boundary.md): gRPC methods, + protobuf translation, and worker scheduling. +- [Limits and failure behavior](reference/limits-and-failures.md): enforced + bounds, error outcomes, retention, and measured latency. diff --git a/projects/privacy-guard/docs/operations.md b/projects/privacy-guard/docs/operations.md new file mode 100644 index 0000000..52fe063 --- /dev/null +++ b/projects/privacy-guard/docs/operations.md @@ -0,0 +1,282 @@ +--- +title: Run and operate Privacy Guard +description: Start Privacy Guard, register it with OpenShell, configure logging and timeouts, and troubleshoot failures. +agent_markdown: true +--- + +# Run and operate Privacy Guard + +Privacy Guard runs as a gRPC service reachable by the OpenShell gateway and +sandbox supervisors. The CLI uses the built-in `RegexEngine` registry unless +you supply a custom registry factory. + +## Protection boundary and local persistence + +Privacy Guard is experimental. It evaluates provider-bound network requests at +the OpenShell middleware boundary; it is not a guarantee that sensitive data +cannot leak. + +The middleware does not run before harness filesystem writes. A harness may +persist raw prompts, tool output, transcripts, or session history before +OpenShell sends a provider request to Privacy Guard. Configure the harness to +disable or minimize persistence where possible, restrict access to sandbox and +host storage, set an appropriate retention policy, and remove sensitive session +artifacts after use. + +## Install the project environment + +From the repository checkout: + +```bash +cd projects/privacy-guard +uv sync --locked +``` + +Confirm the installed engines and policy schema: + +```bash +uv run privacy-guard engines +uv run privacy-guard configuration-schema +``` + +## Start the service + +For host-only testing: + +```bash +uv run privacy-guard serve +``` + +The default address is `127.0.0.1:50051`. + +For OpenShell sandbox supervisors running outside the host network namespace: + +```bash +uv run privacy-guard serve \ + --listen 0.0.0.0:50051 \ + --timeout-seconds 4 +``` + +`--timeout-seconds` is one deadline shared by every stage in a request. It +defaults to 1 second and must not exceed 30 seconds. OpenShell's middleware +timeout must be longer to cover worker queueing, configuration validation, and +processor preparation. + +The development server uses plaintext gRPC and receives request bodies. +Restrict its listen port to trusted host and sandbox networks. + +## Register the service with OpenShell + +Choose a non-loopback IPv4 address reachable by both the local gateway and +sandbox supervisors: + +```bash +# macOS +ipconfig getifaddr en0 + +# Linux +hostname -I +``` + +Add or update a gateway registration: + +```bash +export PRIVACY_GUARD_HOST_IP=YOUR_HOST_IPV4 + +uv run privacy-guard configure-gateway \ + --host-ip "$PRIVACY_GUARD_HOST_IP" \ + --name privacy-guard \ + --port 50051 +``` + +The command writes to: + +1. the path in `OPENSHELL_GATEWAY_CONFIG`, when set +2. otherwise `$XDG_CONFIG_HOME/openshell/gateway.toml` +3. otherwise `~/.config/openshell/gateway.toml` + +Use `--config PATH` to write a different gateway TOML. Existing unrelated +gateway settings are preserved. + +`configure-gateway` writes a five-second OpenShell middleware timeout. The +four-second processing timeout above leaves one second for queueing, +configuration validation, processor preparation, and transport overhead. If +you select a longer processing timeout, edit the generated registration to add +headroom. Rerunning `configure-gateway` restores the timeout to five seconds. + +The registration name must match the policy's `middleware` field: + +```yaml +network_middlewares: + privacy_guard_replace: + middleware: privacy-guard +``` + +OpenShell does not dynamically reload middleware registrations. Restart the +gateway after changing its configuration. + +## Run a local foreground gateway + +Stop the package-managed service before starting another gateway on the same +port. + +### macOS with Homebrew + +```bash +brew services stop openshell +export OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" +openshell-gateway --config "$HOME/.config/openshell/gateway.toml" +``` + +Restore it after testing: + +```bash +brew services start openshell +``` + +### Linux with a Debian or RPM package + +```bash +systemctl --user stop openshell-gateway +export OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/tls" +openshell-gateway --config "$HOME/.config/openshell/gateway.toml" +``` + +Restore it after testing: + +```bash +systemctl --user start openshell-gateway +``` + +For a custom gateway path, replace the `--config` argument accordingly. + +## Verify connectivity + +```bash +openshell gateway select openshell +openshell status +``` + +Do not create a sandbox until the selected gateway reports as connected. +Sandbox creation validates each referenced external middleware and its policy +configuration. + +## Inspect request results + +Use a finite log window: + +```bash +openshell logs SANDBOX_NAME -n 100 --source sandbox +``` + +For a provider request, inspect: + +| Field | Meaning | +| --- | --- | +| `transformed:true` | Privacy Guard returned a replacement body | +| `transformed:false` | Original body was allowed | +| findings | Aggregated entity, stage, confidence, and count | +| deny reason | `privacy_guard_blocked` or `privacy_guard_limit_exceeded` | + +Framework-controlled finding fields and `RegexEngine` do not add matched values +or surrounding request text. Custom engines must use stable, declared entity +identifiers that are not derived from request text. + +## Privacy Guard logging + +Default logs contain request IDs, evaluation duration, decision, aggregate +finding count, and stable error codes. They exclude request bodies, replacement +text, matches, offsets, regex patterns, headers, targets, and credentials. + +Enable content-safe diagnostic logging: + +```bash +uv run privacy-guard --debug serve +``` + +`--debug-log-content` logs complete input and processed text. Use it only in a +controlled development environment: + +```bash +uv run privacy-guard --debug --debug-log-content serve +``` + +When imported as a library, Privacy Guard uses the standard `privacy_guard` +logger and does not modify application logging unless requested. + +## Use a custom registry + +Pass a trusted `module:function` factory to every CLI command: + +```bash +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + engines + +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + serve \ + --listen 0.0.0.0:50051 +``` + +The function is imported and executed in the Privacy Guard process. It must +return a finalized `EngineRegistry`. See +[Add a custom engine](engines/custom.md). + +## Run the server from Python + +```python +from privacy_guard.engines.registry import create_builtin_registry +from privacy_guard.service import PrivacyGuardServer + +server = PrivacyGuardServer( + create_builtin_registry(), + timeout_seconds=5, +) +server.serve_sync("127.0.0.1:50051") +``` + +Async applications use: + +```python +await server.serve_async("127.0.0.1:50051") +``` + +Listen addresses use `host:port` or bracketed IPv6 `[address]:port` form. The +port must be between 1 and 65535. + +## Troubleshooting + +| Symptom | Check | +| --- | --- | +| Sandbox creation reports unavailable middleware | Confirm Privacy Guard is running, the registration name matches the policy, the host IP is reachable from the sandbox network, and the port is allowed | +| Gateway port is already in use | Stop the package-managed gateway before starting a foreground instance | +| Foreground gateway cannot find certificates | Set `OPENSHELL_LOCAL_TLS_DIR` to the package-specific path shown above | +| Policy config is rejected | Run `privacy-guard configuration-schema` with the same registry factory used by the server | +| Relative Regex catalog is not found | Start Privacy Guard from the directory against which the catalog path is defined | +| Request is denied with `privacy_guard_limit_exceeded` | Inspect Privacy Guard logs, reduce input/output/detections, simplify stages, or increase the processing timeout with OpenShell headroom | +| Registry factory cannot be loaded | Install the module or add its parent directory to `PYTHONPATH`; verify `module:function` spelling | +| Gateway accepts config but behavior does not change | Restart the gateway; middleware registrations are not reloaded dynamically | +| CLI and gateway reject policy fields differently | Confirm `openshell`, `openshell-gateway`, and Privacy Guard use the same protocol version | + +## Shutdown and cleanup + +Stop foreground Privacy Guard and gateway processes with `Ctrl-C`. Delete test +sandboxes explicitly: + +```bash +openshell sandbox delete SANDBOX_NAME +``` + +Restore the package-managed gateway and verify connectivity: + +```bash +openshell gateway select openshell +openshell status +``` + +## Next steps + +- [Configure policies](configuration.md) +- [RegexEngine](engines/regex.md) +- [Limits and failure behavior](reference/limits-and-failures.md) diff --git a/projects/privacy-guard/docs/reference/limits-and-failures.md b/projects/privacy-guard/docs/reference/limits-and-failures.md new file mode 100644 index 0000000..004f524 --- /dev/null +++ b/projects/privacy-guard/docs/reference/limits-and-failures.md @@ -0,0 +1,245 @@ +--- +title: Limits and failure behavior +description: Privacy Guard request, processing, Regex, concurrency, logging, and failure reference. +agent_markdown: true +--- + +# Limits and failure behavior + +Privacy Guard enforces bounds at the transport, processor, engine, and result +boundaries. Limit failures are atomic: they return no partial replacement text +or partial findings. + +Package-wide values are defined in +`projects/privacy-guard/src/privacy_guard/constants.py`. + +## Request and transport limits + +| Item | Limit | Enforcement | +| --- | ---: | --- | +| Request body | 4 MiB | Service | +| Request context | 4 KiB | Service | +| Policy config `Struct` | 64 KiB | Service | +| Target | 32 KiB | Service | +| Headers | 128 entries and 64 KiB total | Service | +| gRPC receive message | 5 MiB | gRPC server | +| Replacement body | 4 MiB | Service | + +The gRPC allowance includes 1 MiB for the protobuf envelope above the +advertised body limit. + +A non-empty request body must be valid UTF-8. Detect and block leave the original +bytes unchanged. Replace returns UTF-8 encoded final text. + +## Processing limits + +| Item | Limit | +| --- | ---: | +| Input or intermediate UTF-8 text | 4 MiB | +| Engine output UTF-8 text | 4 MiB | +| Stages per policy | 10 | +| Detections per stage | 256 | +| Detections per request | 4,096 | +| Default shared timeout | 1 second | +| Maximum shared timeout | 30 seconds | +| Active processing workers | 4 | +| Concurrent gRPC calls | 16 | + +One monotonic timeout covers every stage and final result validation. A stage +can consume the complete timeout budget. Detections remain subject to both the +256-per-stage and 4,096-per-request limits. + +Set the processing timeout with: + +```bash +privacy-guard serve --timeout-seconds 4 +``` + +OpenShell's middleware timeout must be longer than the Privacy Guard processing +timeout. Include headroom for worker queueing, configuration validation, and +processor preparation. `configure-gateway` writes a five-second OpenShell +timeout, so edit that registration before using a processing timeout of five +seconds or longer. + +## Finding and diagnostic limits + +| Item | Limit | +| --- | ---: | +| Diagnostic string | 1,024 UTF-8 bytes | +| Metadata entries per detection | 32 | +| Aggregated OpenShell finding groups | 32 | +| Encoded bytes per finding | 4 KiB | + +Diagnostic bounds apply to stage names, entity names, metadata keys and values, +and request IDs used in logs. Custom configuration fields such as model-profile +names are bounded only when their engine config declares an explicit constraint. +Invalid request IDs are replaced with a constant placeholder for logging and do +not change evaluation. + +If aggregated summaries cannot fit the OpenShell finding representation, +Privacy Guard returns `privacy_guard_limit_exceeded` with no partial findings. + +## Regex limits + +| Item | Limit | +| --- | ---: | +| Entities per catalog | 2,000 | +| Rules per catalog | 10,000 | +| Entity or supplied rule name | 128 ASCII bytes | +| Pattern | 16 KiB | +| Catalog file | 16 MiB | +| Compiled catalog cache | 128 entries | +| Compiled catalog cache weight | 32 MiB | + +Entity and rule names use: + +```text +[A-Za-z_][A-Za-z0-9_-]* +``` + +The 64 KiB policy transport limit still applies to inline catalogs. A catalog +can fit Regex limits while being too large to send inline. Use a relative +file-backed catalog in that case. + +## Regex execution controls + +`RegexEngine`: + +- rejects empty or invalid patterns +- rejects patterns that match empty input during validation +- rejects user-defined named groups and inline flags +- passes the shared remaining timeout into every backend search +- validates that each configured match consumed text +- caps detections before replacement +- calculates replacement UTF-8 size before allocation +- returns no partial detections or text after failure + +The engine uses the timeout-capable `regex` package rather than Python's +standard `re` package. + +## Request outcomes + +| Condition | Result | Reason or status | +| --- | --- | --- | +| No policy detection | Allow | No deny reason | +| Detection with `detect` | Allow original | Findings | +| Detection with `replace` | Allow replacement | Findings | +| Detection with `block` | Deny | `privacy_guard_blocked` | +| Shared timeout expires | Deny | `privacy_guard_limit_exceeded` | +| Processing or result limit exceeded | Deny | `privacy_guard_limit_exceeded` | +| Invalid request or config | RPC failure | `INVALID_ARGUMENT` | +| Engine or service failure | RPC failure | `INTERNAL` | + +Policy and limit denials are successful gRPC results. RPC failures use the +OpenShell middleware registration's `on_error` behavior. + +## Respond to a limit denial + +When a request returns `privacy_guard_limit_exceeded`: + +1. inspect Privacy Guard logs for the content-safe limit kind +2. reduce request or replacement size +3. reduce detections, stages, or rules +4. simplify expensive Regex patterns +5. increase `--timeout-seconds` when processing time is the bound +6. increase the OpenShell middleware timeout with additional headroom + +Do not retry the same request without changing the limiting condition. + +## Error information + +Production errors use stable `PrivacyGuardError` codes. Error responses and +logs do not expose raw Pydantic, regex backend, protobuf, engine, or collaborator +exception messages. + +Expected custom-engine failures must be translated into Privacy Guard's +content-safe engine exception hierarchy. + +## Logging + +Default operational logs include: + +- request ID +- evaluation duration +- allow or deny decision +- aggregate finding count +- stable error code +- stage and strategy at debug level + +Default logs exclude: + +- input and replacement text +- matches and surrounding text +- offsets +- Regex patterns and catalogs +- headers, targets, and credentials +- model endpoints +- arbitrary exception text + +`--debug` adds content-safe diagnostics. + +`--debug-log-content` logs complete input and processed text. Enable it only in +a controlled development environment. + +## State and retention + +Privacy Guard retains one active validated policy and its configured engines +until replacement or shutdown. It does not retain request text, detections, or +replacement mappings across requests. + +The compiled Regex cache is separate from the active processor: + +| Cache property | Value | +| --- | ---: | +| Entry cap | 128 catalogs | +| Weight budget | 32 MiB | +| Entry weight | Canonical catalog bytes plus 4 KiB per rule | + +An entry larger than the cache budget can be used for the current operation but +is not retained. Cache eviction does not invalidate an active processor that +already references compiled rules. + +## Measured latency + +The following proof-of-concept data uses the built-in `RegexEngine` with +synthetic repeated text. It contains 96 service-duration measurements from +18,291 to 1,140,979 prompt tokens and 3 to 429 detections per request. + +```text +latency_ms = 6.69 + 23.28 * (prompt_tokens / 100,000) +R² = 0.976 +``` + +![Scatter plot of Privacy Guard latency against prompt tokens on logarithmic axes. Point color maps the number of entities detected from 3 to 429. Ninety-six service measurements follow one fitted line. A dashed line marks the nominal one-million-token context threshold.](../assets/analysis/privacy-guard-latency-vs-prompt-size.svg) + +Across the 12 completed large-context turns with end-to-end timing, Privacy +Guard service processing averaged 0.56% of request-admission-to-last-output +time. + +Use these measurements for proof-of-concept timeout planning only. Prompt size +and detection count increased together, and the run used one host, sandbox, +policy, and Claude Code session. Measure production engines, policies, inputs, +and concurrency in the target deployment. + +The source data and deterministic renderer are in the +[`projects/privacy-guard/analysis`](https://github.com/NVIDIA/OpenShell-Research/tree/main/projects/privacy-guard/analysis) +directory. + +## Changing a limit + +For maintainers: + +1. identify every layer that enforces or advertises the limit +2. update exact-boundary tests +3. verify the failure result immediately above the boundary +4. update policy schema and examples when affected +5. run `make check` from `projects/privacy-guard` +6. benchmark changes that affect compilation or request processing +7. coordinate protocol changes with OpenShell + +## Related pages + +- [Run and operate Privacy Guard](../operations.md) +- [RegexEngine](../engines/regex.md) +- [Request lifecycle](../architecture/request-lifecycle.md) +- [Service boundary](../architecture/service-boundary.md) diff --git a/scripts/build-docs.sh b/scripts/build-docs.sh index 16bee84..70457c2 100755 --- a/scripts/build-docs.sh +++ b/scripts/build-docs.sh @@ -30,6 +30,7 @@ fi python -m pip install --upgrade pip python -m pip install -r requirements-docs.txt +python scripts/stage-privacy-guard-docs.py python scripts/render-dev-notes.py zensical build --clean --strict python scripts/publish-agent-markdown.py diff --git a/scripts/stage-privacy-guard-docs.py b/scripts/stage-privacy-guard-docs.py new file mode 100644 index 0000000..bf277ba --- /dev/null +++ b/scripts/stage-privacy-guard-docs.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Stage canonical Privacy Guard documentation in the site source tree.""" + +from __future__ import annotations + +import os +from pathlib import Path +import shutil + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_SOURCE = ROOT / "projects" / "privacy-guard" / "docs" +DEFAULT_DESTINATION = ROOT / "docs" / "documentation" / "privacy-guard" + + +def stage_privacy_guard_docs(source: Path, destination: Path) -> None: + """Replace the generated site mirror with one canonical project-docs tree.""" + + source = source.resolve() + destination_is_symlink = destination.is_symlink() + destination = destination.resolve(strict=False) + if not source.is_dir(): + raise ValueError(f"documentation source does not exist: {source}") + if destination_is_symlink: + raise ValueError(f"documentation destination must not be a symlink: {destination}") + if source.is_relative_to(destination) or destination.is_relative_to(source): + raise ValueError("documentation source and destination must not overlap") + + symlinks = sorted(path for path in source.rglob("*") if path.is_symlink()) + if symlinks: + paths = "\n".join(f" - {path}" for path in symlinks) + raise ValueError(f"project documentation must not contain symlinks:\n{paths}") + + destination.parent.mkdir(parents=True, exist_ok=True) + staging = destination.with_name(f".{destination.name}.staging") + if staging.exists(): + shutil.rmtree(staging) + shutil.copytree(source, staging) + + if destination.exists(): + shutil.rmtree(destination) + os.replace(staging, destination) + + +def main() -> int: + stage_privacy_guard_docs(DEFAULT_SOURCE, DEFAULT_DESTINATION) + print(f"Staged Privacy Guard documentation from {DEFAULT_SOURCE}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_stage_privacy_guard_docs.py b/tests/test_stage_privacy_guard_docs.py new file mode 100644 index 0000000..0230e9c --- /dev/null +++ b/tests/test_stage_privacy_guard_docs.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "stage-privacy-guard-docs.py" + +SPEC = importlib.util.spec_from_file_location("stage_privacy_guard_docs", SCRIPT) +if SPEC is None or SPEC.loader is None: + raise RuntimeError(f"could not load {SCRIPT}") +STAGER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(STAGER) + + +class StagePrivacyGuardDocsTests(unittest.TestCase): + def test_stage_replaces_destination_with_source_tree(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source = root / "project-docs" + destination = root / "site-docs" + (source / "architecture").mkdir(parents=True) + (source / "index.md").write_text("# Overview\n", encoding="utf-8") + (source / "architecture/index.md").write_text( + "# Architecture\n", + encoding="utf-8", + ) + destination.mkdir() + (destination / "stale.md").write_text("# Stale\n", encoding="utf-8") + + STAGER.stage_privacy_guard_docs(source, destination) + + self.assertEqual( + (destination / "index.md").read_text(encoding="utf-8"), + "# Overview\n", + ) + self.assertTrue((destination / "architecture/index.md").is_file()) + self.assertFalse((destination / "stale.md").exists()) + + def test_stage_rejects_symlinks_in_source(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source = root / "project-docs" + source.mkdir() + target = root / "outside.md" + target.write_text("# Outside\n", encoding="utf-8") + (source / "linked.md").symlink_to(target) + + with self.assertRaisesRegex(ValueError, "must not contain symlinks"): + STAGER.stage_privacy_guard_docs(source, root / "site-docs") + + def test_stage_rejects_destination_inside_source(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + source = Path(temporary_directory) / "project-docs" + source.mkdir() + + with self.assertRaisesRegex(ValueError, "must not overlap"): + STAGER.stage_privacy_guard_docs(source, source / "published") + + def test_stage_rejects_source_inside_destination(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + destination = Path(temporary_directory) / "site-docs" + source = destination / "project-docs" + source.mkdir(parents=True) + + with self.assertRaisesRegex(ValueError, "must not overlap"): + STAGER.stage_privacy_guard_docs(source, destination) + + +if __name__ == "__main__": + unittest.main() diff --git a/zensical.toml b/zensical.toml index b6a4ac8..2cdaf9b 100644 --- a/zensical.toml +++ b/zensical.toml @@ -9,7 +9,10 @@ copyright = """ """ docs_dir = "docs" site_dir = "site" -extra_css = ["stylesheets/dev-notes.css"] +extra_css = [ + "stylesheets/dev-notes.css", + "stylesheets/documentation.css" +] extra_javascript = ["javascripts/navigation-drawer.js"] nav = [ {"Dev Notes" = [ @@ -23,13 +26,23 @@ nav = [ {"Documentation" = [ "documentation/index.md", {"Privacy Guard" = [ + "documentation/privacy-guard/index.md", + {"Guides" = [ + {"Configure policies" = "documentation/privacy-guard/configuration.md"}, + {"Run and operate Privacy Guard" = "documentation/privacy-guard/operations.md"} + ]}, + {"Engines" = [ + "documentation/privacy-guard/engines/index.md", + {"RegexEngine" = "documentation/privacy-guard/engines/regex.md"}, + {"Add a custom engine" = "documentation/privacy-guard/engines/custom.md"} + ]}, {"Architecture" = [ "documentation/privacy-guard/architecture/index.md", {"Request lifecycle" = "documentation/privacy-guard/architecture/request-lifecycle.md"}, - {"Entity-processing engines" = "documentation/privacy-guard/architecture/entity-processing-engines.md"}, - {"Configuration and text boundary" = "documentation/privacy-guard/architecture/configuration.md"}, - {"Service boundary" = "documentation/privacy-guard/architecture/service-boundary.md"}, - {"Safety and limits" = "documentation/privacy-guard/architecture/safety-and-limits.md"} + {"Service boundary" = "documentation/privacy-guard/architecture/service-boundary.md"} + ]}, + {"Reference" = [ + {"Limits and failure behavior" = "documentation/privacy-guard/reference/limits-and-failures.md"} ]} ]} ]}