From ed77b3992383c84a7a9481b6cb0085c6a8534039 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Tue, 28 Jul 2026 16:49:47 -0400 Subject: [PATCH 1/8] Reframe Privacy Guard documentation --- .../privacy-guard-latency-vs-prompt-size.svg | 162 ++++++++ .../diagrams/component-architecture.svg | 80 ++++ .../diagrams/processing-pipeline.svg | 94 +++++ .../diagrams/request-lifecycle.svg | 108 +++++ .../privacy-guard/diagrams/request-path.svg | 74 ++++ docs/documentation/index.md | 5 +- .../architecture/configuration.md | 149 ------- .../architecture/entity-processing-engines.md | 334 --------------- .../privacy-guard/architecture/index.md | 287 ++++++------- .../architecture/request-lifecycle.md | 245 ++++++----- .../architecture/safety-and-limits.md | 286 ------------- .../architecture/service-boundary.md | 384 +++++------------- .../privacy-guard/configuration.md | 281 +++++++++++++ .../privacy-guard/engines/custom.md | 301 ++++++++++++++ .../privacy-guard/engines/regex.md | 215 ++++++++++ docs/documentation/privacy-guard/index.md | 217 ++++++++++ .../documentation/privacy-guard/operations.md | 261 ++++++++++++ .../reference/limits-and-failures.md | 240 +++++++++++ docs/stylesheets/documentation.css | 21 + projects/privacy-guard/README.md | 342 +++++----------- projects/privacy-guard/docs/index.md | 8 +- zensical.toml | 22 +- 22 files changed, 2535 insertions(+), 1581 deletions(-) create mode 100644 docs/assets/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg create mode 100644 docs/assets/privacy-guard/diagrams/component-architecture.svg create mode 100644 docs/assets/privacy-guard/diagrams/processing-pipeline.svg create mode 100644 docs/assets/privacy-guard/diagrams/request-lifecycle.svg create mode 100644 docs/assets/privacy-guard/diagrams/request-path.svg delete mode 100644 docs/documentation/privacy-guard/architecture/configuration.md delete mode 100644 docs/documentation/privacy-guard/architecture/entity-processing-engines.md delete mode 100644 docs/documentation/privacy-guard/architecture/safety-and-limits.md create mode 100644 docs/documentation/privacy-guard/configuration.md create mode 100644 docs/documentation/privacy-guard/engines/custom.md create mode 100644 docs/documentation/privacy-guard/engines/regex.md create mode 100644 docs/documentation/privacy-guard/index.md create mode 100644 docs/documentation/privacy-guard/operations.md create mode 100644 docs/documentation/privacy-guard/reference/limits-and-failures.md create mode 100644 docs/stylesheets/documentation.css diff --git a/docs/assets/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg b/docs/assets/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg new file mode 100644 index 0000000..e6b1866 --- /dev/null +++ b/docs/assets/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg @@ -0,0 +1,162 @@ + +Privacy Guard latency versus prompt size +Scatter plot of 96 Privacy Guard service latency measurements from 18 thousand to 1.141 million prompt tokens, with one linear fit and a one-million-token threshold. Privacy Guard averaged 0.56 percent of end-to-end time across 12 completed turns. + + + + + + + + + +Privacy Guard latency (ms) · log scale + +10k + +30k + +100k + +300k + +1M + +5 + +10 + +20 + +50 + +100 + +250 + +1M threshold + +18,291 tokens; 5.5 ms; 3 entities detected +18,722 tokens; 6.0 ms; 3 entities detected +18,923 tokens; 6.0 ms; 4 entities detected +19,120 tokens; 10.5 ms; 4 entities detected +19,378 tokens; 12.5 ms; 6 entities detected +19,798 tokens; 21.7 ms; 6 entities detected +20,335 tokens; 7.8 ms; 6 entities detected +21,404 tokens; 6.9 ms; 11 entities detected +22,377 tokens; 8.8 ms; 11 entities detected +22,934 tokens; 9.7 ms; 11 entities detected +23,528 tokens; 7.8 ms; 15 entities detected +23,866 tokens; 10.9 ms; 16 entities detected +26,472 tokens; 10.9 ms; 22 entities detected +26,853 tokens; 11.7 ms; 22 entities detected +28,021 tokens; 12.0 ms; 22 entities detected +28,867 tokens; 11.0 ms; 26 entities detected +31,351 tokens; 12.8 ms; 26 entities detected +33,610 tokens; 14.6 ms; 26 entities detected +34,193 tokens; 14.5 ms; 26 entities detected +20,747 tokens; 10.8 ms; 19 entities detected +22,310 tokens; 18.2 ms; 20 entities detected +23,103 tokens; 16.4 ms; 20 entities detected +27,015 tokens; 28.3 ms; 22 entities detected +31,441 tokens; 13.0 ms; 22 entities detected +38,362 tokens; 12.4 ms; 23 entities detected +39,098 tokens; 18.3 ms; 23 entities detected +39,471 tokens; 17.0 ms; 23 entities detected +40,025 tokens; 10.9 ms; 23 entities detected +47,124 tokens; 14.8 ms; 23 entities detected +47,960 tokens; 11.6 ms; 23 entities detected +48,251 tokens; 17.6 ms; 23 entities detected +51,102 tokens; 13.0 ms; 23 entities detected +51,552 tokens; 11.3 ms; 23 entities detected +52,216 tokens; 14.3 ms; 23 entities detected +52,634 tokens; 11.1 ms; 23 entities detected +53,244 tokens; 20.7 ms; 23 entities detected +62,423 tokens; 13.7 ms; 23 entities detected +63,068 tokens; 17.8 ms; 23 entities detected +63,486 tokens; 17.5 ms; 23 entities detected +64,295 tokens; 26.0 ms; 23 entities detected +76,372 tokens; 18.9 ms; 23 entities detected +77,031 tokens; 18.1 ms; 23 entities detected +82,339 tokens; 26.1 ms; 24 entities detected +83,184 tokens; 36.0 ms; 24 entities detected +88,677 tokens; 20.9 ms; 24 entities detected +89,748 tokens; 18.7 ms; 24 entities detected +89,970 tokens; 20.0 ms; 24 entities detected +90,530 tokens; 21.0 ms; 24 entities detected +91,420 tokens; 21.7 ms; 24 entities detected +92,901 tokens; 22.7 ms; 24 entities detected +93,678 tokens; 36.0 ms; 24 entities detected +95,694 tokens; 25.2 ms; 24 entities detected +96,704 tokens; 30.2 ms; 24 entities detected +96,868 tokens; 27.0 ms; 24 entities detected +97,886 tokens; 30.3 ms; 24 entities detected +98,356 tokens; 24.5 ms; 24 entities detected +99,265 tokens; 35.3 ms; 24 entities detected +101,025 tokens; 34.2 ms; 24 entities detected +101,193 tokens; 23.3 ms; 24 entities detected +102,071 tokens; 26.9 ms; 24 entities detected +102,481 tokens; 23.7 ms; 24 entities detected +104,787 tokens; 29.4 ms; 24 entities detected +105,313 tokens; 31.9 ms; 24 entities detected +105,801 tokens; 35.6 ms; 24 entities detected +105,958 tokens; 39.6 ms; 24 entities detected +106,160 tokens; 33.4 ms; 25 entities detected +106,428 tokens; 24.1 ms; 25 entities detected +106,807 tokens; 47.7 ms; 25 entities detected +106,981 tokens; 43.3 ms; 25 entities detected +107,412 tokens; 27.1 ms; 25 entities detected +107,647 tokens; 32.0 ms; 25 entities detected +108,052 tokens; 38.2 ms; 25 entities detected +108,800 tokens; 36.9 ms; 25 entities detected +114,425 tokens; 32.0 ms; 25 entities detected +114,584 tokens; 32.3 ms; 25 entities detected +115,395 tokens; 24.7 ms; 25 entities detected +115,852 tokens; 39.1 ms; 25 entities detected +117,730 tokens; 42.9 ms; 25 entities detected +120,732 tokens; 36.0 ms; 25 entities detected +122,710 tokens; 47.4 ms; 27 entities detected +127,634 tokens; 38.6 ms; 59 entities detected +136,816 tokens; 49.1 ms; 59 entities detected +143,349 tokens; 41.2 ms; 59 entities detected +323,611 tokens; 93.4 ms; 259 entities detected +324,158 tokens; 96.5 ms; 259 entities detected +325,804 tokens; 95.2 ms; 259 entities detected +329,592 tokens; 91.0 ms; 259 entities detected +656,648 tokens; 160.3 ms; 323 entities detected +665,140 tokens; 164.3 ms; 323 entities detected +963,788 tokens; 220.3 ms; 389 entities detected +965,007 tokens; 216.3 ms; 389 entities detected +1,140,979 tokens; 268.7 ms; 429 entities detected +228,409 tokens; 54.2 ms; 43 entities detected +428,405 tokens; 103.4 ms; 83 entities detected +628,418 tokens; 196.2 ms; 123 entities detected +801,456 tokens; 176.0 ms; 163 entities detected +Privacy Guard averaged 0.56% +of end-to-end turn time +across 12 completed turns +Entities detected + +Prompt tokens · log scale +Single fit across 96 measurements: 6.69 + 23.28 ms per 100k tokens · R² 0.976 + +3 + +100 + +200 + +300 + +429 + diff --git a/docs/assets/privacy-guard/diagrams/component-architecture.svg b/docs/assets/privacy-guard/diagrams/component-architecture.svg new file mode 100644 index 0000000..e101301 --- /dev/null +++ b/docs/assets/privacy-guard/diagrams/component-architecture.svg @@ -0,0 +1,80 @@ + + Privacy Guard component architecture + Four architecture layers show transport and startup adapters, typed configuration and registry, request processing, and entity-processing engines. Downward arrows show configuration flowing into the request processor and the processor invoking concrete engines through their shared wrapper. + + + + + + + + + TRANSPORT AND STARTUP + + CLI and gateway config + cli.py · gateway_config.py + + OpenShell gRPC adapter + service/ · protobuf validation · result serialization + + + TYPED POLICY AND RUNTIME INVENTORY + + Policy configuration + config.py · ordered stages · final action + + Engine registry + config union · resources · engine construction + + + REQUEST PROCESSING + + RequestProcessor + stage order · one timeout · aggregation · allow or deny + + + ENTITY PROCESSING + + Engine wrapper · contract and bounds + + RegexEngine + built-in implementation + + Custom engines + trusted integrations + + + validate config + + validated policy + + construct engines + run stages + + + diff --git a/docs/assets/privacy-guard/diagrams/processing-pipeline.svg b/docs/assets/privacy-guard/diagrams/processing-pipeline.svg new file mode 100644 index 0000000..05d7818 --- /dev/null +++ b/docs/assets/privacy-guard/diagrams/processing-pipeline.svg @@ -0,0 +1,94 @@ + + Privacy Guard processing pipeline + Validated policy and input text produce one detect or replace strategy and shared timeout. Ordered engine stages run and validate their output. Privacy Guard aggregates detections and applies the final detect, block, or replace action. + + + + + + + + + 1 + Input + Validated policy + UTF-8 text + + + + + 2 + Invocation controls + DETECT or REPLACE + one Timeout + + + + + 3 + Stage 1 + engine.run(current text) + + + + + 4 + Validate result + Text · spans · limits · timeout + + + text + + + 5 + Next stage + Repeat in configured order + + + + + 6 + Aggregate + Stage + entity + confidence + + + + + 7 + Apply the policy action + + + detect + allow original + + block + deny on detection + + replace + allow final text + + + repeat until final stage + diff --git a/docs/assets/privacy-guard/diagrams/request-lifecycle.svg b/docs/assets/privacy-guard/diagrams/request-lifecycle.svg new file mode 100644 index 0000000..340cb18 --- /dev/null +++ b/docs/assets/privacy-guard/diagrams/request-lifecycle.svg @@ -0,0 +1,108 @@ + + Privacy Guard request lifecycle + The request lifecycle has four phases: validate the OpenShell transport, validate and activate configuration, decode and process text, and serialize the result. Empty bodies bypass engine processing. Invalid input fails the RPC, while policy and limit decisions return successful allow or deny results. + + + + + + + + + 1 · TRANSPORT + + Receive + HttpRequestEvaluation + pre-credentials phase + + + Validate bounds + context · config + target · headers · body + phase + + + + + 2 · CONFIGURATION + + Validate policy + typed union · engines + resources · action + + + Resolve processor + reuse equal config + or prepare + activate + + + + + 3 · PROCESSING + + Decode text + strict UTF-8 + empty body bypass + + + Run pipeline + ordered stages + one shared timeout + aggregate findings + + + + + 4 · RESULT + + Apply + detect + block + replace + + + Serialize + allow or deny + body + findings + + + RPC FAILURE + Invalid input or internal failure + INVALID_ARGUMENT or INTERNAL + OpenShell applies middleware on_error + + + + + SUCCESSFUL MIDDLEWARE RESULT + Allow, policy deny, or limit deny + privacy_guard_blocked + privacy_guard_limit_exceeded + + diff --git a/docs/assets/privacy-guard/diagrams/request-path.svg b/docs/assets/privacy-guard/diagrams/request-path.svg new file mode 100644 index 0000000..d19a5a8 --- /dev/null +++ b/docs/assets/privacy-guard/diagrams/request-path.svg @@ -0,0 +1,74 @@ + + Privacy Guard request path + A provider-bound request travels from a sandbox application through the OpenShell supervisor to Privacy Guard. Privacy Guard returns allow, replacement, or deny. OpenShell attaches credentials only after an allow result and then sends the request to the provider. + + + + + + + + + + + Sandbox application + Creates a provider-bound HTTP request + + + request + + + + + OpenShell supervisor + Routes the request before credentials are attached + + + body + policy over gRPC + + + + PRE-CREDENTIALS + + + + Privacy Guard + Detects entities and applies the configured action + + + allow original, replacement, or deny + + + + + OpenShell supervisor + Stops denied requests; attaches credentials after allow + + + authorized request + + + + + Provider + diff --git a/docs/documentation/index.md b/docs/documentation/index.md index 5579bdd..ea10c6b 100644 --- a/docs/documentation/index.md +++ b/docs/documentation/index.md @@ -9,6 +9,5 @@ agent_markdown: true Technical documentation and references for installing, using, and extending OpenShell Research projects. -- [Privacy Guard architecture](privacy-guard/architecture/index.md): component - boundaries, request flow, extension interfaces, concurrency, failures, and - limits +- [Privacy Guard](privacy-guard/index.md): quickstart, policy configuration, + operations, engines, architecture, and limits. diff --git a/docs/documentation/privacy-guard/architecture/configuration.md b/docs/documentation/privacy-guard/architecture/configuration.md deleted file mode 100644 index 2aadd71..0000000 --- a/docs/documentation/privacy-guard/architecture/configuration.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -title: Configuration and text boundary -description: Runtime configuration ownership and the single-text processing contract. -agent_markdown: true ---- - -# Configuration and text boundary - -Privacy Guard processes the complete request body as one text string. Its -runtime configuration is supplied by OpenShell for each evaluation and -normalized into strict structured models before processing. - -## Bytes and text - -The service owns the transport boundary: - -- it validates the advertised 4 MiB request-body limit -- it allows an empty body without invoking an engine -- it decodes each non-empty body as strict UTF-8 -- it passes exactly one `str` to `RequestProcessor` -- detect and block return no body mutation -- replace UTF-8 encodes the final processed text - -Because detect and block return no mutation, OpenShell retains the exact -original bytes. Replace returns the final text even when it happens to equal -the input. - -Headers and media types do not change this behavior. Privacy Guard does not -parse JSON, select nested values, create document regions, or reconstruct -structured payloads. - -## Policy configuration - -The OpenShell policy owns: - -- ordered entity-processing stages -- each stage's exact engine configuration -- entity definitions and detection settings -- engine-specific replacement recipes -- the final detect, block, or replace action - -For example: - -```yaml -entity_processing: - stages: - - name: identifiers - config: - engine: regex - pattern_catalog: - entities: - - name: customer-id - rules: - - name: prefixed-eight-digit-id - pattern: '\bCUST-[0-9]{8}\b' - confidence: high - replacement: - strategy: template - template: "[{entity}]" -on_detection: - action: replace -``` - -`config.engine` is the Pydantic discriminator. `EngineRegistry.finalize()` -builds the complete policy model from the exact config type registered for each -engine. Engine-specific fields therefore validate and serialize without a -generic mapping layer. - -Deployment startup owns operational resources rather than privacy behavior: -installed engine implementations, approved model profiles, clients, endpoints, -credentials, and data-egress constraints. - -## Regex catalogs - -`RegexEngineConfig.pattern_catalog` accepts either a structured -`RegexPatternCatalog` or a relative `.yaml` or `.yml` path: - -```yaml -pattern_catalog: patterns.yaml -``` - -Paths resolve beneath Privacy Guard's working directory. Absolute paths, -traversal, and symlinks are rejected. Catalog files must be bounded UTF-8 YAML -without aliases, duplicate keys, or unsafe tags. - -File-backed and inline inputs normalize to the same `RegexPatternCatalog`. -The complete validated configuration contains the structured catalog rather -than its source path. File-backed catalogs are read and validated on each policy -validation. The normalized immutable catalog keys a bounded compiled-rule -cache, so equivalent rules do not need to be recompiled. A content change -produces a different validated configuration and causes the next evaluation to -prepare an active-policy replacement. - -Privacy Guard maintains the catalog schema and safety limits but does not ship -an authoritative pattern set. Repository catalogs are examples to copy and -adapt, not presets or runtime defaults. - -`ValidateConfig` proves that each Regex pattern has valid syntax and rejects -patterns that immediately match empty input. It cannot prove that every -context-dependent branch consumes text. If a boundary, lookaround, or -alternation produces a zero-width match only when triggering request text is -observed, evaluation rejects the policy as `config_invalid` rather than treating -the result as an internal engine failure. Lookarounds and boundaries remain -valid when the complete match consumes text. - -## Current transport constraint - -The copied OpenShell protocol carries policy configuration in a -per-evaluation `google.protobuf.Struct` limited to 64 KiB. An inline catalog is -bounded by that transport. A file-backed configuration carries only its bounded -relative path through the protocol and loads the deployment-mounted catalog in -the middleware process. - -The service rejects an encoded configuration above that limit before protobuf -conversion or policy validation. A policy may contain at most ten ordered -entity-processing stages. The service validates the complete bounded -configuration and compares it with the single active policy. Equal validated -configurations reuse the active processor; a different valid configuration pays -full engine and processor preparation before atomic replacement. Reuse avoids -repeated engine initialization but does not increase either limit. - -`Struct` carries every number as a double. At this transport boundary only, -Privacy Guard converts finite integral values in the safe integer range -`-(2^53 - 1)` through `2^53 - 1` to Python integers before strict policy -validation. Non-integral and out-of-range values remain floats, so integer -fields reject them. Direct Python registry validation remains strict and does -not perform this transport normalization. Custom-engine integer settings must -therefore remain within the safe range when supplied through OpenShell policy. - -A future self-contained transport for larger expanded catalogs requires an -upstream OpenShell contract for preparing configuration and referring to it -during evaluation. Privacy Guard must not create a private protocol fork. - -## Policy identity - -Privacy Guard compares the complete immutable validated configuration, -including every concrete engine field, nested replacement variant, and expanded -Regex catalog. Equal configurations reuse the active processor. A different -configuration is an update candidate and replaces the active policy only after -complete preparation succeeds. - -The current protocol carries neither an explicit update operation nor a policy -version. Privacy Guard therefore cannot distinguish an intentional user update -from any other changed per-evaluation configuration. OpenShell must send one -consistent policy stream to a process; interleaved old and new configurations -can repeatedly replace one another. A future protocol can make preparation and -activation explicit and let evaluations refer to a versioned policy. - -[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md b/docs/documentation/privacy-guard/architecture/entity-processing-engines.md deleted file mode 100644 index d92d8c9..0000000 --- a/docs/documentation/privacy-guard/architecture/entity-processing-engines.md +++ /dev/null @@ -1,334 +0,0 @@ ---- -title: Entity-processing engines -description: Engine configuration, lifecycle, output contract, and extension rules. -agent_markdown: true ---- - -# Entity-processing engines - -An `EntityProcessingEngine` detects sensitive entities in one text string and -may replace them when requested. It has no access to the user-facing policy -action, request metadata, gRPC, or protobuf messages. - -Each configured stage owns one engine instance. The same instance may serve -concurrent requests, while separate stages may use the same implementation -with different configurations. - -## Engine interface - -An engine: - -1. declares a concrete configuration type and, when needed, a resource type - through generics -2. declares its non-empty set of `supported_strategies` -3. receives validated configuration and operator-owned resources through the - base constructor -4. optionally derives immutable reusable state in `_initialize()` -5. implements `_run(text, strategy, timeout)` -6. returns one `TextProcessingResult` - -The public `run()` method validates input, strategy support, the timeout, and -the complete output contract. Custom engines implement `_run()` and do not -override `run()` or define `__init__`. `_initialize()` is optional, and -`@override` is not required. The base constructor and public wrapper are final -lifecycle methods. Registration rejects direct or inherited overrides and -directs custom engines to `_initialize()` and `_run()` instead. - -## Configuration - -Every concrete config subclasses `EngineConfig` and declares exactly one -literal `engine` discriminator: - -```python -class AcmeEngineConfig(EngineConfig): - engine: Literal["acme-pii"] = "acme-pii" - replacement: AcmeReplacement | None = None -``` - -The object under `EntityProcessingStage.config` is this exact concrete model. -It is validated and serialized as a member of the registry-built Pydantic -discriminated union, then passed unchanged to the engine constructor. - -Custom integer fields remain strict for direct Python callers. OpenShell policy -uses protobuf `Struct`, whose numeric representation is a double, so the service -normalizes finite integral values only within the safe range `-(2^53 - 1)` -through `2^53 - 1`. Values outside that range and non-integral values remain -floats and fail strict integer fields. Nested models and lists follow the same -transport rule. - -`EngineConfig` is a nominal, strict base model. It does not prescribe a -`replacement` field or any other algorithm-specific setting. An engine that -needs a replacement recipe declares that field on its concrete config. Engines -with multiple replacement algorithms may define a nested discriminated union, -using fields such as `replacement.strategy`. An engine whose underlying tool -has intrinsic replacement behavior may support `REPLACE` without declaring a -replacement field at all. - -Configuration contains privacy behavior. An optional `EngineResources` object -contains operator-owned runtime dependencies such as initialized model clients, -SDK adapters, endpoints, credential providers, or approved model profiles. -Resources are registered by the operator and never serialized into policy. - -`EngineResources` is a nominal contract. A concrete resource bundle subclasses -it, is typed as the engine's second generic argument, and must: - -- contain operational dependencies rather than policy behavior -- retain no request text or mutable per-request state -- expose only dependencies that are safe for concurrent engine calls -- avoid relying on construction or mutation during request processing - -For example: - -```python -from dataclasses import dataclass - -from privacy_guard.engines import EngineResources - - -@dataclass(frozen=True) -class AcmeResources(EngineResources): - client: AcmeClient - - -class AcmeEngine(EntityProcessingEngine[AcmeEngineConfig, AcmeResources]): - ... -``` - -Resources are optional. A resource-free engine omits the second generic -argument and receives `None` from the base class: - -```python -class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig]): - ... -``` - -## Invocation strategy - -```python -class EntityProcessingStrategy(StrEnum): - DETECT = "detect" - REPLACE = "replace" -``` - -An engine declares exactly the invocation strategies it supports. A -detection-only engine declares: - -```python -supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) -``` - -A replacement-only engine declares: - -```python -supported_strategies = frozenset({EntityProcessingStrategy.REPLACE}) -``` - -An engine that exposes both operations includes both enum values. Supporting -`REPLACE` does not imply that the engine exposes `DETECT`, even when replacement -requires internal detection. Blocking does not appear here: the processor runs -engines with `DETECT` and applies the block disposition afterward. - -The registry calls `validate_run_config()` with the strategy derived from the -policy action. The base implementation verifies strategy support. An engine -may add technique-specific requirements through `_validate_run_config()`, such -as requiring a template only when `REPLACE` is requested. The engine receives -the strategy, never the user-facing `PolicyAction`. - -## Result contract - -`TextProcessingResult` contains: - -| Field | Meaning | -| --- | --- | -| `text` | Authoritative text returned by this stage | -| `detections` | Bounded tuple of all `EntityDetection` occurrences | - -Each `EntityDetection` contains: - -| Field | Meaning | -| --- | --- | -| `entity` | Bounded entity label | -| `start` | Inclusive Unicode code-point offset in the stage input | -| `end` | Exclusive Unicode code-point offset in the stage input | -| `confidence` | Optional `low`, `medium`, or `high` certainty | -| `metadata` | Optional bounded engine-specific attribution retained inside the processing boundary | - -A detection span is non-empty and must fall within the stage input. The public -wrapper independently enforces stage detection and output-size limits for every -returned result. Engines that produce detections lazily should use -`TextProcessingResult.from_detections()`; this optional convenience stops -consuming the iterable as soon as the stage limit is exceeded, but it is not -the enforcement boundary. - -For `DETECT`, output text must exactly equal input text. For `REPLACE`, a -successful return is the engine's authoritative completed result. Text may not -change without at least one detection. Engines must raise on native partial -failure instead of returning partial output. - -Privacy Guard preserves categorical confidence as supplied by the engine and -does not compare confidence across engines. - -## Custom engine example - -```python -import re -from typing import Literal - -from pydantic import Field - -from privacy_guard.engines import ( - EngineConfig, - EntityDetection, - EntityProcessingEngine, - EntityProcessingStrategy, - TextProcessingResult, -) -from privacy_guard.timeout import Timeout - - -class KeywordEngineConfig(EngineConfig): - engine: Literal["keyword"] = "keyword" - keyword: str = Field(min_length=1) - - -class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig]): - supported_strategies = frozenset({EntityProcessingStrategy.DETECT}) - - def _run( - self, - text: str, - *, - strategy: EntityProcessingStrategy, - timeout: Timeout, - ) -> TextProcessingResult: - matches = re.finditer(re.escape(self.config.keyword), text) - return TextProcessingResult.from_detections( - text=text, - detections=( - EntityDetection( - entity="keyword", - start=match.start(), - end=match.end(), - ) - for match in matches - ), - ) -``` - -Register the implementation and, when required, its resources before finalizing -the registry: - -```python -registry = EngineRegistry(include_builtin_engines=True) -registry.register(KeywordEngine) -registry = registry.finalize() -``` - -The opt-in constructor argument registers the engines shipped by Privacy Guard -before application-specific engines. Omit it when the registry should contain -only explicitly registered implementations. - -Finalization freezes registration and constructs the exact policy config type, -JSON Schema, and engine discovery metadata. - -The complete example at -`projects/privacy-guard/examples/custom-engine/README.md` keeps the engine, -configuration, and registry factory in one Python file and runs that registry -through discovery, schema generation, serving, and an OpenShell policy. - -## Timeout and concurrency - -One `Timeout` is created for the processor run and passed through every stage. -The public engine wrapper checks it immediately before and after `_run()`, so -ordinary in-memory implementations do not repeat those checks. An engine must -not create a fresh per-stage duration. - -When a delegated API accepts a timeout, pass the remaining duration: - -```python -remaining = timeout.remaining_seconds() -``` - -When that API raises Python's `TimeoutError` on expiration, use the shared -context manager to translate it into Privacy Guard's domain error: - -```python -with timeout.enforce(): - result = client.process(text, timeout=timeout.remaining_seconds()) -``` - -Unique long-running loops may also call `timeout.raise_if_expired()`. -Operations that cannot be interrupted must be documented and bounded -independently. - -Engine configuration, derived state, and injected resources are shared across -concurrent requests. Keep request text, detections, and counters local to -`_run()`. Resources and engines must be concurrency-safe. - -## RegexEngine - -`RegexEngine` owns both regular-expression detection and deterministic template -replacement. Its `RegexPatternCatalog` contains structured entities and ordered -rules. Each rule combines one regex pattern with its optional diagnostic name, -confidence, and flags. Privacy Guard maintains the schema and limits but no -authoritative pattern set. - -Important properties: - -- rules compile once during configuration validation and initialization -- a non-capturing wrapper plus a private trailing named marker preserves - numeric backreferences and proves the configured match completed -- user-defined named groups and inline flags are rejected -- `ignore_case`, `multiline`, `dot_all`, and `ascii` are explicit fields -- detection retains overlapping matches within and across rules -- each backend search receives the shared remaining timeout -- rule names are optional; unnamed rules receive deterministic - diagnostic identities without changing serialized configuration -- catalogs may be supplied inline or through a bounded relative YAML path and - normalize to the same structured configuration -- replacement resolves overlaps by categorical confidence, span length, - offsets, entity, and rule identity -- templates allow literal text and `{entity}` only -- replacement size is projected before output allocation - -The third-party `regex` backend is used because it can interrupt an individual -search when the timeout expires. - -## Tool-specific custom engines - -Tool integrations belong in custom engines until they have a complete, -production-backed implementation. Privacy Guard does not ship placeholder -engine types or runtime protocols that merely resemble a third-party tool. - -The first NeMo Anonymizer integration will be a custom engine. Its configuration -and replacement types should preserve Anonymizer's native concepts, while the -engine itself owns all translation to and from the actual Anonymizer SDK. - -## State - -Cross-request entity memory is intentionally out of scope. The engine API has -no state argument, session identifier, persistent replacement map, or -placeholder storage interface. - -Future memory support requires separate decisions about tenant isolation, -retention, deletion, consistency, and policy ownership. - -## Testing an engine - -Test engines without gRPC: - -- valid and invalid exact configuration and resource types -- detection-only immutability -- replacement behavior and native partial failure -- Unicode offsets and invalid spans -- categorical confidence -- output and detection limits -- timeout propagation and expiration -- deterministic ordering where relevant -- concurrent calls over immutable initialized state -- content-safe errors that omit input and engine secrets - -Processor tests should cover only ordered multi-stage behavior, shared timeout, -policy disposition, aggregate limits, and stage-qualified findings. - -[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/index.md b/docs/documentation/privacy-guard/architecture/index.md index 2e8d932..d03502e 100644 --- a/docs/documentation/privacy-guard/architecture/index.md +++ b/docs/documentation/privacy-guard/architecture/index.md @@ -1,171 +1,136 @@ --- -title: Privacy Guard architecture -description: System structure, component boundaries, and request flow for Privacy Guard. +title: System architecture +description: Privacy Guard components, boundaries, state, and request-processing structure. agent_markdown: true --- -# Privacy Guard architecture - -Privacy Guard is OpenShell middleware that detects and optionally replaces -sensitive entities in provider-bound HTTP request text before OpenShell adds -provider credentials. - -This architecture is a clean break from the earlier implementation. A processor -run receives one UTF-8 text value and invokes an ordered pipeline of -entity-processing engines. Structured-body parsing and compatibility with the -previous extension and policy APIs are intentionally out of scope. - -Privacy Guard can allow the original body, allow a replacement body, or deny -the request. It never sends the provider request itself. - -## Where Privacy Guard runs - -```text -Sandbox process - | - | provider-bound HTTP request - v -OpenShell supervisor - | - | gRPC: SupervisorMiddleware - v -Privacy Guard - | - | allow original body, allow replacement body, or deny - v -OpenShell supervisor - | - | credentials added only after middleware allows - v -Provider -``` - -Privacy Guard implements the OpenShell-owned `SupervisorMiddleware` gRPC -service and advertises one HTTP-request binding in the pre-credentials phase. -The checked-in protocol and generated bindings are canonical copies from -OpenShell and must not be edited locally. - -## Component boundaries - -Source paths on these pages are relative to +# System architecture + +Privacy Guard is a pre-credentials OpenShell middleware. It translates an +OpenShell HTTP-request evaluation into one text-processing run and returns an +allow, replacement, or deny result. + +## System boundary + +![Privacy Guard component architecture. The service layer translates +OpenShell gRPC messages, the request processor applies policy, and registered +engines detect or replace entities without depending on the transport +layer.](../../../assets/privacy-guard/diagrams/component-architecture.svg) + +OpenShell owns request routing and provider credential attachment. Privacy +Guard does not send provider requests. + +## Components + +Source paths below are relative to `projects/privacy-guard/src/privacy_guard/`. -- `service/` owns gRPC, protobuf conversion, UTF-8 decoding and encoding, - bounded worker scheduling, active-processor preparation and replacement, and - finding serialization. Outside generated `bindings/`, no other package - imports gRPC or generated bindings. -- `cli.py` owns command parsing, registry-factory loading, engine discovery, - configuration-schema output, logging options, and the adapter that starts - the programmatic server. Top-level `logging.py` provides the shared, - standard-library logging configuration used by the CLI and available to - programmatic deployments. -- `request_processor.py` runs configured stages over one text value, shares one - timeout across them, aggregates detections, and applies the user-facing - policy action. It does not import gRPC or implement an engine's algorithms. -- `engines/` defines the custom-engine contract, registers engine - implementations and operator-owned resources, builds the exact Pydantic - discriminated union, and contains the built-in Regex implementation. Each - engine owns its detection and replacement algorithms. -- `config.py` defines ordered stages and the required policy action. -- top-level `base.py` defines the package-wide strict immutable domain-model - base. -- `string_validators.py` defines shared string validators and field types. - -The OpenShell policy is the single source of privacy behavior: stage order, -each stage's exact engine configuration, entity definitions, replacement -recipes, and the final `detect`, `block`, or `replace` action. Deployment -configuration registers implementations and injects operational resources such -as model profiles, endpoints, clients, and credentials. - -## Request flow - -```text -HttpRequestEvaluation protobuf - | - v -PrivacyGuardMiddleware - validates phase and body size - validates and normalizes policy configuration - reuses the matching active RequestProcessor or prepares and activates - a replacement - decodes a non-empty body as strict UTF-8 - | - v -RequestProcessor.process(text) - derives DETECT or REPLACE engine strategy - creates one shared Timeout - | - v -stage 1 engine.run(current text) - | - v -stage 2 engine.run(stage 1 text) - | - ... - | - v -RequestProcessor - aggregates stage-qualified detections - applies detect, block, or replace - | - v -RequestProcessingResult - | - v -PrivacyGuardMiddleware - serializes bounded findings - encodes replacement text only for replace - | - v -HttpRequestResult protobuf -``` - -The processor passes only `EntityProcessingStrategy.DETECT` or -`EntityProcessingStrategy.REPLACE` to engines. Blocking is a request -disposition and never crosses the engine boundary. - -The processor is synchronous. The service runs it in a dedicated thread pool -so engine work does not block the gRPC event loop. - -## Core data types - -| Type | Meaning | +| Component | Responsibility | | --- | --- | -| `PrivacyGuardConfig` | Ordered entity-processing stages and the required action on detection | -| `EntityProcessingStage` | One configured engine invocation with an optional diagnostic name | -| `EngineConfig` | Nominal strict base for an engine's exact policy configuration | -| `EntityProcessingStrategy` | Per-run engine selection: detect or replace | -| `EntityDetection` | One occurrence with stage-input offsets and optional confidence | -| `TextProcessingResult` | One engine's authoritative output text and detections | -| `Timeout` | One monotonic deadline shared across all stages | -| `EntityDetectionSummary` | Bounded stage/entity/confidence aggregate for audit output | -| `RequestProcessingResult` | Allow or deny decision, detection summaries, and replacement text when requested | - -Pydantic domain models are strict, frozen, reject unknown fields, hide rejected -input from validation errors, and suppress sensitive fields from normal -representations. - -## Deliberate omissions - -- Cross-request entity memory is not part of v0. -- Engines do not receive transport metadata or the user-facing policy action. -- There is no parallel execution-plan model; preparation constructs a - `RequestProcessor` directly. -- There is no generic replacement field or replacement-strategy enum. Each - engine owns any replacement settings appropriate to its underlying - algorithm. -- Regex catalogs may be supplied inline or as bounded relative YAML paths - beneath Privacy Guard's working directory. +| `cli.py` | CLI commands, registry-factory loading, engine discovery, schema output, and server startup | +| `gateway_config.py` | Safe updates to OpenShell gateway middleware registrations | +| `service/` | gRPC lifecycle, protobuf conversion, transport validation, active configuration, worker scheduling, and result serialization | +| `request_processor.py` | Ordered stage execution, shared timeout, aggregation, and policy action | +| `engines/base.py` | Engine lifecycle and result-contract enforcement | +| `engines/registry.py` | Engine and resource registration, policy schema construction, discovery, and engine creation | +| `engines/regex.py` | Regex catalog validation, matching, overlap handling, caching, and replacement | +| `config.py` | Policy stages and action models | +| `base.py` | Shared strict immutable domain-model base | +| `timeout.py` | Monotonic shared request deadline | +| `errors.py` | Stable content-safe error catalog | +| `logging.py` | Standard-library logging configuration and content-safe records | + +Only `service/` imports gRPC and generated protobuf bindings. Engines and the +request processor use domain models and can be tested without the transport +layer. + +## Configuration and runtime resources + +Privacy behavior comes from the OpenShell policy: + +- ordered stages +- concrete engine configuration +- entity definitions +- replacement recipes +- final `detect`, `block`, or `replace` action + +Operational resources come from the running application: + +- installed engine implementations +- clients and SDK adapters +- endpoints and credential providers +- approved models or profiles +- processing timeout + +`EngineRegistry.finalize()` constructs one Pydantic discriminated union from +the exact config type registered for each engine. `stage.config.engine` selects +the union member. + +## Text boundary + +The service validates request bytes and decodes a non-empty body as strict +UTF-8. The request processor receives exactly one `str`. + +Headers, target, request ID, content type, and protobuf values do not cross the +processor boundary. Privacy Guard does not parse JSON or create structured +regions. + +Detect and block leave the original bytes untouched. Replace UTF-8 encodes the +final text and returns it as the new body. + +## Processing pipeline + +![Privacy Guard processing pipeline from validated policy and input text +through strategy selection, one shared timeout, ordered engine stages, +validation, aggregation, and the final policy +decision.](../../../assets/privacy-guard/diagrams/processing-pipeline.svg) + +Engines receive `DETECT` or `REPLACE`, not the user-facing policy action. +Blocking is applied after detection by `RequestProcessor`. + +## State + +Privacy Guard retains: + +- one active validated policy +- one configured `RequestProcessor` +- immutable configured engine instances +- operator-injected resources +- a bounded compiled Regex catalog cache + +It does not retain request text, detections, replacement mappings, or +cross-request entity memory. + +Equal validated configuration reuses the active processor. A changed +configuration is fully prepared and atomically activated. Failed preparation +does not replace the active processor. + +## Concurrency + +The gRPC server accepts at most 16 concurrent RPCs. Synchronous configuration +and processing use a four-slot worker pool. + +Engine instances and resources can be used by multiple worker threads. They +must keep per-request state local to the call and support concurrent access. +Policy preparation is serialized; evaluations already using the previous +processor may finish while later evaluations use the replacement. + +## Result boundary + +The processor returns only: + +- allow or deny +- final replacement text when applicable +- stage-qualified detection summaries +- a stable deny reason when applicable + +Detection summaries omit matched text, context, offsets, patterns, and raw +engine metadata. ## Read next -- [Request lifecycle](request-lifecycle.md) explains configuration resolution, - ordered execution, actions, and output behavior. -- [Entity-processing engines](entity-processing-engines.md) defines the - extension contract and built-in engines. -- [Configuration and text boundary](configuration.md) covers the one-text - contract, configuration ownership, and current catalog limits. -- [Service boundary](service-boundary.md) covers gRPC adaptation, policy - activation, and concurrency. -- [Safety and limits](safety-and-limits.md) records failure behavior and - resource bounds. +- [Request lifecycle](request-lifecycle.md) +- [Service boundary](service-boundary.md) +- [Configure policies](../configuration.md) +- [Add a custom engine](../engines/custom.md) +- [Limits and failure behavior](../reference/limits-and-failures.md) diff --git a/docs/documentation/privacy-guard/architecture/request-lifecycle.md b/docs/documentation/privacy-guard/architecture/request-lifecycle.md index 9a0cda9..73c79c4 100644 --- a/docs/documentation/privacy-guard/architecture/request-lifecycle.md +++ b/docs/documentation/privacy-guard/architecture/request-lifecycle.md @@ -1,100 +1,82 @@ --- title: Request lifecycle -description: How Privacy Guard prepares policy, runs ordered engines, and applies one request action. +description: How Privacy Guard validates configuration, runs stages, and returns an OpenShell request decision. agent_markdown: true --- # Request lifecycle -`RequestProcessor` owns the complete protobuf-free flow for one text value. It -runs configured entity-processing stages in order, aggregates detections, and -applies the policy action. - -The gRPC service owns the surrounding bytes/text and configuration transport -boundaries. - -## Policy configuration - -The policy supplies: - -```yaml -entity_processing: - stages: - - name: optional-diagnostic-name - config: - engine: regex - pattern_catalog: - entities: - - name: email - rules: - - pattern: '...' - confidence: high - replacement: - strategy: template - template: "[{entity}]" -on_detection: - action: detect -``` +Each OpenShell evaluation carries request metadata, body bytes, and complete +Privacy Guard configuration. The service validates the transport, resolves the +active processor, processes one text value, and serializes the result. -`entity_processing` groups the ordered stages and owns their non-empty-list and -unique-diagnostic-name validation. V0 defines only `stages`. +## Lifecycle summary -Each `EntityProcessingStage` contains: +![Privacy Guard request lifecycle. The service validates the transport and +policy, resolves the active processor, handles an empty body or decodes UTF-8 +text, runs the ordered engine stages, then returns either a gRPC failure or a +successful allow, replacement, or deny +result.](../../../assets/privacy-guard/diagrams/request-lifecycle.svg) -- optional `name`, used only as bounded diagnostic provenance -- required `config`, which is the exact concrete configuration model owned by - the selected engine +## 1. Validate the transport -`config.engine` is a Pydantic discriminator. After every implementation is -registered, `EngineRegistry.finalize()` constructs a real discriminated union -of their concrete config models. Engine-specific fields and nested replacement -variants therefore validate, serialize, and appear in JSON Schema without a -generic mapping or translation layer. +The service checks: -When a stage name is omitted, Privacy Guard derives a deterministic one-based -label such as `regex[1]`. All resulting diagnostic names must be unique. +- pre-credentials middleware phase +- request context size +- policy configuration size +- target size +- header count and size +- request body size -## Configuration resolution and preparation +Input at another middleware phase or outside a transport bound fails the RPC as +invalid input. -For each evaluation under the current OpenShell protocol, the service: +## 2. Validate configuration -1. converts the protobuf `Struct` to a mapping -2. validates it through the finalized registry-backed Pydantic model -3. validates each concrete config against its registered implementation and - injected resources -4. validates the action/replacement compatibility -5. reuses the active `RequestProcessor` when its complete immutable validated - configuration is equal -6. otherwise serializes preparation, constructs every configured engine and a - candidate processor, and atomically activates the candidate only after full - success +The service converts the protobuf `Struct` to a mapping and validates it through +the finalized registry-backed Pydantic model. -The first evaluation establishes the active policy and pays its preparation -cost. A failed validation or preparation leaves an existing active processor -unchanged and fails the triggering evaluation. `ValidateConfig` performs -validation without constructing engines or changing the active policy. +Validation includes: -There is no separate execution-plan abstraction. The validated stage order -already contains the necessary policy structure, and the active processor -privately retains the corresponding ordered engine instances. +- exact fields and strict types +- known engine discriminator +- stage count and unique stage names +- engine-specific config +- registered resource compatibility +- action and replacement-strategy compatibility +- catalog shape and Regex pattern compilation -## Text input +`ValidateConfig` performs these checks without constructing engines or changing +the active processor. -The service validates the pre-credentials phase and the request body byte -limit before processing. It still validates configuration and resolves the -active processor for an empty body, then immediately allows that body without -invoking an engine. +## 3. Resolve the active processor -A non-empty body must decode as strict UTF-8. The decoded `str` is the only -request input passed to `RequestProcessor`; headers, content type, request ID, -target, and protobuf messages do not cross that boundary. +The service compares the complete immutable validated configuration with the +active one: + +| State | Behavior | +| --- | --- | +| Equal configuration | Reuse the active processor | +| No active processor | Construct every stage engine and activate the processor | +| Changed configuration | Prepare a complete candidate and atomically replace the active processor | +| Validation or preparation failure | Keep the active processor unchanged and fail the triggering evaluation | -The processor validates the encoded UTF-8 byte bound before running the -pipeline. +Only one preparation path runs at a time. Configuration validation still runs +for every evaluation. + +## 4. Decode the request body + +An empty body is allowed without invoking an engine after configuration +validation and processor resolution. + +A non-empty body must decode as strict UTF-8. The decoded `str` is the only +request input passed to `RequestProcessor`. Headers, target, content type, +request ID, and protobuf objects remain in the service layer. -## Ordered stage execution +## 5. Select the engine strategy -The processor derives one invocation strategy for the whole pipeline: +The processor derives one strategy for the complete stage pipeline: | Policy action | Engine strategy | | --- | --- | @@ -102,77 +84,90 @@ The processor derives one invocation strategy for the whole pipeline: | `block` | `DETECT` | | `replace` | `REPLACE` | -`PolicyAction` is never passed to an engine. +Engines never receive `PolicyAction`. -The processor then: +## 6. Run stages in order -1. creates one monotonic `Timeout` -2. calls each stage exactly once in policy order -3. passes the current text, invocation strategy, and shared timeout to - `engine.run()` -4. validates intermediate UTF-8 byte and detection limits -5. passes the returned text to the next stage -6. checks the same timeout after the final result +The processor: -In detect and block mode, the public engine contract requires returned text to -equal that stage's input. In replace mode, each later stage sees the preceding -stage's processed text. +1. validates the input UTF-8 size +2. creates one monotonic `Timeout` +3. calls each stage once in policy order +4. passes the current text, strategy, and shared timeout +5. validates each returned result and intermediate size +6. passes returned text to the next stage +7. checks the timeout after final result validation -Detection offsets always refer to the input revision seen by the producing -stage. Privacy Guard does not reinterpret earlier offsets after a later stage -changes the text. +In `DETECT`, each engine must return its input text unchanged. In `REPLACE`, a +later stage receives the preceding stage's output. -If a stage times out, exceeds an execution limit, or fails, its partial text and -detections are discarded. No later stage runs. +If a stage times out, exceeds a limit, or fails, its partial output is discarded +and later stages do not run. -## Detection aggregation +## 7. Validate engine output -After all stages succeed, the processor aggregates detections by: +The public engine wrapper checks: + +- returned model type +- supported strategy +- non-empty spans within the stage input +- per-stage detection count +- output UTF-8 size +- unchanged text in `DETECT` +- at least one detection when text changes in `REPLACE` + +Detection offsets refer to the stage input. Privacy Guard does not remap earlier +offsets after later replacement stages change the text. + +## 8. Aggregate detections + +After all stages succeed, the processor aggregates occurrences by: ```text -source stage + entity + confidence representation +source stage + entity + confidence ``` -It does not deduplicate across stages. Two stages may have inspected different -text revisions, and confidence values from different tools are not assumed to -be calibrated. +It does not deduplicate across stages. Stages may inspect different text +revisions, and confidence values from different engines are not assumed to be +calibrated. -The aggregate `EntityDetectionSummary` intentionally omits matched text, -surrounding text, offsets, patterns, and raw engine metadata. +Aggregated summaries exclude matched text, surrounding text, offsets, patterns, +and engine metadata. -## Applying the policy action - -The processor owns the final disposition: +## 9. Apply the policy action | Action | No detections | One or more detections | | --- | --- | --- | -| `detect` | Allow original body | Allow original body and report detection summaries | -| `block` | Allow original body | Deny with `privacy_guard_blocked` and report detection summaries | -| `replace` | Allow final processed text | Allow final processed text and report detection summaries | +| `detect` | Allow original body | Allow original body and report summaries | +| `block` | Allow original body | Deny with `privacy_guard_blocked` and report summaries | +| `replace` | Allow final text | Allow final text and report summaries | + +A timeout or processing-limit failure returns a deny with +`privacy_guard_limit_exceeded` and no partial summaries or replacement. -For `replace`, configuration validation requires every stage to advertise -replacement support and to include its valid engine-specific replacement -recipe. A recipe may remain configured but dormant when the action is changed -to detect or block. +## 10. Serialize the result -Replacement behavior belongs to each engine. For example, `RegexEngine` -selects deterministic non-overlapping matches and renders its constrained -template. A custom engine backed by another tool owns that tool's native -replacement operation. `RequestProcessor` does not reproduce either algorithm. +The service maps the domain result to OpenShell: -## Output +| Domain result | OpenShell result | +| --- | --- | +| Detect allow | `DECISION_ALLOW`, `has_body=false` | +| Block with no detections | `DECISION_ALLOW`, `has_body=false` | +| Replace allow | `DECISION_ALLOW`, `has_body=true`, final UTF-8 body | +| Policy block | `DECISION_DENY`, `privacy_guard_blocked` | +| Limit deny | `DECISION_DENY`, `privacy_guard_limit_exceeded` | -`RequestProcessor.process()` returns a `RequestProcessingResult`: +For detect and block, OpenShell keeps the exact original bytes. For replace, +Privacy Guard returns the final encoded text even when it equals the input. -- detect and block-without-detections return `ALLOW` without replacement text -- block-with-detections returns `DENY`, detection summaries, and - `privacy_guard_blocked` -- replace returns `ALLOW`, the final text, and detection summaries -- timeout or execution-limit exhaustion returns `DENY` with - `privacy_guard_limit_exceeded` and no partial summaries or replacement +## Failure outcomes -The service leaves the original request bytes untouched for detect and block. -For replace it UTF-8 encodes the final text and sets `has_body=true`, including -when the final text happens to equal the input. +| Outcome | Transport status | Request effect | +| --- | --- | --- | +| Policy block | Successful RPC | Deny | +| Processing limit | Successful RPC | Deny | +| Invalid request or configuration | `INVALID_ARGUMENT` | OpenShell applies middleware `on_error` | +| Internal engine or service error | `INTERNAL` | OpenShell applies middleware `on_error` | -[Back to the architecture overview](index.md) +See [Limits and failure behavior](../reference/limits-and-failures.md) for the +complete error and bound reference. diff --git a/docs/documentation/privacy-guard/architecture/safety-and-limits.md b/docs/documentation/privacy-guard/architecture/safety-and-limits.md deleted file mode 100644 index 3b22fec..0000000 --- a/docs/documentation/privacy-guard/architecture/safety-and-limits.md +++ /dev/null @@ -1,286 +0,0 @@ ---- -title: Safety and limits -description: Validation ownership, bounded entity processing, failure behavior, and logging rules. -agent_markdown: true ---- - -# Safety and limits - -Privacy Guard bounds request text, engine output, detections, regex execution, -transport output, and concurrency. Limits are part of the architecture, not -optional tuning defaults. - -Package-wide values live in `constants.py`. - -## Validation ownership - -Each layer validates facts it can establish: - -| Layer | Validates | -| --- | --- | -| Policy and engine config models | Strict types, exact fields, discriminators, local field rules, catalog bounds, and regex validity | -| `EngineRegistry` | Registration contracts, exact config and resource types, pure resource-backed config checks, and action/replacement compatibility | -| `EntityProcessingEngine.run()` | Text and invocation types, supported strategy, timeout, result model, spans, detection-only immutability, mutation attribution, and stage output bounds | -| Concrete engine | Algorithm-specific execution, replacement completeness, backend timeout propagation, and native failure normalization | -| `RequestProcessor` | Request text bounds, shared timeout, intermediate output, request-wide detection count, aggregation, and policy disposition | -| Service | Phase, transport body size, strict UTF-8, protobuf conversion, replacement encoding, finding representation, and response limits | - -Keep request-wide policy decisions in the processor and algorithm-specific -behavior in the concrete engine. The service adapts transport but does not -reimplement either. - -## Text, processing, and transport limits - -| Limit | Constant | Value | Owner | -| --- | --- | ---: | --- | -| Incoming request body | `MAX_BODY_BYTES` | 4 MiB | Service | -| Input or intermediate UTF-8 bytes | `MAX_BODY_BYTES` | 4 MiB | Processor | -| Engine output UTF-8 bytes | `MAX_BODY_BYTES` | 4 MiB | Engine wrapper | -| Encoded replacement body | `MAX_BODY_BYTES` | 4 MiB | Service | -| gRPC receive message | `MAX_RECEIVE_MESSAGE_BYTES` | 5 MiB | gRPC server | -| Detections returned by one stage | `MAX_DETECTIONS_PER_STAGE` | 256 | Engine wrapper | -| Detections across one request | `MAX_DETECTIONS_PER_REQUEST` | 4,096 | Processor | -| Default shared timeout | `DEFAULT_TIMEOUT_SECONDS` | 1 second | Server and processor | -| Maximum configured timeout | `MAX_TIMEOUT_SECONDS` | 30 seconds | Server, processor, and `Timeout` | -| Active processor workers | `MAX_CONCURRENT_PROCESSING` | 4 | Service | -| Concurrent gRPC calls | `MAX_CONCURRENT_RPCS` | 16 | gRPC server | - -The gRPC receive allowance is 1 MiB larger than the request body limit for the -protobuf envelope. The service still enforces the advertised 4 MiB body bound. - -One monotonic `Timeout` is shared by all stages and final result validation. -`RegexEngine` passes the remaining duration into every backend search. -Third-party calls that accept a timeout should receive the same remaining -duration; calls that cannot be interrupted continue to occupy a service worker -until they exit. - -Operators configure the internal bound with `--timeout-seconds` or the -programmatic `PrivacyGuardServer(timeout_seconds=...)` argument. OpenShell's -outer middleware timeout must include additional headroom for worker queueing -and configuration or engine preparation beyond this internal bound. - -## Diagnostic and result limits - -| Limit | Constant | Value | -| --- | --- | ---: | -| Bounded diagnostic string | `MAX_DIAGNOSTIC_TEXT_BYTES` | 1,024 UTF-8 bytes | -| Metadata entries per detection | `MAX_FINDING_METADATA_ENTRIES` | 32 | -| Aggregated protobuf finding groups | `MAX_PROTO_FINDING_GROUPS` | 32 | -| Encoded size per protobuf finding | `MAX_PROTO_FINDING_BYTES` | 4 KiB | - -The diagnostic-string bound applies to stage names, entity names, metadata -keys and values, model-profile names, and other audit-safe identifiers built -from the shared domain field type. These values must be non-empty printable -Unicode without control or bidirectional formatting characters. Untrusted -request IDs use the same content and size rules for logging; an invalid request -ID is represented by a constant placeholder and does not change the evaluation -result. The human-readable log message quotes request IDs and escapes ASCII -spaces so their text cannot introduce another key/value token; structured log -records retain the exact validated ID. Regex entity and supplied rule names -have a stricter ASCII grammar and limit described below. - -The processor aggregates occurrences before the service applies protobuf -limits. If a safe result cannot be represented, the service returns a limit -deny with no partial findings. - -## Regex and catalog limits - -| Limit | Constant | Value | -| --- | --- | ---: | -| Regex entity or supplied rule name | `MAX_REGEX_NAME_BYTES` | 128 ASCII bytes | -| Entities per catalog | `MAX_REGEX_ENTITIES_PER_CATALOG` | 2,000 | -| Rules per catalog | `MAX_REGEX_RULES_PER_CATALOG` | 10,000 | -| Pattern string | `MAX_REGEX_PATTERN_BYTES` | 16 KiB | - -Entity and rule names use -`[A-Za-z_][A-Za-z0-9_-]*`. Rule names are optional; their deterministic -derived diagnostic identities are not serialized back into configuration. - -The per-evaluation config `Struct` is limited to 64 KiB. A catalog may satisfy -Privacy Guard's engine limits yet remain too large for the current upstream -protocol. Internal caching does not raise that transport ceiling. - -## Regex execution safety - -`RegexEngine` validates and compiles the complete catalog before accepting -configuration. It: - -- rejects invalid and empty pattern strings -- rejects patterns that match empty input -- rejects user-defined named groups and inline flags -- verifies its private trailing marker on every match -- treats a context-dependent zero-width match as an atomic configuration - failure -- evaluates patterns independently to retain overlaps -- uses the timeout-capable third-party `regex` backend -- caps detections per stage -- projects exact UTF-8 replacement size before rendering - -No timeout, limit, or pattern failure returns partial stage detections or -mutated text. - -## When a limit is exceeded - -The processor returns a successful deny with -`privacy_guard_limit_exceeded` and no partial findings or replacement when: - -- the shared timeout expires -- an engine exceeds its detection or output limit -- intermediate text exceeds the UTF-8 byte limit -- aggregate request detections exceed the limit - -The service returns the same bounded deny when: - -- findings exceed the protobuf group or encoded-size limit -- replacement text cannot be encoded within the body limit -- a deny reason code is not safely representable - -The processor emits a content-safe `timeout` or `resource` limit kind, and the -service adapter emits `resource` for representation limits. Neither log includes -request content or collaborator error text. The deny reason directs users to -check that log, then reduce the request or replacement size, simplify the -configured stages and rules, or increase `--timeout-seconds` or -`PrivacyGuardServer(timeout_seconds=...)` up to 30 seconds. When increasing the -processing timeout, users must give OpenShell's middleware timeout additional -headroom for worker queueing and configuration or engine preparation. These -options reduce bounded output pressure or provide appropriate time without -weakening the fail-closed behavior. - -Malformed input and engine contract or execution failures instead abort the RPC -with a cataloged error. OpenShell then applies the middleware registration's -failure behavior. - -The distinction is deliberate: - -- an exhausted declared processing bound is a closed, successful deny -- invalid user input is an `INVALID_ARGUMENT` RPC failure -- an engine or middleware defect is an `INTERNAL` RPC failure - -## Error catalog - -Production failures use `PrivacyGuardError` and a stable `ErrorCode`. Each -error spec defines: - -- invalid-input or internal classification -- responsible component -- failed operation -- content-safe summary -- remediation hint - -Successful policy and safety-limit denials use stable reason codes plus -content-safe recovery guidance; they do not add domain fields solely for -diagnostic logging. CLI validation failures name the rejected option and state -the accepted form or next action. Cataloged server-startup failures print one -content-safe message and exit nonzero rather than exposing a traceback. -Internal engine and third-party failures remain concise because the owning -boundary translates them before they reach users. - -Caught extension and third-party exceptions are translated at their trust -boundary. Raw exception messages and chains are not exposed. Configuration -errors do not include rejected pattern strings, paths, credentials, provider -endpoints, or request text. - -Use a new catalog entry for a genuinely new externally observable failure. Do -not return raw Pydantic, regex-backend, engine, or protobuf exceptions. - -## Logging - -Default operational logs include: - -- request ID -- evaluation duration -- allow or deny action -- aggregate finding count -- stable error code -- stage source and invocation strategy at debug level - -They do not include: - -- request or replacement text -- matched or surrounding text -- offsets -- patterns or catalog contents -- headers, targets, or credentials -- model endpoints -- arbitrary exception text - -Sensitive domain fields use `repr=False` to reduce accidental exposure. This -does not replace safe exception handling. - -`--debug` enables content-safe processing diagnostics. -`--debug-log-content` deliberately logs complete input and processed text and -emits a warning; use it only in a controlled development environment. - -## Extension requirements - -Custom engines should: - -- use the base constructor and public `run()` wrapper -- return the exact `TextProcessingResult` contract -- return stable, declared entity identifiers rather than values derived from - request text -- keep request data local to `_run()` -- keep initialized state immutable and make resources concurrency-safe -- pass the shared remaining timeout to delegated APIs when they support it -- fail the stage on native partial failure -- use `TextProcessingResult.from_detections()` to bound lazy detection streams -- add manual bounds only when a low-level collaborator allocates proportional - output before Privacy Guard can consume it -- translate expected collaborator failures to the content-safe engine - exception hierarchy -- avoid logging input or caught exception text - -Privacy Guard keeps framework-controlled fields and identifiers that pass its -shared validation content-safe for findings and operational logs. -Operator-installed Python engine code is trusted deployment code: it can access -request text and must honor the declared identifier and logging contract. The -framework cannot make arbitrary trusted Python code non-exfiltrating, so -operators must review and install custom engines with the same care as other -code in the middleware process. - -Do not add retries, fallback providers, persistent request artifacts, or extra -validation without a concrete failure mode and a clear owning layer. - -## State and retention - -Cross-request entity memory is not implemented. Privacy Guard retains one active -validated policy and its immutable configured engine state, but never request -text, detections, or replacement mappings. The active processor is deliberate -process state, not an LRU entry: it remains until successful replacement or -shutdown. - -During an update, the fully prepared candidate may coexist with the active -processor. Evaluations that already captured the prior processor may keep it -alive until they finish. A failed candidate leaves active state unchanged. - -Regex's supporting compiled-rule cache remains bounded: - -| Cache | Weight budget | Entry cap | Entry weight | -| --- | ---: | ---: | --- | -| Compiled Regex rules | 32 MiB | 128 | canonical catalog bytes plus 4 KiB per rule | - -The compiled-rule allowance accounts for retained backend state that is much -larger than short pattern text. One entry that exceeds the budget remains valid -and usable for the current operation but is not retained there. This does not -invalidate an active or candidate processor holding those rules. The active -policy is not rejected by cache admission; policy and schema limits bound its -shape, while custom-engine prepared memory remains an operator responsibility. -Regex cache skip and eviction records contain only the cache name and weights, -plus eviction counts. - -## Changing a limit - -A limit change may affect policy schema, processor behavior, engine contracts, -protobuf serialization, tests, examples, and the OpenShell middleware -registration. - -Before changing a limit: - -1. identify every layer that enforces or advertises it -2. update exact-boundary tests -3. check the failure result immediately above the boundary -4. run the complete project validation -5. benchmark changes that affect regex compilation or request processing -6. coordinate upstream when the copied OpenShell protocol is involved - -[Back to the architecture overview](index.md) diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/docs/documentation/privacy-guard/architecture/service-boundary.md index 01ca64f..6a99389 100644 --- a/docs/documentation/privacy-guard/architecture/service-boundary.md +++ b/docs/documentation/privacy-guard/architecture/service-boundary.md @@ -1,234 +1,152 @@ --- title: Service boundary -description: gRPC adaptation, policy activation, worker scheduling, and server lifecycle. +description: Privacy Guard gRPC methods, protobuf translation, active policy management, and worker scheduling. agent_markdown: true --- # Service boundary -Outside the generated `bindings/` package, `service/` is the only layer that -imports gRPC or generated protobuf bindings. It translates between OpenShell's -transport contract and Privacy Guard's one-text domain contract. +The `service/` package adapts OpenShell's `SupervisorMiddleware` gRPC contract +to Privacy Guard's protobuf-free domain model. No other handwritten package +imports gRPC or generated bindings. -The `.proto` file and generated bindings checked into this project are copied -from OpenShell. Protocol changes must land upstream and be adopted from a new -canonical copy; Privacy Guard must not create a private fork. +The checked-in protocol and generated bindings are canonical OpenShell copies. ## gRPC methods -Privacy Guard implements the three methods currently defined by -`SupervisorMiddleware`: - | RPC | Behavior | | --- | --- | -| `Describe` | Advertises service identity, one pre-credentials HTTP binding, and the 4 MiB body limit | -| `ValidateConfig` | Validates supplied policy configuration and registered resources | -| `EvaluateHttpRequest` | Validates transport input, resolves the active processor, processes text, and returns a decision | +| `Describe` | Advertise service identity, the pre-credentials HTTP binding, and the 4 MiB body limit | +| `ValidateConfig` | Validate policy configuration against the active engine registry and resources | +| `EvaluateHttpRequest` | Validate transport input, resolve the configured processor, process text, and return a decision | `Describe` advertises only -`SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS`. Evaluations at any other phase -are invalid input. +`SUPERVISOR_MIDDLEWARE_PHASE_PRE_CREDENTIALS`. -The current manifest message has no field for the finalized policy schema or -engine discovery metadata. Those are available through the local -`privacy-guard configuration-schema` and `privacy-guard engines` commands. +Engine discovery and the finalized policy schema are available through: -## Configuration lifecycle +```bash +privacy-guard engines +privacy-guard configuration-schema +``` -The current protocol carries a complete `google.protobuf.Struct` on both -validation and every request evaluation. +## Configuration transport -`ValidateConfig`: +OpenShell supplies the complete policy config as `google.protobuf.Struct` for +validation and evaluation. The service: -1. rejects an encoded `Struct` larger than 64 KiB before conversion +1. rejects an encoded config larger than 64 KiB 2. converts the `Struct` to a mapping -3. validates the registry-built discriminated policy model -4. validates each exact engine config against registered resources -5. validates replacement support for a replace action -6. returns `valid=true`, or a content-safe reason - -It does not construct engines, change the active policy, contact model providers, -download resources, or write artifacts. Validation runs in the bounded worker -pool so catalog parsing and engine-specific checks do not block the async gRPC -event loop. - -During evaluation, the service validates and normalizes the same config. It -retains one active pair: the complete immutable validated configuration and its -configured `RequestProcessor`. Equal config reuses the processor. When no -processor is active or config differs, one serialized update path constructs -engines from the exact stage configs and operator-injected resources, constructs -a candidate processor, and atomically replaces the active pair only after -preparation succeeds. Failure preserves the prior active pair and fails the -triggering evaluation. - -Validation, resolution or preparation, strict UTF-8 decoding, and processing all -run in the bounded worker pool. Validation still occurs for every evaluation. -Equivalent normalized Regex catalogs may reuse an entry in the independent -compiled-rule cache during validation and engine construction. - -Regex's compiled-rule LRU retains at most 128 catalogs and 32 MiB. Each entry -weighs its canonical catalog bytes plus 4 KiB per rule. Evicting an entry only -removes the cache's reference. A processor that already uses those rules remains -valid. - -An otherwise valid compiled-rule entry larger than the byte budget is built and -used for the current operation but is not retained in that cache. The active -processor itself has no LRU admission budget: it is deliberate process state -retained until replacement or shutdown. Regex cache eviction or a skipped entry -never invalidates it. Process restart discards the active processor, and the -first later evaluation reconstructs it from supplied configuration. - -Each active processor receives the server's operational processing timeout. -The default is 1 second shared across every configured stage. Operators may set -up to 30 seconds with `privacy-guard serve --timeout-seconds`; programmatic -applications pass the same `timeout_seconds` argument to `PrivacyGuardServer`. -OpenShell's outer middleware `timeout` must include the processing timeout plus -additional headroom for worker queueing, repeated configuration validation, -and occasional candidate preparation so the supervisor does not end the RPC -first. - -## Incoming requests +3. normalizes finite integral doubles in the safe integer range +4. validates the registry-built policy model +5. validates engine config against registered resources +6. checks replacement support for `replace` -For each evaluation, the service: +`Struct` represents every number as a double. The service converts finite +integral values only within `-(2^53 - 1)` through `2^53 - 1` before strict +model validation. Other values remain floats. -1. validates the pre-credentials phase -2. validates the encoded context (4 KiB), configuration (64 KiB), target - (32 KiB), headers (128 entries and 64 KiB), and body (4 MiB) limits -3. acquires one bounded worker slot -4. converts transport-safe integral numbers, validates the maximum ten stages, - and resolves or updates the active policy in that worker -5. allows an empty body without invoking an engine -6. decodes a non-empty body as strict UTF-8 -7. calls `RequestProcessor.process(text)` in the same worker +## Active policy -Request context, target, headers, middleware name, and protobuf values remain at -the service boundary. The request ID is used only for content-safe operational -logging. +The service retains one pair: -## Outgoing results +```text +validated PrivacyGuardConfig + configured RequestProcessor +``` -The processor returns `RequestProcessingResult`. The service maps it to -`HttpRequestResult`: +Equal validated configuration reuses this pair. A changed configuration enters +a serialized preparation path: -| Domain result | Protobuf result | -| --- | --- | -| Detect allow | `DECISION_ALLOW`, `has_body=false` | -| Block allow with no detections | `DECISION_ALLOW`, `has_body=false` | -| Replace allow | `DECISION_ALLOW`, `has_body=true`, final UTF-8 body | -| Policy block | `DECISION_DENY`, `privacy_guard_blocked`, no body | -| Limit deny | `DECISION_DENY`, `privacy_guard_limit_exceeded`, no body or partial findings | +1. construct each configured engine from its exact config and registered + resources +2. construct the candidate processor +3. activate the complete candidate atomically -The service checks the encoded replacement size again before serialization. +Failed preparation leaves the active pair unchanged. An evaluation that already +captured the previous processor can finish with it while later evaluations use +the replacement. -### Findings +## Request adaptation -The processor has already aggregated occurrences by source stage, entity, and -confidence. For each `EntityDetectionSummary`, the service emits: +For each evaluation, the service: -- `type`: `detected_entity` -- `label`: `entity (source-stage)` -- `confidence`: the categorical value or empty when absent -- `count`: aggregate occurrence count +1. validates the phase +2. validates context, config, target, headers, and body limits +3. acquires a processing slot +4. validates config and resolves the processor in a worker +5. allows an empty body without engine invocation +6. decodes a non-empty body as strict UTF-8 +7. runs `RequestProcessor.process(text)` in the same worker -The current OpenShell `Finding` message has no dedicated source field. -Stage provenance is therefore secondary text in the bounded label while the -entity remains primary. Matched content, offsets, patterns, and raw engine -metadata never cross the protobuf boundary. +Request metadata stays in the service layer. The request ID is used only for +content-safe logging. -## Concurrency model +## Result adaptation -The gRPC server accepts at most 16 concurrent RPCs. Synchronous processing uses -a dedicated executor and semaphore with four active slots: +The service maps `RequestProcessingResult` to `HttpRequestResult` and validates +the encoded replacement and finding representation before returning. -```text -gRPC event loop - | - | acquire processing slot - v -4-slot semaphore - | - v -4-thread executor - | - v -configuration validation -and active-policy resolution - | - v -RequestProcessor.process - | - v -ordered engine pipeline -``` +### Findings -This keeps synchronous configuration and engine work off the async event loop -and bounds the number of active operations. +Each aggregated detection summary becomes one OpenShell finding: -The active processor, engine instances, and injected resources may be used by -multiple worker threads. They must retain no mutable per-request state and must -be safe for concurrent access. Policy preparation is serialized. A candidate is -fully prepared before the active reference is swapped. An evaluation that -already captured the old processor may finish with it while later evaluations -use the replacement; this is an atomic reference cutover, not a draining -cutover. +| Finding field | Value | +| --- | --- | +| `type` | `detected_entity` | +| `label` | `entity (source-stage)` | +| `confidence` | `low`, `medium`, `high`, or empty | +| `count` | Number of aggregated occurrences | -## Cancellation +Matched content, context, offsets, patterns, and raw engine metadata do not +cross the protobuf boundary. -Cancelling an async RPC cannot stop Python code already running in its worker -thread. The service shields the worker bridge and releases the semaphore slot -only after that worker actually finishes. +If the complete result cannot fit OpenShell's finding bounds, the service +returns `privacy_guard_limit_exceeded` without partial findings. -Cancellation therefore cannot create more than four active synchronous -operations. -An engine should pass the remaining shared timeout to any delegated API that -supports bounded execution. A non-preemptible call continues to occupy its slot -until it exits. +## Worker scheduling -During shutdown, the server stops gRPC and waits for active executor work. +The async gRPC server accepts at most 16 concurrent RPCs. Synchronous config +validation, processor preparation, UTF-8 decoding, and engine processing run in +a dedicated four-thread executor guarded by a four-slot semaphore. -## Error mapping +This prevents synchronous engine work from blocking the gRPC event loop and +bounds active processing. -`PrivacyGuardError` classifies each cataloged failure as invalid input or -internal failure: +## Cancellation -| Error kind | gRPC status | -| --- | --- | -| `invalid_input` | `INVALID_ARGUMENT` | -| `internal` | `INTERNAL` | +Cancelling the async RPC does not stop Python code already running in a worker +thread. The service keeps the worker bridge alive and releases the semaphore +slot only when that worker exits. -This mapping applies to evaluation. `ValidateConfig` instead returns -`valid=false` with a content-safe reason. +Engines should pass the shared remaining timeout to interruptible collaborators. +A non-interruptible call continues to occupy its worker slot. -Unexpected failures become `unexpected_service_failure`. The service does not -return caught collaborator messages or exception chains. +Server shutdown stops gRPC, waits for active executor work, and closes the +middleware executor. -A gRPC failure is distinct from a successful policy deny. OpenShell applies the -middleware registration's failure behavior when an RPC fails. A policy deny is -a successful RPC result that explicitly stops the request. +## Error mapping -## Programmatic server lifecycle +| Privacy Guard error kind | gRPC status | +| --- | --- | +| Invalid input | `INVALID_ARGUMENT` | +| Internal failure | `INTERNAL` | -`PrivacyGuardServer` is the high-level programmatic API. It owns: +`ValidateConfig` returns `valid=false` with a content-safe reason instead of a +gRPC error for invalid policy configuration. -```text -EngineRegistry - -> PrivacyGuardMiddleware - -> gRPC server -``` +Unexpected failures map to the stable `unexpected_service_failure` error. +Collaborator exception messages and chains are not returned. + +A gRPC failure is different from a successful policy deny: -The built-in registry includes `RegexEngine`. Operators register custom engines -and resource-backed tool integrations before registry finalization, then pass -that registry to `PrivacyGuardServer`. +- OpenShell applies middleware `on_error` after a gRPC failure. +- OpenShell follows the explicit deny after `privacy_guard_blocked` or + `privacy_guard_limit_exceeded`. -The registry is an explicit application-scoped dependency, not a global -singleton. `PrivacyGuardServer` and `PrivacyGuardMiddleware` reject unfinalized -registries. A deployment creates and finalizes one registry during startup; -the middleware constructs configured stage engines and the active processor -from that registry. -Different middleware applications in the same process may intentionally use -different engine inventories or runtime resources. +## Server API -Synchronous applications use: +`PrivacyGuardServer` owns the finalized registry, middleware adapter, gRPC +server, and shutdown lifecycle: ```python from privacy_guard.engines.registry import create_builtin_registry @@ -241,106 +159,18 @@ server = PrivacyGuardServer( server.serve_sync("127.0.0.1:50051") ``` -Async applications call `await server.serve_async(address)` instead. The -explicit suffixes make the blocking behavior visible at each call site. Both -entry points use the same server instance and lifecycle. - -The listen address uses `host:port` or bracketed IPv6 `[address]:port` form, -with an explicit port from 1 through 65535. Privacy Guard rejects zero and -out-of-range numeric ports before calling gRPC, then verifies that gRPC bound -the requested port before starting the service. - -The server: - -1. creates an unstarted async gRPC server with receive and concurrency limits -2. binds the configured address -3. starts and waits for termination -4. stops gRPC -5. closes the middleware executor - -A bind or asynchronous gRPC startup `RuntimeError` becomes the stable -`server_bind_failed` error. Its catalog operation is the broader `server.start` -phase so the rendered failure does not mislabel a post-bind startup error. - -The server module has no command framework, module-import-string, discovery, -schema-rendering, or command-logging responsibilities. - -## Command-line application - -`privacy_guard.cli` owns the Typer application. The `--registry-factory` option -identifies a Python function in `module:function` form. At startup, the CLI -imports the module, calls the function once, and uses the returned finalized -`EngineRegistry` for the selected command: - -```bash -privacy-guard --registry-factory my_engines:create_registry engines -privacy-guard --registry-factory my_engines:create_registry configuration-schema -privacy-guard --registry-factory my_engines:create_registry serve -``` - -This is a deployment setting, not part of an OpenShell policy. The person -starting Privacy Guard chooses the function, and importing its module executes -Python code, so it must refer to code that the deployer controls. A policy -cannot select a factory or cause Privacy Guard to import a module. - -Applications that start `PrivacyGuardServer` from Python do not need this CLI -option. They create the registry normally and pass it to the server: - -```python -registry = create_registry() -PrivacyGuardServer(registry, timeout_seconds=5).serve_sync() -``` - -The CLI exposes: - -```bash -privacy-guard engines -privacy-guard configuration-schema -privacy-guard serve --listen 127.0.0.1:50051 --timeout-seconds 5 -``` - -Entity behavior comes from policy configuration, not server startup flags. -Engine implementations and operational resources come from the selected -application registry. The processing timeout is an operational server bound, -not policy-controlled behavior. - -## Upstream protocol work - -The intended large-catalog and discovery experience requires coordinated -OpenShell changes for: - -- explicit preparation and activation that accepts expanded configuration - larger than the current 64 KiB evaluation limit -- versioned policy references on evaluations -- finalized policy schema and engine discovery in the manifest -- a dedicated finding source field - -These are protocol evolution items, not local implementation hooks. Until they -land upstream, Privacy Guard continues to validate the per-evaluation config, -treat a changed valid config as an active-policy update, expose discovery -through the CLI, and render stage provenance inside the finding label. - -The protocol cannot distinguish an intentional update from any other changed -configuration. Each process must receive one consistent policy stream. -Interleaved old and new configurations can repeatedly switch the active policy. - -## Testing the boundary +Async applications use `await server.serve_async(address)`. -Service tests should cover: +The server rejects an unfinalized registry, invalid listen address, or port +outside 1 through 65535. Bind and startup failures use the stable +`server_bind_failed` error. -- protobuf/domain translation -- manifest fields supported by the current protocol -- pure policy validation -- phase, body-size, and UTF-8 validation -- first activation, equal-config reuse, serialized replacement, failed-update - preservation, and concurrent cutover -- detect, replacement, block, and limit serialization -- finding encoding and limits -- gRPC status mapping -- worker-slot behavior under cancellation -- startup, bind failure, and shutdown +See [Run and operate Privacy Guard](../operations.md) for CLI startup and +gateway registration. -Engine algorithms and ordered policy semantics belong in engine and processor -tests unless transport translation is essential to the case. +## Related pages -[Back to the architecture overview](index.md) +- [System architecture](index.md) +- [Request lifecycle](request-lifecycle.md) +- [Add a custom engine](../engines/custom.md) +- [Limits and failure behavior](../reference/limits-and-failures.md) diff --git a/docs/documentation/privacy-guard/configuration.md b/docs/documentation/privacy-guard/configuration.md new file mode 100644 index 0000000..393ac7d --- /dev/null +++ b/docs/documentation/privacy-guard/configuration.md @@ -0,0 +1,281 @@ +--- +title: Configure policies +description: Configure Privacy Guard stages, actions, catalogs, and OpenShell middleware routing. +agent_markdown: true +--- + +# Configure policies + +Privacy Guard configuration is embedded in an OpenShell +`network_middlewares` entry. The policy determines: + +- which provider endpoints use Privacy Guard +- the order of entity-processing stages +- each engine's exact configuration +- whether detections are reported, blocked, or replaced +- OpenShell's behavior when the middleware RPC fails + +Privacy Guard validates the complete configuration before processing a request. + +## Complete middleware entry + +```yaml +network_middlewares: + privacy_guard_replace: + name: Replace email addresses and customer IDs + middleware: privacy-guard + order: 0 + config: + entity_processing: + stages: + - name: identifiers + config: + engine: regex + pattern_catalog: + entities: + - name: email + rules: + - name: conventional-email + pattern: '(? stage 1 -> stage 2 -> final replacement text +``` + +Detection offsets belong to the input revision seen by the stage that produced +them. Findings aggregate by stage, entity, and confidence. + +## Detection actions + +Set `on_detection.action` to one of: + +| Action | Engine strategy | No detections | Detections | +| --- | --- | --- | --- | +| `detect` | `DETECT` | Allow original body | Allow original body and report findings | +| `block` | `DETECT` | Allow original body | Deny with `privacy_guard_blocked` | +| `replace` | `REPLACE` | Allow final stage output | Allow final stage output and report findings | + +`replace` requires every configured stage to support replacement and to satisfy +its engine-specific replacement requirements. A replacement recipe may remain +configured when the action is `detect` or `block`; it is not used in those +modes. + +## Common policy recipes + +### Detect without changing the request + +```yaml +entity_processing: + stages: + - name: identifiers + config: + engine: regex + pattern_catalog: patterns.yaml +on_detection: + action: detect +``` + +Use this to observe findings while leaving the provider-bound body unchanged. + +### Block requests containing configured entities + +```yaml +entity_processing: + stages: + - name: restricted-values + config: + engine: regex + pattern_catalog: patterns.yaml +on_detection: + action: block +``` + +The request is denied only when at least one configured entity is detected. + +### Replace entities + +```yaml +entity_processing: + stages: + - name: identifiers + config: + engine: regex + pattern_catalog: patterns.yaml + replacement: + strategy: template + template: "[{entity}]" +on_detection: + action: replace +``` + +`{entity}` is replaced with the catalog entity name. For example, +`user@example.com` becomes `[email]`. + +### Run multiple stages + +```yaml +entity_processing: + stages: + - name: structured-identifiers + config: + engine: regex + pattern_catalog: identifiers.yaml + replacement: + strategy: template + template: "[{entity}]" + - name: organization-model + config: + engine: acme-pii + model_profile: organization-default + replacement: + strategy: native +on_detection: + action: replace +``` + +The `acme-pii` engine and its configuration are examples of a custom +installation. The running registry must contain every engine named by the +policy. + +## Regex catalogs + +`RegexEngine` accepts an inline catalog or a relative YAML path. + +Inline: + +```yaml +pattern_catalog: + entities: + - name: customer-id + rules: + - name: prefixed-eight-digit-id + pattern: '\bCUST-[0-9]{8}\b' + confidence: high +``` + +File-backed: + +```yaml +pattern_catalog: patterns.yaml +``` + +Relative paths resolve beneath Privacy Guard's working directory. The path must +end in `.yaml` or `.yml`. Absolute paths, `..` traversal, and symlinks are +rejected. Start Privacy Guard from the directory that contains the referenced +catalog, or use a path relative to that directory. + +See [Use RegexEngine](engines/regex.md) for the complete catalog schema. + +## Inspect and validate configuration + +List the engines installed in the selected registry: + +```bash +uv run privacy-guard engines +``` + +Print the exact JSON Schema accepted by that registry: + +```bash +uv run privacy-guard configuration-schema +``` + +For a custom registry, pass the same factory to inspection and serving: + +```bash +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + engines + +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + configuration-schema + +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + serve +``` + +Sandbox creation calls `ValidateConfig`. A successful creation proves that the +middleware registration is reachable and that the supplied config matches the +running registry. + +## Policy and deployment ownership + +Keep privacy behavior in policy: + +- stage order +- entity definitions +- detection settings +- engine-specific replacement recipes +- final action + +Keep operational resources in the Privacy Guard deployment: + +- installed engine implementations +- model clients and SDK adapters +- endpoints and credentials +- approved model profiles +- processing timeout + +A policy cannot select a registry factory or import Python code. + +## Configuration activation + +OpenShell sends the complete configuration on each evaluation. Privacy Guard +validates it and compares the normalized immutable result with the active +configuration: + +- equal configuration reuses the active processor +- changed valid configuration is fully prepared, then atomically activated +- failed validation or preparation leaves the active processor unchanged and + fails the triggering evaluation + +Send one consistent configuration stream to each Privacy Guard process. +Interleaving configurations causes the active processor to switch between them. + +The transport configuration is limited to 64 KiB. File-backed Regex catalogs +carry only their relative path through the transport and are loaded by the +Privacy Guard process. + +## Next steps + +- [Use RegexEngine](engines/regex.md) +- [Add a custom engine](engines/custom.md) +- [Run and operate Privacy Guard](operations.md) +- [Limits and failure behavior](reference/limits-and-failures.md) diff --git a/docs/documentation/privacy-guard/engines/custom.md b/docs/documentation/privacy-guard/engines/custom.md new file mode 100644 index 0000000..18f99fa --- /dev/null +++ b/docs/documentation/privacy-guard/engines/custom.md @@ -0,0 +1,301 @@ +--- +title: Add a custom engine +description: Implement, register, run, and test a typed Privacy Guard entity-processing engine. +agent_markdown: true +--- + +# Add a custom engine + +A custom engine integrates another detector or replacement tool with Privacy +Guard. It receives one text string, an invocation strategy, a shared timeout, +and validated engine-specific configuration. It returns processed text and +bounded detections. + +Custom engine code runs inside the Privacy Guard process and can access request +text. Install only reviewed, trusted implementations. + +## Engine contract + +A custom engine defines: + +1. a concrete `EngineConfig` +2. optional typed `EngineResources` +3. supported invocation strategies +4. optional immutable initialization in `_initialize()` +5. request processing in `_run()` + +Do not override `__init__()` or the public `run()` method. The framework-owned +wrapper validates strategy support, timeouts, detection spans, detection +cardinality, output size, and mutation behavior. + +## Minimal detection engine + +```python +import re +from typing import Literal + +from pydantic import Field + +from privacy_guard.engines import ( + EngineConfig, + EntityDetection, + EntityProcessingEngine, + EntityProcessingStrategy, + TextProcessingResult, +) +from privacy_guard.timeout import Timeout + + +class KeywordEngineConfig(EngineConfig): + engine: Literal["keyword"] = "keyword" + keyword: str = Field(min_length=1, max_length=256) + + +class KeywordEngine(EntityProcessingEngine[KeywordEngineConfig]): + supported_strategies = frozenset( + {EntityProcessingStrategy.DETECT} + ) + + def _run( + self, + text: str, + *, + strategy: EntityProcessingStrategy, + timeout: Timeout, + ) -> TextProcessingResult: + matches = re.finditer(re.escape(self.config.keyword), text) + return TextProcessingResult.from_detections( + text=text, + detections=( + EntityDetection( + entity="keyword", + start=match.start(), + end=match.end(), + ) + for match in matches + ), + ) +``` + +`TextProcessingResult.from_detections()` stops consuming a lazy detection +stream when the per-stage limit is exceeded. The public engine wrapper remains +the enforcement boundary. + +## Configuration + +Each config class must: + +- subclass `EngineConfig` +- declare one literal `engine` discriminator +- use strict typed fields for all policy-owned behavior +- reject unknown fields through the shared base model +- keep sensitive values out of normal representations when applicable + +```python +class AcmeEngineConfig(EngineConfig): + engine: Literal["acme-pii"] = "acme-pii" + model_profile: str + replacement: AcmeReplacement | None = None +``` + +Privacy Guard adds the exact config type to the registry-built Pydantic +discriminated union. The policy object is passed unchanged to the engine. + +OpenShell transports numbers through protobuf `Struct`. Integer settings must +fit the safe range `-(2^53 - 1)` through `2^53 - 1`. + +## Operational resources + +Use `EngineResources` for deployment-owned clients, adapters, endpoints, +credential providers, or preloaded models: + +```python +from dataclasses import dataclass + +from privacy_guard.engines import EngineResources + + +@dataclass(frozen=True) +class AcmeResources(EngineResources): + client: AcmeClient + + +class AcmeEngine( + EntityProcessingEngine[AcmeEngineConfig, AcmeResources] +): + ... +``` + +Resources must: + +- contain operational dependencies, not policy behavior +- retain no request text or per-request state +- be safe for concurrent use +- be created before request processing + +A resource-free engine omits the second generic argument. + +## Supported strategies + +Declare the exact operations exposed by the engine: + +```python +supported_strategies = frozenset( + { + EntityProcessingStrategy.DETECT, + EntityProcessingStrategy.REPLACE, + } +) +``` + +`block` is not an engine strategy. The processor invokes `DETECT` and applies +the block decision after successful engine execution. + +Override `_validate_run_config()` when a strategy requires additional +configuration. For example, a replacement engine can require a replacement +recipe only when invoked with `REPLACE`. + +## Result requirements + +Return `TextProcessingResult` with: + +| Field | Requirement | +| --- | --- | +| `text` | Complete authoritative stage output | +| `detections` | Every bounded occurrence produced by the stage | + +Each `EntityDetection` provides: + +| Field | Requirement | +| --- | --- | +| `entity` | Stable declared identifier, never a value derived from request text | +| `start` | Inclusive Unicode code-point offset in stage input | +| `end` | Exclusive non-empty offset in stage input | +| `confidence` | Optional `low`, `medium`, or `high` | +| `metadata` | Optional bounded internal attribution | + +For `DETECT`, returned text must exactly equal input text. For `REPLACE`, text +may change only when the result contains at least one detection. Do not return +partial text or detections after a collaborator failure. + +## Timeouts + +One `Timeout` is shared across the complete stage pipeline. Pass its remaining +duration to APIs that accept a timeout: + +```python +result = client.process( + text, + timeout=timeout.remaining_seconds(), +) +``` + +Translate Python `TimeoutError` with the shared context manager: + +```python +with timeout.enforce(): + result = client.process( + text, + timeout=timeout.remaining_seconds(), + ) +``` + +Long-running local loops may call `timeout.raise_if_expired()`. Document and +bound operations that cannot be interrupted. + +## Concurrency + +One configured engine instance may process requests concurrently. Keep request +text, detections, counters, and temporary objects local to `_run()`. Treat +configuration and derived initialization state as immutable. Ensure injected +clients and resources support concurrent calls. + +## Errors and logging + +Translate expected collaborator failures into Privacy Guard's content-safe +engine exceptions. Do not include: + +- input or replacement text +- matched values or surrounding text +- credentials or endpoints +- raw exception messages +- model or SDK response bodies + +Stable engine and entity identifiers may appear in findings and diagnostic +logs when they satisfy shared validation. + +## Register the engine + +Create one application-scoped registry factory: + +```python +from privacy_guard.engines.registry import EngineRegistry + + +def create_registry() -> EngineRegistry: + registry = EngineRegistry(include_builtin_engines=True) + registry.register(KeywordEngine) + return registry.finalize() +``` + +Pass resources during registration when required: + +```python +registry.register( + AcmeEngine, + resources=AcmeResources(client=client), +) +``` + +Use `include_builtin_engines=True` to add the built-in `RegexEngine`. Omit it +when the registry should contain only explicitly registered custom engines. + +## Inspect and run the registry + +```bash +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + engines + +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + configuration-schema + +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + serve \ + --listen 0.0.0.0:50051 +``` + +The module must be installed or available on `PYTHONPATH`. The factory is +trusted deployment code and executes for each CLI invocation. + +The complete runnable example is in +[`projects/privacy-guard/examples/custom-engine`](https://github.com/NVIDIA/OpenShell-Research/tree/main/projects/privacy-guard/examples/custom-engine). + +## Verify the integration + +Before deploying a custom engine: + +1. Run its unit tests directly against `run()` for every supported strategy. + Assert the exact returned text, entity identifiers, spans, and confidence. + Include Unicode input when offsets come from another library. +2. Run `engines` and `configuration-schema` with the registry factory. Confirm + that the engine and its policy fields appear. +3. Send a representative request through a running Privacy Guard service with + the deployment policy. Confirm the OpenShell decision, replacement body, and + findings. +4. Force collaborator timeouts and failures. Confirm that the request fails + without partial output and that responses and logs contain no request text, + credentials, or raw collaborator errors. + +Test engine-specific behavior and integrations. Privacy Guard's own suite +covers the shared wrapper contract, processor ordering, request-wide limits, +and gRPC result mapping. + +## Related pages + +- [Configure policies](../configuration.md) +- [Run and operate Privacy Guard](../operations.md) +- [System architecture](../architecture/index.md) +- [Limits and failure behavior](../reference/limits-and-failures.md) diff --git a/docs/documentation/privacy-guard/engines/regex.md b/docs/documentation/privacy-guard/engines/regex.md new file mode 100644 index 0000000..5458035 --- /dev/null +++ b/docs/documentation/privacy-guard/engines/regex.md @@ -0,0 +1,215 @@ +--- +title: Use RegexEngine +description: Configure RegexEngine catalogs, matching flags, findings, and deterministic replacement. +agent_markdown: true +--- + +# Use RegexEngine + +`RegexEngine` is the built-in Privacy Guard engine. It detects every configured +regular-expression match and can replace a deterministic non-overlapping subset +with a constrained template. + +Privacy Guard provides the catalog schema and execution bounds. It does not +provide an authoritative pattern catalog. Define and test patterns for the data +your deployment handles. + +## Engine configuration + +```yaml +engine: regex +pattern_catalog: patterns.yaml +replacement: + strategy: template + template: "[{entity}]" +``` + +| Field | Required | Purpose | +| --- | --- | --- | +| `engine` | Yes | Must be `regex` | +| `pattern_catalog` | Yes | Inline catalog or relative YAML path | +| `replacement` | For `replace` actions | Template replacement configuration | + +## Catalog structure + +```yaml +entities: + - name: email + rules: + - name: conventional-email + pattern: '(? finalized Pydantic policy union (config.engine discriminator) - -> reuse the matching active RequestProcessor, or fully prepare and - atomically activate a replacement - -> strict UTF-8 decode - -> RequestProcessor.process(one text string) - -> stage 1 engine.run(current text) - -> stage 2 engine.run(stage 1 text) - -> ... - -> policy action: detect, block, or replace - -> safe aggregated entity findings - -> OpenShell HttpRequestResult -``` - -The policy action never crosses the engine boundary. Engines receive only -`EntityProcessingStrategy.DETECT` or `EntityProcessingStrategy.REPLACE`. -Blocking is a request-level disposition owned by `RequestProcessor`. +`entity_processing.stages` runs in order. In replace mode, each stage receives +the text returned by the preceding stage. Detect and block invoke engines with +the detection-only strategy. -The copied `proto/supervisor_middleware.proto` and generated bindings are owned -by OpenShell. Update them only through the repository's middleware-kit workflow; -never hand-edit them. Today's protocol carries a `google.protobuf.Struct` -configuration on each evaluation. Privacy Guard validates it every time and -retains one active configured processor. The first evaluation prepares that -processor. A different validated configuration is fully prepared and atomically -replaces it only after preparation succeeds; failure leaves the active processor -unchanged. - -The protocol has no explicit update or policy-version marker, so any changed -per-evaluation configuration is treated as an update. Each Privacy Guard process -must therefore receive one consistent policy stream. Large-catalog preparation -RPCs, versioned policy references, manifest schema fields, and a dedicated -finding-source field require a coordinated change in the canonical OpenShell -protocol rather than a private proto fork. - -## Built-in engines - -### RegexEngine - -`RegexEngine` compiles configured rules once and supports overlapping -detection and deterministic template replacement. It preserves numeric -backreferences by wrapping each rule's pattern in a non-capturing group followed -by a private named marker. Rule names are optional diagnostic identities; -`pattern` is the only field containing the regex string. Equivalent -normalized catalogs reuse a bounded compiled-rule cache across validation and -engine construction. - -The third-party `regex` backend provides enforceable per-search timeouts. -Explicit `ignore_case`, `multiline`, `dot_all`, and `ascii` flags are supported; -inline flags and user-defined named groups are rejected to protect the wrapper -contract. - -Privacy Guard owns the catalog schema but maintains no authoritative patterns. - -## Custom engines - -Custom engines are a first-class extension point. Authors declare one typed -config, optional typed resources, `supported_strategies`, and `_run`. They do not -write `__init__`; `_initialize` is optional, and `@override` is not required. -The public wrapper validates strategy support, input, timeout, spans, output -size, mutation behavior, and result cardinality for every returned result. -`TextProcessingResult.from_detections()` is an optional safe materializer for -lazy detection streams, not the enforcement boundary. - -Resource-backed engines define an `EngineResources` subclass containing their -operator-owned runtime dependencies. Resource bundles may contain initialized -tool clients, SDK adapters, models, endpoints, or credential providers, but -must contain no policy behavior or per-request state and must be safe for -concurrent use. Resource-free engines omit the second -`EntityProcessingEngine` generic argument entirely. - -The first NeMo Anonymizer integration will be implemented as a custom engine, -not as a built-in or placeholder abstraction in Privacy Guard. - -Application startup registers engines and operator-owned resources, then -returns one finalized registry: +`RegexEngine` accepts an inline catalog or a relative `.yaml` or `.yml` path: -```python -from privacy_guard.engines.registry import EngineRegistry - - -def create_registry() -> EngineRegistry: - registry = EngineRegistry(include_builtin_engines=True) - registry.register(AcmeEngine, resources=AcmeResources(client=client)) - return registry.finalize() +```yaml +pattern_catalog: patterns.yaml ``` -Set `include_builtin_engines=True` when a custom deployment should extend the -standard engine inventory. Leave it at the default `False` for an intentionally -isolated registry. +Relative paths resolve beneath Privacy Guard's working directory. Absolute +paths, traversal, symlinks, unsafe YAML tags, aliases, duplicate keys, invalid +UTF-8, and oversized catalogs are rejected. -Pass that factory to every CLI operation so discovery, schema generation, and -the running server use the same engine inventory. The module must be installed -or otherwise present on Python's import path: +## CLI ```bash -uv run privacy-guard --registry-factory my_engines:create_registry engines -uv run privacy-guard --registry-factory my_engines:create_registry configuration-schema -uv run privacy-guard --registry-factory my_engines:create_registry serve +uv run privacy-guard engines +uv run privacy-guard configuration-schema +uv run privacy-guard configure-gateway --host-ip YOUR_HOST_IPV4 +uv run privacy-guard serve \ + --listen 0.0.0.0:50051 \ + --timeout-seconds 5 ``` -The -[custom engine end-to-end example](https://github.com/NVIDIA/OpenShell-Research/blob/main/projects/privacy-guard/examples/custom-engine/README.md) -contains one Python file with a typed detection config, engine implementation, -and registry factory, plus its OpenShell policy and walkthrough. It also shows -the explicit `PYTHONPATH` setup needed when the factory is a standalone local -module rather than an installed package. +`configure-gateway` adds or updates a Privacy Guard registration in the +OpenShell gateway TOML. Its registration name must match the policy's +`middleware` field. Restart the gateway after changing registrations. -The registry is application-scoped, not a process-global singleton. A -`PrivacyGuardServer` requires an explicit finalized registry. The finalized -registry builds a Pydantic discriminated union containing the exact config type -of every registered engine, so `stage.config` round-trips without dropping -engine-specific or replacement-variant fields. +The processing timeout is one bound shared by every stage. It defaults to 1 +second and cannot exceed 30 seconds. Configure a longer OpenShell middleware +timeout to include worker queueing, configuration validation, and processor +preparation. -The base installation has an explicit built-in registry containing -`RegexEngine`: +Use a trusted registry factory for custom engines: -```python -from privacy_guard.engines.registry import create_builtin_registry +```bash +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + engines -registry = create_builtin_registry() +uv run privacy-guard \ + --registry-factory my_engines:create_registry \ + serve ``` -## Programmatic server - -The server is a library API independent of the command-line application: +## Python server API ```python from privacy_guard.engines.registry import create_builtin_registry @@ -187,117 +126,40 @@ server = PrivacyGuardServer( server.serve_sync("127.0.0.1:50051") ``` -`serve_sync()` is the blocking entry point. Async applications use the -explicitly named asynchronous counterpart: +Async applications use: ```python await server.serve_async("127.0.0.1:50051") ``` -Listen addresses use `host:port` or bracketed IPv6 `[address]:port` form. The -port must be an explicit integer from 1 through 65535. - -Custom applications pass their own finalized registry to -`PrivacyGuardServer`. The server owns the middleware adapter, bounded gRPC -transport, worker executor, and shutdown lifecycle; it does not load Python -modules, select engines, generate schemas, or parse command-line options. - -Privacy Guard emits records through Python's standard `privacy_guard` logger -and does not configure logging when imported. Applications may use their own -standard-library logging setup or opt into Privacy Guard's concise console -format: - -```python -from privacy_guard.logging import ColorMode, LoggingConfig, configure_logging - -configure_logging(LoggingConfig(level="DEBUG", color_mode=ColorMode.AUTO)) -``` +## Documentation -Repeated calls replace only the handler installed by `configure_logging()`; -application-owned handlers are preserved. The CLI uses this same configuration -path. The default format includes the date, time, padded level, logger name, -and message. It adds level-aware ANSI colors when writing to an interactive -terminal and stays plain when redirected. Set `color_mode=ColorMode.ALWAYS` or -`color_mode=ColorMode.NEVER` for an explicit choice instead of the default -`ColorMode.AUTO`. Its default level is `INFO`, and `--debug` enables -content-safe `DEBUG` diagnostics. `--debug-log-content` additionally logs -complete request and processed text and should be enabled only in a controlled -environment. +- [Overview and end-to-end quickstart](../../docs/documentation/privacy-guard/index.md) +- [Configure policies](../../docs/documentation/privacy-guard/configuration.md) +- [Run and operate Privacy Guard](../../docs/documentation/privacy-guard/operations.md) +- [Use RegexEngine](../../docs/documentation/privacy-guard/engines/regex.md) +- [Add a custom engine](../../docs/documentation/privacy-guard/engines/custom.md) +- [System architecture](../../docs/documentation/privacy-guard/architecture/index.md) +- [Limits and failure behavior](../../docs/documentation/privacy-guard/reference/limits-and-failures.md) -## CLI +## Runnable examples -```bash -uv run privacy-guard engines -uv run privacy-guard configuration-schema -uv run privacy-guard configure-gateway --host-ip YOUR_HOST_IPV4 -uv run privacy-guard serve \ - --listen 127.0.0.1:50051 \ - --timeout-seconds 5 -``` +- [`examples/regex-engine`](examples/regex-engine/README.md): detect and replace + email addresses and customer IDs with the built-in engine. +- [`examples/custom-engine`](examples/custom-engine/README.md): implement, + register, and run a typed custom engine. -Replace `YOUR_HOST_IPV4` with the non-loopback IPv4 address that the OpenShell -gateway and sandbox supervisors use to reach this host. `configure-gateway` -adds or updates a registration named `privacy-guard` in the path set by -OpenShell's `OPENSHELL_GATEWAY_CONFIG` override, or OpenShell's standard -per-user gateway config location when that variable is unset: -`$XDG_CONFIG_HOME/openshell/gateway.toml` (normally -`~/.config/openshell/gateway.toml`). Privacy Guard does not guess which host -interface is correct. Use `--name` when the policy references a different -registration name, `--port` when Privacy Guard does not listen on 50051, and -`--config` for a nonstandard gateway TOML. OpenShell middleware registration -names must use 1–128 ASCII bytes containing only letters, digits, `.`, `_`, `-`, -or `/`; the `openshell/` prefix is reserved. The command preserves unrelated -settings and updates the named registration when it already exists. Start -Privacy Guard before restarting the gateway, because OpenShell connects to -registered middleware during startup. - -Entity behavior is supplied by OpenShell policy config, not server startup -flags. Deployment startup owns only installed engine implementations and -operator resources such as model profiles, endpoints, clients, and credentials. -The processing timeout is an operational bound shared by every stage; it -defaults to 1 second and may be increased to at most 30 seconds. When increasing -it, configure OpenShell's middleware `timeout` with additional headroom for -worker queueing and configuration or engine preparation so the supervisor does -not end the RPC first. Use `--registry-factory module:factory` for a custom -engine installation. -Registry factories execute operator Python code; load only trusted modules. -The `privacy_guard.cli` module owns the executable application and adapts its -`serve` command to the programmatic server API. - -## Safety and limits - -- Input and replacement bodies are limited to 4 MiB. -- One monotonic `Timeout` is shared across every stage and result validation. -- Regex searches receive the remaining timeout and fail atomically. -- Intermediate text and detection cardinality are bounded. -- Detect and block never return a body mutation; replace returns final text. -- Findings expose entity, bounded confidence, count, and stage provenance. - Framework-controlled fields and compliant custom engines never expose matched - or surrounding text, offsets, patterns, or raw tool metadata. Custom engines - must return stable, declared identifiers rather than request-derived values. -- Engine instances and injected resources must be safe for concurrent requests. -- Cross-request entity memory is intentionally out of scope. - -## Updating the OpenShell protocol - -Privacy Guard uses -[`openshell-middleware-kit`](https://github.com/NVIDIA/OpenShell-Research/blob/main/projects/openshell-middleware-kit/README.md) -to keep its copied protocol and generated Python bindings aligned with an -OpenShell release. -Install the repository's local `omkit`, then update: +## Logging -```bash -uv tool install --force ../openshell-middleware-kit -omkit update --openshell-version v0.0.90 -``` +`--debug` enables content-safe diagnostic records. -The updater replaces only the copied protocol, generated bindings, lockfile, and -`.openshell-middleware-manifest.json` from a validated temporary copy. Review -those generated changes and run `make check`. +`--debug-log-content` logs complete input and processed text. Use it only in a +controlled development environment. -## Development validation +Imported applications can configure the standard `privacy_guard` logger +themselves or use `privacy_guard.logging.configure_logging()`. -The project Makefile exposes the normal workflow: +## Development ```bash make help @@ -307,6 +169,10 @@ make check make check-py311 ``` -`make check` delegates to `scripts/check.sh`, the authoritative local and CI -gate. It runs tests, formatting, lint, `ty`, an import smoke check, and a -dependency audit. +`make check` runs tests, formatting, lint, type checking, an import smoke check, +and a dependency audit. + +The copied `proto/supervisor_middleware.proto` and generated bindings are owned +by OpenShell. Update them through the repository's +[`openshell-middleware-kit`](../openshell-middleware-kit/README.md), then run +`make check`. diff --git a/projects/privacy-guard/docs/index.md b/projects/privacy-guard/docs/index.md index f006a10..d6cebcb 100644 --- a/projects/privacy-guard/docs/index.md +++ b/projects/privacy-guard/docs/index.md @@ -1,6 +1,8 @@ # Privacy Guard documentation -Technical documentation for people and coding agents working on Privacy Guard. +Start with the repository documentation: -- [Architecture](architecture/index.md): component boundaries, request flow, - extension interfaces, concurrency, failures, and limits +- [Privacy Guard overview and quickstart](../../../docs/documentation/privacy-guard/index.md) +- [Configure policies](../../../docs/documentation/privacy-guard/configuration.md) +- [Run and operate Privacy Guard](../../../docs/documentation/privacy-guard/operations.md) +- [System architecture](../../../docs/documentation/privacy-guard/architecture/index.md) diff --git a/zensical.toml b/zensical.toml index b6a4ac8..e3dea9e 100644 --- a/zensical.toml +++ b/zensical.toml @@ -9,7 +9,10 @@ copyright = """ """ docs_dir = "docs" site_dir = "site" -extra_css = ["stylesheets/dev-notes.css"] +extra_css = [ + "stylesheets/dev-notes.css", + "stylesheets/documentation.css" +] extra_javascript = ["javascripts/navigation-drawer.js"] nav = [ {"Dev Notes" = [ @@ -23,13 +26,22 @@ nav = [ {"Documentation" = [ "documentation/index.md", {"Privacy Guard" = [ + "documentation/privacy-guard/index.md", + {"Guides" = [ + {"Configure policies" = "documentation/privacy-guard/configuration.md"}, + {"Run and operate Privacy Guard" = "documentation/privacy-guard/operations.md"} + ]}, + {"Engines" = [ + {"Use RegexEngine" = "documentation/privacy-guard/engines/regex.md"}, + {"Add a custom engine" = "documentation/privacy-guard/engines/custom.md"} + ]}, {"Architecture" = [ "documentation/privacy-guard/architecture/index.md", {"Request lifecycle" = "documentation/privacy-guard/architecture/request-lifecycle.md"}, - {"Entity-processing engines" = "documentation/privacy-guard/architecture/entity-processing-engines.md"}, - {"Configuration and text boundary" = "documentation/privacy-guard/architecture/configuration.md"}, - {"Service boundary" = "documentation/privacy-guard/architecture/service-boundary.md"}, - {"Safety and limits" = "documentation/privacy-guard/architecture/safety-and-limits.md"} + {"Service boundary" = "documentation/privacy-guard/architecture/service-boundary.md"} + ]}, + {"Reference" = [ + {"Limits and failure behavior" = "documentation/privacy-guard/reference/limits-and-failures.md"} ]} ]} ]} From e7460873ffa97bbe76bb3f952962e8f1f7adc2fb Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Tue, 28 Jul 2026 16:52:50 -0400 Subject: [PATCH 2/8] Publish documentation on main --- .github/workflows/docs.yml | 6 ++++-- docs/development/index.md | 16 ++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2a40601..2f98ced 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -59,7 +59,8 @@ jobs: - name: Upload production documentation if: >- github.ref == 'refs/heads/main' && - github.event_name == 'workflow_dispatch' + (github.event_name == 'push' || + github.event_name == 'workflow_dispatch') uses: actions/upload-artifact@v7 with: name: production-docs-${{ github.run_id }} @@ -71,7 +72,8 @@ jobs: name: Deploy documentation if: >- github.ref == 'refs/heads/main' && - github.event_name == 'workflow_dispatch' + (github.event_name == 'push' || + github.event_name == 'workflow_dispatch') needs: validate runs-on: ubuntu-latest permissions: diff --git a/docs/development/index.md b/docs/development/index.md index 0d29668..070eb0d 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -110,11 +110,11 @@ previews on the production documentation origin. The `gh-pages` branch stores the composite production site and active previews; GitHub Pages remains configured with **GitHub Actions** as its publishing -source. Pushes to `main` validate documentation without publishing it. When the -current documentation is ready for production, manually run the **Docs** -workflow from `main`; that run updates the production site while preserving -active previews, then deploys the complete branch through the official Pages -artifact workflow. The first production deployment creates `gh-pages` -automatically. To roll back to a revision before preview support, leave the -Pages source set to **GitHub Actions** and rerun the restored documentation -workflow. +source. Every successful push to `main` updates the production site while +preserving active previews, then deploys the complete branch through the +official Pages artifact workflow. Merging a pull request therefore publishes +its documentation after the **Docs** workflow passes. A manual **Docs** +workflow dispatch from `main` can republish the current revision without a new +commit. The first production deployment creates `gh-pages` automatically. To +roll back to a revision before preview support, leave the Pages source set to +**GitHub Actions** and rerun the restored documentation workflow. From b5568fec46f6af439052ce5ca22cba0bf60374c0 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Tue, 28 Jul 2026 17:05:01 -0400 Subject: [PATCH 3/8] Co-locate Privacy Guard documentation --- .github/workflows/docs-preview-deploy.yml | 6 +- .github/workflows/docs-preview.yml | 6 +- .gitignore | 1 + AGENTS.md | 6 +- docs/development/index.md | 52 +++-- docs/documentation/privacy-guard/index.md | 217 ----------------- docs/stylesheets/documentation.css | 4 +- projects/privacy-guard/AGENTS.md | 4 +- projects/privacy-guard/README.md | 14 +- projects/privacy-guard/analysis/README.md | 4 +- .../privacy-guard-latency-vs-prompt-size.svg | 162 ------------- .../analysis/render_latency_plot.py | 8 +- projects/privacy-guard/docs/architecture | 1 - .../privacy-guard/docs}/architecture/index.md | 4 +- .../docs}/architecture/request-lifecycle.md | 2 +- .../docs}/architecture/service-boundary.md | 0 .../privacy-guard-latency-vs-prompt-size.svg | 0 .../diagrams/component-architecture.svg | 0 .../assets}/diagrams/processing-pipeline.svg | 0 .../assets}/diagrams/request-lifecycle.svg | 0 .../docs/assets}/diagrams/request-path.svg | 0 .../privacy-guard/docs}/configuration.md | 0 .../privacy-guard/docs}/engines/custom.md | 0 .../privacy-guard/docs}/engines/regex.md | 0 projects/privacy-guard/docs/index.md | 221 +++++++++++++++++- .../privacy-guard/docs}/operations.md | 0 .../docs}/reference/limits-and-failures.md | 2 +- scripts/build-docs.sh | 1 + scripts/stage-privacy-guard-docs.py | 63 +++++ tests/test_stage_privacy_guard_docs.py | 60 +++++ 30 files changed, 408 insertions(+), 430 deletions(-) delete mode 100644 docs/documentation/privacy-guard/index.md delete mode 100644 projects/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg delete mode 120000 projects/privacy-guard/docs/architecture rename {docs/documentation/privacy-guard => projects/privacy-guard/docs}/architecture/index.md (97%) rename {docs/documentation/privacy-guard => projects/privacy-guard/docs}/architecture/request-lifecycle.md (98%) rename {docs/documentation/privacy-guard => projects/privacy-guard/docs}/architecture/service-boundary.md (100%) rename {docs/assets/privacy-guard => projects/privacy-guard/docs/assets}/analysis/privacy-guard-latency-vs-prompt-size.svg (100%) rename {docs/assets/privacy-guard => projects/privacy-guard/docs/assets}/diagrams/component-architecture.svg (100%) rename {docs/assets/privacy-guard => projects/privacy-guard/docs/assets}/diagrams/processing-pipeline.svg (100%) rename {docs/assets/privacy-guard => projects/privacy-guard/docs/assets}/diagrams/request-lifecycle.svg (100%) rename {docs/assets/privacy-guard => projects/privacy-guard/docs/assets}/diagrams/request-path.svg (100%) rename {docs/documentation/privacy-guard => projects/privacy-guard/docs}/configuration.md (100%) rename {docs/documentation/privacy-guard => projects/privacy-guard/docs}/engines/custom.md (100%) rename {docs/documentation/privacy-guard => projects/privacy-guard/docs}/engines/regex.md (100%) rename {docs/documentation/privacy-guard => projects/privacy-guard/docs}/operations.md (100%) rename {docs/documentation/privacy-guard => projects/privacy-guard/docs}/reference/limits-and-failures.md (98%) create mode 100644 scripts/stage-privacy-guard-docs.py create mode 100644 tests/test_stage_privacy_guard_docs.py diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 91d56ef..ef587cf 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -99,14 +99,18 @@ jobs: 'scripts/build-docs.sh', 'scripts/publish-agent-markdown.py', 'scripts/render-dev-notes.py', + 'scripts/stage-privacy-guard-docs.py', 'tests/test_agent_markdown.py', 'tests/test_docs_404.py', 'tests/test_render_dev_notes.py', + 'tests/test_stage_privacy_guard_docs.py', 'zensical.toml', ]); const docsChanged = files.some( ({ filename }) => filename.startsWith('docs/') || - filename.startsWith('overrides/') || exactInputs.has(filename), + filename.startsWith('overrides/') || + filename.startsWith('projects/privacy-guard/docs/') || + exactInputs.has(filename), ); operation = docsChanged ? 'deploy' : 'remove'; } diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index dcc1ce2..6f54a22 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -47,14 +47,18 @@ jobs: 'scripts/build-docs.sh', 'scripts/publish-agent-markdown.py', 'scripts/render-dev-notes.py', + 'scripts/stage-privacy-guard-docs.py', 'tests/test_agent_markdown.py', 'tests/test_docs_404.py', 'tests/test_render_dev_notes.py', + 'tests/test_stage_privacy_guard_docs.py', 'zensical.toml', ]); const docsChanged = files.some( ({ filename }) => filename.startsWith('docs/') || - filename.startsWith('overrides/') || exactInputs.has(filename), + filename.startsWith('overrides/') || + filename.startsWith('projects/privacy-guard/docs/') || + exactInputs.has(filename), ); core.setOutput('operation', docsChanged ? 'deploy' : 'remove'); diff --git a/.gitignore b/.gitignore index 7d964c6..ee6c962 100644 --- a/.gitignore +++ b/.gitignore @@ -121,6 +121,7 @@ lib/ # Static site and docs output public/ site/ +docs/documentation/privacy-guard/ .docusaurus/ .vitepress/cache/ .vitepress/dist/ diff --git a/AGENTS.md b/AGENTS.md index 03de54f..85d4fe9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,8 @@ documentation for work that uses OpenShell as its runtime. ## Work routing - Put self-contained implementations and experiments in `projects//`. -- Put durable user-facing guides and references in `docs/documentation/`. +- Put project-specific user guides and references in `projects//docs/`. +- Put cross-project user-facing documentation in `docs/documentation/`. - Put Dev Notes (human-written technical notes) `docs/dev-notes/`. - Put agent-facing repository maintenance workflows in `docs/development/`. @@ -32,6 +33,9 @@ read `docs/development/index.md`. `.env.example`. - Do not hand-edit generated Dev Notes cards, bylines, or navigation; use `python3 scripts/render-dev-notes.py`. +- Do not hand-edit generated project-documentation mirrors under + `docs/documentation/`; use the project's canonical `docs/` tree and its + documented staging script. - When adding a project or workflow with distinct conventions, add a nested `AGENTS.md` in that directory instead of expanding this root file. diff --git a/docs/development/index.md b/docs/development/index.md index 070eb0d..ceef7e5 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -5,35 +5,39 @@ description: Agent instructions for maintaining the OpenShell Research documenta # Documentation site development -Follow these instructions for changes under `docs/`, `zensical.toml`, the Dev -Notes renderer, or the documentation workflow. Run commands from the repository -root. Use Python 3.10 or newer. +Follow these instructions for changes under `docs/`, a project's `docs/` tree, +`zensical.toml`, the Dev Notes renderer, or the documentation workflow. Run +commands from the repository root. Use Python 3.10 or newer. ## Content routing -- Put durable software knowledge—installation, usage, reproducibility, and known - limitations—under `docs/documentation/`. +- Put project-specific software knowledge—installation, usage, reproducibility, + and known limitations—under `projects//docs/`. The clean build stages + configured project documentation under `docs/documentation/` for publishing. +- Put cross-project software documentation directly under + `docs/documentation/`. - Put dated experiments, benchmarks, releases, use cases, and engineering updates under `docs/dev-notes/`. -- When a Dev Note introduces reusable software, add its durable guide under - `docs/documentation/`, link it to the originating Dev Note, and add it to - `zensical.toml`. +- When a Dev Note introduces reusable software, add its durable guide to the + owning project, link it to the originating Dev Note, and add its staged site + path to `zensical.toml`. ## Agent-readable Markdown -Every canonical content page under `docs/dev-notes/posts/` and -`docs/documentation/` must declare `agent_markdown: true` in its front matter. -The clean site build copies those sources byte-for-byte into `site/` at the same -path relative to `docs/`, and each rendered page links to its same-origin -Markdown source for people and agents. The generated copies under `site/` must -not be edited. - -Every page in the two canonical content directories is published, including -the Documentation index. Keep presentation-only landing pages such as the -homepage and Dev Notes card index, redirect-only pages, obsolete or orphan -project pages, internal development documentation, and the 404 page outside -those directories and do not add the marker to them. Those pages are not -canonical content for agent consumption. +Every canonical content page under `docs/dev-notes/posts/`, +`docs/documentation/`, and a staged project documentation tree must declare +`agent_markdown: true` in its front matter. The clean site build first stages +project documentation, then copies all published Markdown sources byte-for-byte +into `site/` at their site paths. Each rendered page links to its same-origin +Markdown source for people and agents. Generated copies under +`docs/documentation/` and `site/` must not be edited. + +Every page in a published content tree is included, including the Documentation +index. Keep presentation-only landing pages such as the homepage and Dev Notes +card index, redirect-only pages, obsolete or orphan project pages, internal +development documentation, and the 404 page outside those trees and do not add +the marker to them. Those pages are not canonical content for agent +consumption. ## Dev Notes @@ -88,8 +92,10 @@ scripts/build-docs.sh ``` `scripts/build-docs.sh` recreates `.venv-docs`, installs the pinned toolchain, -renders Dev Notes metadata, and runs `zensical build --clean --strict`. Do not -report success unless it completes without issues. +stages canonical Privacy Guard documentation from +`projects/privacy-guard/docs/`, renders Dev Notes metadata, and runs +`zensical build --clean --strict`. Do not report success unless it completes +without issues. For documentation-site changes, serve the complete built artifact before handing the task back: diff --git a/docs/documentation/privacy-guard/index.md b/docs/documentation/privacy-guard/index.md deleted file mode 100644 index 225ee04..0000000 --- a/docs/documentation/privacy-guard/index.md +++ /dev/null @@ -1,217 +0,0 @@ ---- -title: Privacy Guard -description: Overview and quickstart for running Privacy Guard with OpenShell. -agent_markdown: true ---- - -# Privacy Guard - -Privacy Guard is an OpenShell supervisor middleware. It examines provider-bound -HTTP request bodies before OpenShell attaches provider credentials. A policy can -instruct Privacy Guard to: - -| Action | Result when entities are detected | -| --- | --- | -| `detect` | Allow the original body and report bounded findings | -| `block` | Deny the request and report bounded findings | -| `replace` | Allow a body produced by the configured replacement engines and report bounded findings | - -Privacy Guard processes the complete request body as UTF-8 text. It does not -parse JSON fields, inspect files in the sandbox, modify provider responses, or -send network requests to the provider. - -## How it fits into OpenShell - -![A provider-bound request travels from the sandbox through OpenShell and -Privacy Guard. Privacy Guard returns an allow, replacement, or deny decision -before OpenShell attaches credentials and sends an allowed request to the -provider.](../../assets/privacy-guard/diagrams/request-path.svg) - -Privacy Guard exposes OpenShell's `SupervisorMiddleware` gRPC service. It -registers for the pre-credentials request phase so replacement happens before -the provider receives the request. - -## Quickstart - -This quickstart uses the built-in `RegexEngine` example to replace an email -address and customer ID. It requires: - -- Python 3.11 or newer -- `uv` 0.11 or newer -- OpenShell and `openshell-gateway` at the version recorded by the Privacy - Guard project -- a Docker or Podman backend supported by OpenShell - -Run the commands from a checkout of OpenShell Research. - -### 1. Prepare the example - -```bash -cd projects/privacy-guard/examples/regex-engine -uv sync --locked -uv run privacy-guard engines -``` - -The engine list should contain: - -```text -regex detect,replace -``` - -The example files are: - -| File | Purpose | -| --- | --- | -| `patterns.yaml` | Email and customer-ID rules | -| `privacy-guard-config.yaml` | Standalone Privacy Guard policy configuration | -| `policy.yaml` | Complete OpenShell sandbox policy with the same configuration | - -### 2. Start Privacy Guard - -From the example directory: - -```bash -uv run privacy-guard serve --listen 0.0.0.0:50051 -``` - -Keep the process running. The development server uses plaintext gRPC and -receives request bodies. Restrict port 50051 to the host and trusted sandbox -network. - -### 3. Register Privacy Guard with the gateway - -Choose a non-loopback host IPv4 address that both the gateway and sandbox -supervisor can reach: - -```bash -# macOS -ipconfig getifaddr en0 - -# Linux -hostname -I -``` - -Create a local gateway configuration: - -```bash -export PRIVACY_GUARD_HOST_IP=YOUR_HOST_IPV4 - -uv run privacy-guard configure-gateway \ - --host-ip "$PRIVACY_GUARD_HOST_IP" \ - --name privacy-guard-regex \ - --config gateway.local.toml -``` - -Do not use `127.0.0.1`: loopback inside the sandbox supervisor does not refer to -the host. - -Restart the local gateway with `gateway.local.toml`. For a Homebrew -installation: - -```bash -brew services stop openshell -export OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" -openshell-gateway --config "$PWD/gateway.local.toml" -``` - -For a Debian or RPM installation: - -```bash -systemctl --user stop openshell-gateway -export OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/tls" -openshell-gateway --config "$PWD/gateway.local.toml" -``` - -Keep the foreground gateway running. - -### 4. Create a sandbox - -In another terminal, return to the example directory and verify the gateway: - -```bash -openshell gateway select openshell -openshell status -``` - -Create the example sandbox: - -```bash -openshell sandbox create \ - --name privacy-guard-regex \ - --from base \ - --no-auto-providers \ - --policy "$PWD/policy.yaml" \ - -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude -``` - -After authenticating Claude Code, submit: - -```text -Draft a short greeting for user@example.com about customer CUST-12345678. -``` - -The provider-bound body should contain `[email]` and `[customer-id]` instead of -the original values. - -### 5. Verify the result - -From a host terminal: - -```bash -openshell logs privacy-guard-regex -n 100 --source sandbox -``` - -Find the provider request and confirm: - -- `transformed:true` -- an `email (identifiers)` finding -- a `customer-id (identifiers)` finding -- no matched email address or customer ID in the findings - -See [Run and operate Privacy Guard](operations.md) for cleanup, gateway -configuration, logging, timeouts, and troubleshooting. - -## Core concepts - -### Policies control behavior - -The OpenShell policy supplies the ordered engine stages and final action. -Privacy Guard startup selects which engine implementations and operational -resources are installed. - -### Stages run in order - -Each stage receives the current text. In `replace` mode, later stages receive -the text returned by earlier stages. In `detect` and `block` mode, engines must -return the input text unchanged. - -### Findings do not contain matched values - -Findings contain the entity name, stage, confidence, and occurrence count. -Matched text, surrounding text, offsets, regex patterns, headers, and request -bodies do not cross the service boundary. - -### Processing is bounded - -Request size, output size, detections, regex execution, concurrency, and -processing time have explicit limits. A limit failure denies the request -without returning partial replacement text or partial findings. - -## Documentation map - -- [Configure policies](configuration.md): stages, actions, catalogs, and policy - recipes. -- [Run and operate Privacy Guard](operations.md): CLI, gateway registration, - server lifecycle, logging, and troubleshooting. -- [Use RegexEngine](engines/regex.md): pattern catalogs, flags, matching, and - replacement. -- [Add a custom engine](engines/custom.md): engine contract, resources, - registry factories, and tests. -- [System architecture](architecture/index.md): components, trust boundaries, - state, and concurrency. -- [Request lifecycle](architecture/request-lifecycle.md): validation, - activation, processing, and result mapping. -- [Service boundary](architecture/service-boundary.md): gRPC methods, - protobuf translation, and worker scheduling. -- [Limits and failure behavior](reference/limits-and-failures.md): enforced - bounds, error outcomes, retention, and measured latency. diff --git a/docs/stylesheets/documentation.css b/docs/stylesheets/documentation.css index 89334e8..3348bcc 100644 --- a/docs/stylesheets/documentation.css +++ b/docs/stylesheets/documentation.css @@ -5,7 +5,7 @@ * Keep documentation diagrams subordinate to the surrounding explanation. * Small screens retain the full available width so labels remain legible. */ -.md-typeset img[src*="assets/privacy-guard/diagrams/"] { +.md-typeset img[src*="assets/diagrams/"] { display: block; width: 88%; max-width: 52rem; @@ -15,7 +15,7 @@ } @media screen and (max-width: 48rem) { - .md-typeset img[src*="assets/privacy-guard/diagrams/"] { + .md-typeset img[src*="assets/diagrams/"] { width: 100%; } } diff --git a/projects/privacy-guard/AGENTS.md b/projects/privacy-guard/AGENTS.md index c48302a..d84a913 100644 --- a/projects/privacy-guard/AGENTS.md +++ b/projects/privacy-guard/AGENTS.md @@ -42,8 +42,8 @@ Run focused tests while working and `make check` before handoff. - `src/privacy_guard/string_validators.py`: shared string validators and field types - `src/privacy_guard/service/`: gRPC lifecycle and protobuf adapter - `src/privacy_guard/bindings/`: generated protobuf files; never hand-edit -- [`docs/architecture/`](docs/architecture/index.md): symlink to the canonical - site sources under `../../docs/documentation/privacy-guard/architecture/` +- `docs/`: canonical user and architecture documentation; the repository docs + build stages this tree at the public Privacy Guard documentation route - `tests/`: tests that mirror source boundaries - `examples/`: copyable policy-authoring examples diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index be5bd08..21601cd 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -134,13 +134,13 @@ await server.serve_async("127.0.0.1:50051") ## Documentation -- [Overview and end-to-end quickstart](../../docs/documentation/privacy-guard/index.md) -- [Configure policies](../../docs/documentation/privacy-guard/configuration.md) -- [Run and operate Privacy Guard](../../docs/documentation/privacy-guard/operations.md) -- [Use RegexEngine](../../docs/documentation/privacy-guard/engines/regex.md) -- [Add a custom engine](../../docs/documentation/privacy-guard/engines/custom.md) -- [System architecture](../../docs/documentation/privacy-guard/architecture/index.md) -- [Limits and failure behavior](../../docs/documentation/privacy-guard/reference/limits-and-failures.md) +- [Overview and end-to-end quickstart](docs/index.md) +- [Configure policies](docs/configuration.md) +- [Run and operate Privacy Guard](docs/operations.md) +- [Use RegexEngine](docs/engines/regex.md) +- [Add a custom engine](docs/engines/custom.md) +- [System architecture](docs/architecture/index.md) +- [Limits and failure behavior](docs/reference/limits-and-failures.md) ## Runnable examples diff --git a/projects/privacy-guard/analysis/README.md b/projects/privacy-guard/analysis/README.md index ddeebc6..0e6d176 100644 --- a/projects/privacy-guard/analysis/README.md +++ b/projects/privacy-guard/analysis/README.md @@ -13,10 +13,10 @@ Run from `projects/privacy-guard/`: uv run python analysis/render_latency_plot.py ``` -The command writes: +The command writes the documentation asset: ```text -analysis/privacy-guard-latency-vs-prompt-size.svg +docs/assets/analysis/privacy-guard-latency-vs-prompt-size.svg ``` Verify that the committed figure matches the data and renderer: diff --git a/projects/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg b/projects/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg deleted file mode 100644 index e6b1866..0000000 --- a/projects/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg +++ /dev/null @@ -1,162 +0,0 @@ - -Privacy Guard latency versus prompt size -Scatter plot of 96 Privacy Guard service latency measurements from 18 thousand to 1.141 million prompt tokens, with one linear fit and a one-million-token threshold. Privacy Guard averaged 0.56 percent of end-to-end time across 12 completed turns. - - - - - - - - - -Privacy Guard latency (ms) · log scale - -10k - -30k - -100k - -300k - -1M - -5 - -10 - -20 - -50 - -100 - -250 - -1M threshold - -18,291 tokens; 5.5 ms; 3 entities detected -18,722 tokens; 6.0 ms; 3 entities detected -18,923 tokens; 6.0 ms; 4 entities detected -19,120 tokens; 10.5 ms; 4 entities detected -19,378 tokens; 12.5 ms; 6 entities detected -19,798 tokens; 21.7 ms; 6 entities detected -20,335 tokens; 7.8 ms; 6 entities detected -21,404 tokens; 6.9 ms; 11 entities detected -22,377 tokens; 8.8 ms; 11 entities detected -22,934 tokens; 9.7 ms; 11 entities detected -23,528 tokens; 7.8 ms; 15 entities detected -23,866 tokens; 10.9 ms; 16 entities detected -26,472 tokens; 10.9 ms; 22 entities detected -26,853 tokens; 11.7 ms; 22 entities detected -28,021 tokens; 12.0 ms; 22 entities detected -28,867 tokens; 11.0 ms; 26 entities detected -31,351 tokens; 12.8 ms; 26 entities detected -33,610 tokens; 14.6 ms; 26 entities detected -34,193 tokens; 14.5 ms; 26 entities detected -20,747 tokens; 10.8 ms; 19 entities detected -22,310 tokens; 18.2 ms; 20 entities detected -23,103 tokens; 16.4 ms; 20 entities detected -27,015 tokens; 28.3 ms; 22 entities detected -31,441 tokens; 13.0 ms; 22 entities detected -38,362 tokens; 12.4 ms; 23 entities detected -39,098 tokens; 18.3 ms; 23 entities detected -39,471 tokens; 17.0 ms; 23 entities detected -40,025 tokens; 10.9 ms; 23 entities detected -47,124 tokens; 14.8 ms; 23 entities detected -47,960 tokens; 11.6 ms; 23 entities detected -48,251 tokens; 17.6 ms; 23 entities detected -51,102 tokens; 13.0 ms; 23 entities detected -51,552 tokens; 11.3 ms; 23 entities detected -52,216 tokens; 14.3 ms; 23 entities detected -52,634 tokens; 11.1 ms; 23 entities detected -53,244 tokens; 20.7 ms; 23 entities detected -62,423 tokens; 13.7 ms; 23 entities detected -63,068 tokens; 17.8 ms; 23 entities detected -63,486 tokens; 17.5 ms; 23 entities detected -64,295 tokens; 26.0 ms; 23 entities detected -76,372 tokens; 18.9 ms; 23 entities detected -77,031 tokens; 18.1 ms; 23 entities detected -82,339 tokens; 26.1 ms; 24 entities detected -83,184 tokens; 36.0 ms; 24 entities detected -88,677 tokens; 20.9 ms; 24 entities detected -89,748 tokens; 18.7 ms; 24 entities detected -89,970 tokens; 20.0 ms; 24 entities detected -90,530 tokens; 21.0 ms; 24 entities detected -91,420 tokens; 21.7 ms; 24 entities detected -92,901 tokens; 22.7 ms; 24 entities detected -93,678 tokens; 36.0 ms; 24 entities detected -95,694 tokens; 25.2 ms; 24 entities detected -96,704 tokens; 30.2 ms; 24 entities detected -96,868 tokens; 27.0 ms; 24 entities detected -97,886 tokens; 30.3 ms; 24 entities detected -98,356 tokens; 24.5 ms; 24 entities detected -99,265 tokens; 35.3 ms; 24 entities detected -101,025 tokens; 34.2 ms; 24 entities detected -101,193 tokens; 23.3 ms; 24 entities detected -102,071 tokens; 26.9 ms; 24 entities detected -102,481 tokens; 23.7 ms; 24 entities detected -104,787 tokens; 29.4 ms; 24 entities detected -105,313 tokens; 31.9 ms; 24 entities detected -105,801 tokens; 35.6 ms; 24 entities detected -105,958 tokens; 39.6 ms; 24 entities detected -106,160 tokens; 33.4 ms; 25 entities detected -106,428 tokens; 24.1 ms; 25 entities detected -106,807 tokens; 47.7 ms; 25 entities detected -106,981 tokens; 43.3 ms; 25 entities detected -107,412 tokens; 27.1 ms; 25 entities detected -107,647 tokens; 32.0 ms; 25 entities detected -108,052 tokens; 38.2 ms; 25 entities detected -108,800 tokens; 36.9 ms; 25 entities detected -114,425 tokens; 32.0 ms; 25 entities detected -114,584 tokens; 32.3 ms; 25 entities detected -115,395 tokens; 24.7 ms; 25 entities detected -115,852 tokens; 39.1 ms; 25 entities detected -117,730 tokens; 42.9 ms; 25 entities detected -120,732 tokens; 36.0 ms; 25 entities detected -122,710 tokens; 47.4 ms; 27 entities detected -127,634 tokens; 38.6 ms; 59 entities detected -136,816 tokens; 49.1 ms; 59 entities detected -143,349 tokens; 41.2 ms; 59 entities detected -323,611 tokens; 93.4 ms; 259 entities detected -324,158 tokens; 96.5 ms; 259 entities detected -325,804 tokens; 95.2 ms; 259 entities detected -329,592 tokens; 91.0 ms; 259 entities detected -656,648 tokens; 160.3 ms; 323 entities detected -665,140 tokens; 164.3 ms; 323 entities detected -963,788 tokens; 220.3 ms; 389 entities detected -965,007 tokens; 216.3 ms; 389 entities detected -1,140,979 tokens; 268.7 ms; 429 entities detected -228,409 tokens; 54.2 ms; 43 entities detected -428,405 tokens; 103.4 ms; 83 entities detected -628,418 tokens; 196.2 ms; 123 entities detected -801,456 tokens; 176.0 ms; 163 entities detected -Privacy Guard averaged 0.56% -of end-to-end turn time -across 12 completed turns -Entities detected - -Prompt tokens · log scale -Single fit across 96 measurements: 6.69 + 23.28 ms per 100k tokens · R² 0.976 - -3 - -100 - -200 - -300 - -429 - diff --git a/projects/privacy-guard/analysis/render_latency_plot.py b/projects/privacy-guard/analysis/render_latency_plot.py index d3bff43..da4ff2b 100644 --- a/projects/privacy-guard/analysis/render_latency_plot.py +++ b/projects/privacy-guard/analysis/render_latency_plot.py @@ -11,7 +11,13 @@ _ANALYSIS_DIR = Path(__file__).resolve().parent _DEFAULT_DATA = _ANALYSIS_DIR / "privacy-guard-latency.csv" -_DEFAULT_OUTPUT = _ANALYSIS_DIR / "privacy-guard-latency-vs-prompt-size.svg" +_DEFAULT_OUTPUT = ( + _ANALYSIS_DIR.parent + / "docs" + / "assets" + / "analysis" + / "privacy-guard-latency-vs-prompt-size.svg" +) _WIDTH = 1200 _HEIGHT = 720 diff --git a/projects/privacy-guard/docs/architecture b/projects/privacy-guard/docs/architecture deleted file mode 120000 index d63f714..0000000 --- a/projects/privacy-guard/docs/architecture +++ /dev/null @@ -1 +0,0 @@ -../../../docs/documentation/privacy-guard/architecture \ No newline at end of file diff --git a/docs/documentation/privacy-guard/architecture/index.md b/projects/privacy-guard/docs/architecture/index.md similarity index 97% rename from docs/documentation/privacy-guard/architecture/index.md rename to projects/privacy-guard/docs/architecture/index.md index d03502e..5a006fb 100644 --- a/docs/documentation/privacy-guard/architecture/index.md +++ b/projects/privacy-guard/docs/architecture/index.md @@ -15,7 +15,7 @@ allow, replacement, or deny result. ![Privacy Guard component architecture. The service layer translates OpenShell gRPC messages, the request processor applies policy, and registered engines detect or replace entities without depending on the transport -layer.](../../../assets/privacy-guard/diagrams/component-architecture.svg) +layer.](../assets/diagrams/component-architecture.svg) OpenShell owns request routing and provider credential attachment. Privacy Guard does not send provider requests. @@ -83,7 +83,7 @@ final text and returns it as the new body. ![Privacy Guard processing pipeline from validated policy and input text through strategy selection, one shared timeout, ordered engine stages, validation, aggregation, and the final policy -decision.](../../../assets/privacy-guard/diagrams/processing-pipeline.svg) +decision.](../assets/diagrams/processing-pipeline.svg) Engines receive `DETECT` or `REPLACE`, not the user-facing policy action. Blocking is applied after detection by `RequestProcessor`. diff --git a/docs/documentation/privacy-guard/architecture/request-lifecycle.md b/projects/privacy-guard/docs/architecture/request-lifecycle.md similarity index 98% rename from docs/documentation/privacy-guard/architecture/request-lifecycle.md rename to projects/privacy-guard/docs/architecture/request-lifecycle.md index 73c79c4..2f1f73c 100644 --- a/docs/documentation/privacy-guard/architecture/request-lifecycle.md +++ b/projects/privacy-guard/docs/architecture/request-lifecycle.md @@ -16,7 +16,7 @@ active processor, processes one text value, and serializes the result. policy, resolves the active processor, handles an empty body or decodes UTF-8 text, runs the ordered engine stages, then returns either a gRPC failure or a successful allow, replacement, or deny -result.](../../../assets/privacy-guard/diagrams/request-lifecycle.svg) +result.](../assets/diagrams/request-lifecycle.svg) ## 1. Validate the transport diff --git a/docs/documentation/privacy-guard/architecture/service-boundary.md b/projects/privacy-guard/docs/architecture/service-boundary.md similarity index 100% rename from docs/documentation/privacy-guard/architecture/service-boundary.md rename to projects/privacy-guard/docs/architecture/service-boundary.md diff --git a/docs/assets/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg b/projects/privacy-guard/docs/assets/analysis/privacy-guard-latency-vs-prompt-size.svg similarity index 100% rename from docs/assets/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg rename to projects/privacy-guard/docs/assets/analysis/privacy-guard-latency-vs-prompt-size.svg diff --git a/docs/assets/privacy-guard/diagrams/component-architecture.svg b/projects/privacy-guard/docs/assets/diagrams/component-architecture.svg similarity index 100% rename from docs/assets/privacy-guard/diagrams/component-architecture.svg rename to projects/privacy-guard/docs/assets/diagrams/component-architecture.svg diff --git a/docs/assets/privacy-guard/diagrams/processing-pipeline.svg b/projects/privacy-guard/docs/assets/diagrams/processing-pipeline.svg similarity index 100% rename from docs/assets/privacy-guard/diagrams/processing-pipeline.svg rename to projects/privacy-guard/docs/assets/diagrams/processing-pipeline.svg diff --git a/docs/assets/privacy-guard/diagrams/request-lifecycle.svg b/projects/privacy-guard/docs/assets/diagrams/request-lifecycle.svg similarity index 100% rename from docs/assets/privacy-guard/diagrams/request-lifecycle.svg rename to projects/privacy-guard/docs/assets/diagrams/request-lifecycle.svg diff --git a/docs/assets/privacy-guard/diagrams/request-path.svg b/projects/privacy-guard/docs/assets/diagrams/request-path.svg similarity index 100% rename from docs/assets/privacy-guard/diagrams/request-path.svg rename to projects/privacy-guard/docs/assets/diagrams/request-path.svg diff --git a/docs/documentation/privacy-guard/configuration.md b/projects/privacy-guard/docs/configuration.md similarity index 100% rename from docs/documentation/privacy-guard/configuration.md rename to projects/privacy-guard/docs/configuration.md diff --git a/docs/documentation/privacy-guard/engines/custom.md b/projects/privacy-guard/docs/engines/custom.md similarity index 100% rename from docs/documentation/privacy-guard/engines/custom.md rename to projects/privacy-guard/docs/engines/custom.md diff --git a/docs/documentation/privacy-guard/engines/regex.md b/projects/privacy-guard/docs/engines/regex.md similarity index 100% rename from docs/documentation/privacy-guard/engines/regex.md rename to projects/privacy-guard/docs/engines/regex.md diff --git a/projects/privacy-guard/docs/index.md b/projects/privacy-guard/docs/index.md index d6cebcb..9c24bb7 100644 --- a/projects/privacy-guard/docs/index.md +++ b/projects/privacy-guard/docs/index.md @@ -1,8 +1,217 @@ -# Privacy Guard documentation +--- +title: Privacy Guard +description: Overview and quickstart for running Privacy Guard with OpenShell. +agent_markdown: true +--- -Start with the repository documentation: +# Privacy Guard -- [Privacy Guard overview and quickstart](../../../docs/documentation/privacy-guard/index.md) -- [Configure policies](../../../docs/documentation/privacy-guard/configuration.md) -- [Run and operate Privacy Guard](../../../docs/documentation/privacy-guard/operations.md) -- [System architecture](../../../docs/documentation/privacy-guard/architecture/index.md) +Privacy Guard is an OpenShell supervisor middleware. It examines provider-bound +HTTP request bodies before OpenShell attaches provider credentials. A policy can +instruct Privacy Guard to: + +| Action | Result when entities are detected | +| --- | --- | +| `detect` | Allow the original body and report bounded findings | +| `block` | Deny the request and report bounded findings | +| `replace` | Allow a body produced by the configured replacement engines and report bounded findings | + +Privacy Guard processes the complete request body as UTF-8 text. It does not +parse JSON fields, inspect files in the sandbox, modify provider responses, or +send network requests to the provider. + +## How it fits into OpenShell + +![A provider-bound request travels from the sandbox through OpenShell and +Privacy Guard. Privacy Guard returns an allow, replacement, or deny decision +before OpenShell attaches credentials and sends an allowed request to the +provider.](assets/diagrams/request-path.svg) + +Privacy Guard exposes OpenShell's `SupervisorMiddleware` gRPC service. It +registers for the pre-credentials request phase so replacement happens before +the provider receives the request. + +## Quickstart + +This quickstart uses the built-in `RegexEngine` example to replace an email +address and customer ID. It requires: + +- Python 3.11 or newer +- `uv` 0.11 or newer +- OpenShell and `openshell-gateway` at the version recorded by the Privacy + Guard project +- a Docker or Podman backend supported by OpenShell + +Run the commands from a checkout of OpenShell Research. + +### 1. Prepare the example + +```bash +cd projects/privacy-guard/examples/regex-engine +uv sync --locked +uv run privacy-guard engines +``` + +The engine list should contain: + +```text +regex detect,replace +``` + +The example files are: + +| File | Purpose | +| --- | --- | +| `patterns.yaml` | Email and customer-ID rules | +| `privacy-guard-config.yaml` | Standalone Privacy Guard policy configuration | +| `policy.yaml` | Complete OpenShell sandbox policy with the same configuration | + +### 2. Start Privacy Guard + +From the example directory: + +```bash +uv run privacy-guard serve --listen 0.0.0.0:50051 +``` + +Keep the process running. The development server uses plaintext gRPC and +receives request bodies. Restrict port 50051 to the host and trusted sandbox +network. + +### 3. Register Privacy Guard with the gateway + +Choose a non-loopback host IPv4 address that both the gateway and sandbox +supervisor can reach: + +```bash +# macOS +ipconfig getifaddr en0 + +# Linux +hostname -I +``` + +Create a local gateway configuration: + +```bash +export PRIVACY_GUARD_HOST_IP=YOUR_HOST_IPV4 + +uv run privacy-guard configure-gateway \ + --host-ip "$PRIVACY_GUARD_HOST_IP" \ + --name privacy-guard-regex \ + --config gateway.local.toml +``` + +Do not use `127.0.0.1`: loopback inside the sandbox supervisor does not refer to +the host. + +Restart the local gateway with `gateway.local.toml`. For a Homebrew +installation: + +```bash +brew services stop openshell +export OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/homebrew/tls" +openshell-gateway --config "$PWD/gateway.local.toml" +``` + +For a Debian or RPM installation: + +```bash +systemctl --user stop openshell-gateway +export OPENSHELL_LOCAL_TLS_DIR="$HOME/.local/state/openshell/tls" +openshell-gateway --config "$PWD/gateway.local.toml" +``` + +Keep the foreground gateway running. + +### 4. Create a sandbox + +In another terminal, return to the example directory and verify the gateway: + +```bash +openshell gateway select openshell +openshell status +``` + +Create the example sandbox: + +```bash +openshell sandbox create \ + --name privacy-guard-regex \ + --from base \ + --no-auto-providers \ + --policy "$PWD/policy.yaml" \ + -- env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 claude +``` + +After authenticating Claude Code, submit: + +```text +Draft a short greeting for user@example.com about customer CUST-12345678. +``` + +The provider-bound body should contain `[email]` and `[customer-id]` instead of +the original values. + +### 5. Verify the result + +From a host terminal: + +```bash +openshell logs privacy-guard-regex -n 100 --source sandbox +``` + +Find the provider request and confirm: + +- `transformed:true` +- an `email (identifiers)` finding +- a `customer-id (identifiers)` finding +- no matched email address or customer ID in the findings + +See [Run and operate Privacy Guard](operations.md) for cleanup, gateway +configuration, logging, timeouts, and troubleshooting. + +## Core concepts + +### Policies control behavior + +The OpenShell policy supplies the ordered engine stages and final action. +Privacy Guard startup selects which engine implementations and operational +resources are installed. + +### Stages run in order + +Each stage receives the current text. In `replace` mode, later stages receive +the text returned by earlier stages. In `detect` and `block` mode, engines must +return the input text unchanged. + +### Findings do not contain matched values + +Findings contain the entity name, stage, confidence, and occurrence count. +Matched text, surrounding text, offsets, regex patterns, headers, and request +bodies do not cross the service boundary. + +### Processing is bounded + +Request size, output size, detections, regex execution, concurrency, and +processing time have explicit limits. A limit failure denies the request +without returning partial replacement text or partial findings. + +## Documentation map + +- [Configure policies](configuration.md): stages, actions, catalogs, and policy + recipes. +- [Run and operate Privacy Guard](operations.md): CLI, gateway registration, + server lifecycle, logging, and troubleshooting. +- [Use RegexEngine](engines/regex.md): pattern catalogs, flags, matching, and + replacement. +- [Add a custom engine](engines/custom.md): engine contract, resources, + registry factories, and tests. +- [System architecture](architecture/index.md): components, trust boundaries, + state, and concurrency. +- [Request lifecycle](architecture/request-lifecycle.md): validation, + activation, processing, and result mapping. +- [Service boundary](architecture/service-boundary.md): gRPC methods, + protobuf translation, and worker scheduling. +- [Limits and failure behavior](reference/limits-and-failures.md): enforced + bounds, error outcomes, retention, and measured latency. diff --git a/docs/documentation/privacy-guard/operations.md b/projects/privacy-guard/docs/operations.md similarity index 100% rename from docs/documentation/privacy-guard/operations.md rename to projects/privacy-guard/docs/operations.md diff --git a/docs/documentation/privacy-guard/reference/limits-and-failures.md b/projects/privacy-guard/docs/reference/limits-and-failures.md similarity index 98% rename from docs/documentation/privacy-guard/reference/limits-and-failures.md rename to projects/privacy-guard/docs/reference/limits-and-failures.md index cfdca5f..1c37ad7 100644 --- a/docs/documentation/privacy-guard/reference/limits-and-failures.md +++ b/projects/privacy-guard/docs/reference/limits-and-failures.md @@ -205,7 +205,7 @@ latency_ms = 6.69 + 23.28 * (prompt_tokens / 100,000) R² = 0.976 ``` -![Scatter plot of Privacy Guard latency against prompt tokens on logarithmic axes. Point color maps the number of entities detected from 3 to 429. Ninety-six service measurements follow one fitted line. A dashed line marks the nominal one-million-token context threshold.](../../../assets/privacy-guard/analysis/privacy-guard-latency-vs-prompt-size.svg) +![Scatter plot of Privacy Guard latency against prompt tokens on logarithmic axes. Point color maps the number of entities detected from 3 to 429. Ninety-six service measurements follow one fitted line. A dashed line marks the nominal one-million-token context threshold.](../assets/analysis/privacy-guard-latency-vs-prompt-size.svg) Across the 12 completed large-context turns with end-to-end timing, Privacy Guard service processing averaged 0.56% of request-admission-to-last-output diff --git a/scripts/build-docs.sh b/scripts/build-docs.sh index 16bee84..70457c2 100755 --- a/scripts/build-docs.sh +++ b/scripts/build-docs.sh @@ -30,6 +30,7 @@ fi python -m pip install --upgrade pip python -m pip install -r requirements-docs.txt +python scripts/stage-privacy-guard-docs.py python scripts/render-dev-notes.py zensical build --clean --strict python scripts/publish-agent-markdown.py diff --git a/scripts/stage-privacy-guard-docs.py b/scripts/stage-privacy-guard-docs.py new file mode 100644 index 0000000..4c5588f --- /dev/null +++ b/scripts/stage-privacy-guard-docs.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Stage canonical Privacy Guard documentation in the site source tree.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import shutil + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_SOURCE = ROOT / "projects" / "privacy-guard" / "docs" +DEFAULT_DESTINATION = ROOT / "docs" / "documentation" / "privacy-guard" + + +def stage_privacy_guard_docs(source: Path, destination: Path) -> None: + """Replace the generated site mirror with one canonical project-docs tree.""" + + source = source.resolve() + destination = destination.absolute() + if not source.is_dir(): + raise ValueError(f"documentation source does not exist: {source}") + if destination.is_symlink(): + raise ValueError(f"documentation destination must not be a symlink: {destination}") + if source == destination.resolve(): + raise ValueError("documentation source and destination must differ") + + symlinks = sorted(path for path in source.rglob("*") if path.is_symlink()) + if symlinks: + paths = "\n".join(f" - {path}" for path in symlinks) + raise ValueError(f"project documentation must not contain symlinks:\n{paths}") + + destination.parent.mkdir(parents=True, exist_ok=True) + staging = destination.with_name(f".{destination.name}.staging") + if staging.exists(): + shutil.rmtree(staging) + shutil.copytree(source, staging) + + if destination.exists(): + shutil.rmtree(destination) + os.replace(staging, destination) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, default=DEFAULT_SOURCE) + parser.add_argument("--destination", type=Path, default=DEFAULT_DESTINATION) + args = parser.parse_args() + + try: + stage_privacy_guard_docs(args.source, args.destination) + except ValueError as error: + parser.error(str(error)) + print(f"Staged Privacy Guard documentation from {args.source}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_stage_privacy_guard_docs.py b/tests/test_stage_privacy_guard_docs.py new file mode 100644 index 0000000..a91d59c --- /dev/null +++ b/tests/test_stage_privacy_guard_docs.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "stage-privacy-guard-docs.py" + +SPEC = importlib.util.spec_from_file_location("stage_privacy_guard_docs", SCRIPT) +if SPEC is None or SPEC.loader is None: + raise RuntimeError(f"could not load {SCRIPT}") +STAGER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(STAGER) + + +class StagePrivacyGuardDocsTests(unittest.TestCase): + def test_stage_replaces_destination_with_source_tree(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source = root / "project-docs" + destination = root / "site-docs" + (source / "architecture").mkdir(parents=True) + (source / "index.md").write_text("# Overview\n", encoding="utf-8") + (source / "architecture/index.md").write_text( + "# Architecture\n", + encoding="utf-8", + ) + destination.mkdir() + (destination / "stale.md").write_text("# Stale\n", encoding="utf-8") + + STAGER.stage_privacy_guard_docs(source, destination) + + self.assertEqual( + (destination / "index.md").read_text(encoding="utf-8"), + "# Overview\n", + ) + self.assertTrue((destination / "architecture/index.md").is_file()) + self.assertFalse((destination / "stale.md").exists()) + + def test_stage_rejects_symlinks_in_source(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source = root / "project-docs" + source.mkdir() + target = root / "outside.md" + target.write_text("# Outside\n", encoding="utf-8") + (source / "linked.md").symlink_to(target) + + with self.assertRaisesRegex(ValueError, "must not contain symlinks"): + STAGER.stage_privacy_guard_docs(source, root / "site-docs") + + +if __name__ == "__main__": + unittest.main() From e0c4b3019806efb0358315fe8c4facb0de26cb49 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Tue, 28 Jul 2026 17:19:21 -0400 Subject: [PATCH 4/8] Address documentation panel findings --- projects/privacy-guard/README.md | 15 +++++++----- .../privacy-guard/docs/architecture/index.md | 5 ++-- .../docs/architecture/request-lifecycle.md | 5 ++-- .../docs/architecture/service-boundary.md | 6 +++-- projects/privacy-guard/docs/engines/custom.md | 21 ++++++++++++++++ projects/privacy-guard/docs/index.md | 24 ++++++++++++++----- projects/privacy-guard/docs/operations.md | 12 ++++++++-- .../docs/reference/limits-and-failures.md | 15 ++++++++---- scripts/stage-privacy-guard-docs.py | 22 ++++++----------- tests/test_stage_privacy_guard_docs.py | 17 +++++++++++++ 10 files changed, 102 insertions(+), 40 deletions(-) diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 21601cd..19ca479 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -15,8 +15,10 @@ pipeline of entity-processing engines. | `block` | Deny requests containing configured entities | | `replace` | Allow the final body returned by replacement-capable engines | -Findings contain entity, stage, confidence, and count. They do not contain -matched text, surrounding text, offsets, patterns, headers, or credentials. +Findings contain entity, stage, confidence, and count. Framework-controlled +fields and the built-in `RegexEngine` do not add matched text, surrounding +text, offsets, patterns, headers, or credentials. Custom engines must use +stable entity identifiers that are not derived from request text. ## Developer start @@ -89,7 +91,7 @@ uv run privacy-guard configuration-schema uv run privacy-guard configure-gateway --host-ip YOUR_HOST_IPV4 uv run privacy-guard serve \ --listen 0.0.0.0:50051 \ - --timeout-seconds 5 + --timeout-seconds 4 ``` `configure-gateway` adds or updates a Privacy Guard registration in the @@ -97,9 +99,10 @@ OpenShell gateway TOML. Its registration name must match the policy's `middleware` field. Restart the gateway after changing registrations. The processing timeout is one bound shared by every stage. It defaults to 1 -second and cannot exceed 30 seconds. Configure a longer OpenShell middleware -timeout to include worker queueing, configuration validation, and processor -preparation. +second and cannot exceed 30 seconds. `configure-gateway` writes a five-second +OpenShell middleware timeout, so use a shorter processing timeout or edit the +registration to provide more headroom. Rerunning `configure-gateway` restores +the five-second value. Use a trusted registry factory for custom engines: diff --git a/projects/privacy-guard/docs/architecture/index.md b/projects/privacy-guard/docs/architecture/index.md index 5a006fb..01d32e4 100644 --- a/projects/privacy-guard/docs/architecture/index.md +++ b/projects/privacy-guard/docs/architecture/index.md @@ -124,8 +124,9 @@ The processor returns only: - stage-qualified detection summaries - a stable deny reason when applicable -Detection summaries omit matched text, context, offsets, patterns, and raw -engine metadata. +Framework-controlled detection summaries omit matched text, context, offsets, +patterns, and raw engine metadata. Custom engines must keep entity identifiers +stable and independent of request text. ## Read next diff --git a/projects/privacy-guard/docs/architecture/request-lifecycle.md b/projects/privacy-guard/docs/architecture/request-lifecycle.md index 2f1f73c..16fd498 100644 --- a/projects/privacy-guard/docs/architecture/request-lifecycle.md +++ b/projects/privacy-guard/docs/architecture/request-lifecycle.md @@ -131,8 +131,9 @@ It does not deduplicate across stages. Stages may inspect different text revisions, and confidence values from different engines are not assumed to be calibrated. -Aggregated summaries exclude matched text, surrounding text, offsets, patterns, -and engine metadata. +Framework-controlled summary fields exclude matched text, surrounding text, +offsets, patterns, and engine metadata. Engine-provided entity identifiers +cross the boundary, so custom engines must not derive them from request text. ## 9. Apply the policy action diff --git a/projects/privacy-guard/docs/architecture/service-boundary.md b/projects/privacy-guard/docs/architecture/service-boundary.md index 6a99389..94ab08f 100644 --- a/projects/privacy-guard/docs/architecture/service-boundary.md +++ b/projects/privacy-guard/docs/architecture/service-boundary.md @@ -97,8 +97,10 @@ Each aggregated detection summary becomes one OpenShell finding: | `confidence` | `low`, `medium`, `high`, or empty | | `count` | Number of aggregated occurrences | -Matched content, context, offsets, patterns, and raw engine metadata do not -cross the protobuf boundary. +The protobuf has no fields for matched content, context, offsets, patterns, or +raw engine metadata. Engine-provided entity identifiers and configured stage +names do cross in `label`, so custom engines must keep identifiers independent +of request text. If the complete result cannot fit OpenShell's finding bounds, the service returns `privacy_guard_limit_exceeded` without partial findings. diff --git a/projects/privacy-guard/docs/engines/custom.md b/projects/privacy-guard/docs/engines/custom.md index 18f99fa..4b33d97 100644 --- a/projects/privacy-guard/docs/engines/custom.md +++ b/projects/privacy-guard/docs/engines/custom.md @@ -224,6 +224,27 @@ engine exceptions. Do not include: Stable engine and entity identifiers may appear in findings and diagnostic logs when they satisfy shared validation. +Use a static, content-safe message when translating an operational failure: + +```python +from privacy_guard.engines import EngineExecutionError + +try: + result = self.resources.client.process(text) +except AcmeClientError: + raise EngineExecutionError("Acme processing failed") from None +``` + +| Exception | Use in a custom engine | +| --- | --- | +| `EngineConfigurationError` | Strategy-specific configuration is unusable | +| `EngineExecutionError` | A collaborator or runtime operation failed | +| `EngineLimitExceededError` | Engine-owned bounded work or output exceeded its limit | + +The framework raises `EngineContractError` when returned text or detections +violate the engine contract; custom engines should not use it for collaborator +failures. + ## Register the engine Create one application-scoped registry factory: diff --git a/projects/privacy-guard/docs/index.md b/projects/privacy-guard/docs/index.md index 9c24bb7..838e250 100644 --- a/projects/privacy-guard/docs/index.md +++ b/projects/privacy-guard/docs/index.md @@ -38,9 +38,20 @@ address and customer ID. It requires: - Python 3.11 or newer - `uv` 0.11 or newer -- OpenShell and `openshell-gateway` at the version recorded by the Privacy - Guard project +- OpenShell and `openshell-gateway` `v0.0.90`, the version recorded in the + [middleware manifest](https://github.com/NVIDIA/OpenShell-Research/blob/main/projects/privacy-guard/.openshell-middleware-manifest.json) - a Docker or Podman backend supported by OpenShell +- Claude Code installed and Claude Code subscription access for the final + provider request + +Confirm the installed command versions: + +```bash +uv --version +openshell --version +openshell-gateway --version +claude --version +``` Run the commands from a checkout of OpenShell Research. @@ -185,11 +196,12 @@ Each stage receives the current text. In `replace` mode, later stages receive the text returned by earlier stages. In `detect` and `block` mode, engines must return the input text unchanged. -### Findings do not contain matched values +### Findings use stable identifiers -Findings contain the entity name, stage, confidence, and occurrence count. -Matched text, surrounding text, offsets, regex patterns, headers, and request -bodies do not cross the service boundary. +Framework-controlled fields do not add matched text, surrounding text, offsets, +regex patterns, headers, or request bodies to findings. `RegexEngine` uses +configured entity identifiers. Custom engines must also return stable, +declared entity identifiers that are not derived from request text. ### Processing is bounded diff --git a/projects/privacy-guard/docs/operations.md b/projects/privacy-guard/docs/operations.md index f65c41e..6861a29 100644 --- a/projects/privacy-guard/docs/operations.md +++ b/projects/privacy-guard/docs/operations.md @@ -41,7 +41,7 @@ For OpenShell sandbox supervisors running outside the host network namespace: ```bash uv run privacy-guard serve \ --listen 0.0.0.0:50051 \ - --timeout-seconds 5 + --timeout-seconds 4 ``` `--timeout-seconds` is one deadline shared by every stage in a request. It @@ -85,6 +85,12 @@ The command writes to: Use `--config PATH` to write a different gateway TOML. Existing unrelated gateway settings are preserved. +`configure-gateway` writes a five-second OpenShell middleware timeout. The +four-second processing timeout above leaves one second for queueing, +configuration validation, processor preparation, and transport overhead. If +you select a longer processing timeout, edit the generated registration to add +headroom. Rerunning `configure-gateway` restores the timeout to five seconds. + The registration name must match the policy's `middleware` field: ```yaml @@ -159,7 +165,9 @@ For a provider request, inspect: | findings | Aggregated entity, stage, confidence, and count | | deny reason | `privacy_guard_blocked` or `privacy_guard_limit_exceeded` | -Findings never contain matched values or surrounding request text. +Framework-controlled finding fields and `RegexEngine` do not add matched values +or surrounding request text. Custom engines must use stable, declared entity +identifiers that are not derived from request text. ## Privacy Guard logging diff --git a/projects/privacy-guard/docs/reference/limits-and-failures.md b/projects/privacy-guard/docs/reference/limits-and-failures.md index 1c37ad7..b7629c0 100644 --- a/projects/privacy-guard/docs/reference/limits-and-failures.md +++ b/projects/privacy-guard/docs/reference/limits-and-failures.md @@ -46,17 +46,20 @@ bytes unchanged. Replace returns UTF-8 encoded final text. | Concurrent gRPC calls | 16 | One monotonic timeout covers every stage and final result validation. A stage -can consume the complete request detection or timeout budget. +can consume the complete timeout budget. Detections remain subject to both the +256-per-stage and 4,096-per-request limits. Set the processing timeout with: ```bash -privacy-guard serve --timeout-seconds 5 +privacy-guard serve --timeout-seconds 4 ``` OpenShell's middleware timeout must be longer than the Privacy Guard processing timeout. Include headroom for worker queueing, configuration validation, and -processor preparation. +processor preparation. `configure-gateway` writes a five-second OpenShell +timeout, so edit that registration before using a processing timeout of five +seconds or longer. ## Finding and diagnostic limits @@ -68,8 +71,10 @@ processor preparation. | Encoded bytes per finding | 4 KiB | Diagnostic bounds apply to stage names, entity names, metadata keys and values, -model-profile names, and request IDs used in logs. Invalid request IDs are -replaced with a constant placeholder for logging and do not change evaluation. +and request IDs used in logs. Custom configuration fields such as model-profile +names are bounded only when their engine config declares an explicit constraint. +Invalid request IDs are replaced with a constant placeholder for logging and do +not change evaluation. If aggregated summaries cannot fit the OpenShell finding representation, Privacy Guard returns `privacy_guard_limit_exceeded` with no partial findings. diff --git a/scripts/stage-privacy-guard-docs.py b/scripts/stage-privacy-guard-docs.py index 4c5588f..bf277ba 100644 --- a/scripts/stage-privacy-guard-docs.py +++ b/scripts/stage-privacy-guard-docs.py @@ -6,7 +6,6 @@ from __future__ import annotations -import argparse import os from pathlib import Path import shutil @@ -21,13 +20,14 @@ def stage_privacy_guard_docs(source: Path, destination: Path) -> None: """Replace the generated site mirror with one canonical project-docs tree.""" source = source.resolve() - destination = destination.absolute() + destination_is_symlink = destination.is_symlink() + destination = destination.resolve(strict=False) if not source.is_dir(): raise ValueError(f"documentation source does not exist: {source}") - if destination.is_symlink(): + if destination_is_symlink: raise ValueError(f"documentation destination must not be a symlink: {destination}") - if source == destination.resolve(): - raise ValueError("documentation source and destination must differ") + if source.is_relative_to(destination) or destination.is_relative_to(source): + raise ValueError("documentation source and destination must not overlap") symlinks = sorted(path for path in source.rglob("*") if path.is_symlink()) if symlinks: @@ -46,16 +46,8 @@ def stage_privacy_guard_docs(source: Path, destination: Path) -> None: def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source", type=Path, default=DEFAULT_SOURCE) - parser.add_argument("--destination", type=Path, default=DEFAULT_DESTINATION) - args = parser.parse_args() - - try: - stage_privacy_guard_docs(args.source, args.destination) - except ValueError as error: - parser.error(str(error)) - print(f"Staged Privacy Guard documentation from {args.source}.") + stage_privacy_guard_docs(DEFAULT_SOURCE, DEFAULT_DESTINATION) + print(f"Staged Privacy Guard documentation from {DEFAULT_SOURCE}.") return 0 diff --git a/tests/test_stage_privacy_guard_docs.py b/tests/test_stage_privacy_guard_docs.py index a91d59c..0230e9c 100644 --- a/tests/test_stage_privacy_guard_docs.py +++ b/tests/test_stage_privacy_guard_docs.py @@ -55,6 +55,23 @@ def test_stage_rejects_symlinks_in_source(self) -> None: with self.assertRaisesRegex(ValueError, "must not contain symlinks"): STAGER.stage_privacy_guard_docs(source, root / "site-docs") + def test_stage_rejects_destination_inside_source(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + source = Path(temporary_directory) / "project-docs" + source.mkdir() + + with self.assertRaisesRegex(ValueError, "must not overlap"): + STAGER.stage_privacy_guard_docs(source, source / "published") + + def test_stage_rejects_source_inside_destination(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + destination = Path(temporary_directory) / "site-docs" + source = destination / "project-docs" + source.mkdir(parents=True) + + with self.assertRaisesRegex(ValueError, "must not overlap"): + STAGER.stage_privacy_guard_docs(source, destination) + if __name__ == "__main__": unittest.main() From 4cfc760233ff1cfa876a23b552ebfcb04f80e755 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Tue, 28 Jul 2026 17:23:39 -0400 Subject: [PATCH 5/8] Clarify experimental privacy boundary --- projects/privacy-guard/README.md | 10 ++++++++++ projects/privacy-guard/docs/architecture/index.md | 4 ++++ projects/privacy-guard/docs/engines/custom.md | 5 ++++- projects/privacy-guard/docs/index.md | 15 ++++++++++++--- projects/privacy-guard/docs/operations.md | 13 +++++++++++++ 5 files changed, 43 insertions(+), 4 deletions(-) diff --git a/projects/privacy-guard/README.md b/projects/privacy-guard/README.md index 19ca479..d435c61 100644 --- a/projects/privacy-guard/README.md +++ b/projects/privacy-guard/README.md @@ -7,6 +7,16 @@ OpenShell attaches provider credentials. It processes the complete request body as UTF-8 text through an ordered pipeline of entity-processing engines. +> **Experimental:** Privacy Guard is a proof of concept. It reduces exposure on +> provider-bound network requests that OpenShell routes through the middleware; +> it does not guarantee that sensitive data cannot leak. + +Privacy Guard does not intercept data before a harness writes it to disk. +Prompts, tool output, transcripts, and session histories may therefore retain +raw sensitive values even when the provider-bound request is later replaced or +blocked. Use harness persistence controls and appropriate storage isolation, +retention, and cleanup in addition to Privacy Guard. + ## What it does | Policy action | Behavior | diff --git a/projects/privacy-guard/docs/architecture/index.md b/projects/privacy-guard/docs/architecture/index.md index 01d32e4..ec1ff73 100644 --- a/projects/privacy-guard/docs/architecture/index.md +++ b/projects/privacy-guard/docs/architecture/index.md @@ -10,6 +10,10 @@ Privacy Guard is a pre-credentials OpenShell middleware. It translates an OpenShell HTTP-request evaluation into one text-processing run and returns an allow, replacement, or deny result. +This is an experimental network-middleware boundary. It does not mediate +harness filesystem writes, so raw prompts and session histories may be stored +before a request reaches Privacy Guard. + ## System boundary ![Privacy Guard component architecture. The service layer translates diff --git a/projects/privacy-guard/docs/engines/custom.md b/projects/privacy-guard/docs/engines/custom.md index 4b33d97..990d122 100644 --- a/projects/privacy-guard/docs/engines/custom.md +++ b/projects/privacy-guard/docs/engines/custom.md @@ -230,7 +230,10 @@ Use a static, content-safe message when translating an operational failure: from privacy_guard.engines import EngineExecutionError try: - result = self.resources.client.process(text) + result = self.resources.client.process( + text, + timeout=timeout.remaining_seconds(), + ) except AcmeClientError: raise EngineExecutionError("Acme processing failed") from None ``` diff --git a/projects/privacy-guard/docs/index.md b/projects/privacy-guard/docs/index.md index 838e250..5b04d57 100644 --- a/projects/privacy-guard/docs/index.md +++ b/projects/privacy-guard/docs/index.md @@ -16,6 +16,16 @@ instruct Privacy Guard to: | `block` | Deny the request and report bounded findings | | `replace` | Allow a body produced by the configured replacement engines and report bounded findings | +> **Experimental:** Privacy Guard is a proof of concept, not a guarantee that +> sensitive data cannot leak. It currently protects only provider-bound network +> requests that OpenShell routes through this middleware. + +Privacy Guard does not intercept prompts, tool output, transcripts, or session +history before a harness writes them to disk. Those files may contain raw +sensitive values even when Privacy Guard later replaces or blocks the network +request. Use harness persistence controls and appropriate storage isolation, +retention, and cleanup in addition to Privacy Guard. + Privacy Guard processes the complete request body as UTF-8 text. It does not parse JSON fields, inspect files in the sandbox, modify provider responses, or send network requests to the provider. @@ -41,8 +51,8 @@ address and customer ID. It requires: - OpenShell and `openshell-gateway` `v0.0.90`, the version recorded in the [middleware manifest](https://github.com/NVIDIA/OpenShell-Research/blob/main/projects/privacy-guard/.openshell-middleware-manifest.json) - a Docker or Podman backend supported by OpenShell -- Claude Code installed and Claude Code subscription access for the final - provider request +- Claude Code subscription access for the final provider request; the OpenShell + base sandbox supplies the Claude Code binary Confirm the installed command versions: @@ -50,7 +60,6 @@ Confirm the installed command versions: uv --version openshell --version openshell-gateway --version -claude --version ``` Run the commands from a checkout of OpenShell Research. diff --git a/projects/privacy-guard/docs/operations.md b/projects/privacy-guard/docs/operations.md index 6861a29..5b22004 100644 --- a/projects/privacy-guard/docs/operations.md +++ b/projects/privacy-guard/docs/operations.md @@ -10,6 +10,19 @@ Privacy Guard runs as a gRPC service reachable by the OpenShell gateway and sandbox supervisors. The CLI uses the built-in `RegexEngine` registry unless you supply a custom registry factory. +## Protection boundary and local persistence + +Privacy Guard is experimental. It evaluates provider-bound network requests at +the OpenShell middleware boundary; it is not a guarantee that sensitive data +cannot leak. + +The middleware does not run before harness filesystem writes. A harness may +persist raw prompts, tool output, transcripts, or session history before +OpenShell sends a provider request to Privacy Guard. Configure the harness to +disable or minimize persistence where possible, restrict access to sandbox and +host storage, set an appropriate retention policy, and remove sensitive session +artifacts after use. + ## Install the project environment From the repository checkout: From 2d2f78d25a64f08b5321de7de421bf28526ca15c Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Tue, 28 Jul 2026 17:30:15 -0400 Subject: [PATCH 6/8] Describe Privacy Guard on documentation index --- docs/documentation/index.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/documentation/index.md b/docs/documentation/index.md index ea10c6b..0d1d9e7 100644 --- a/docs/documentation/index.md +++ b/docs/documentation/index.md @@ -9,5 +9,6 @@ agent_markdown: true Technical documentation and references for installing, using, and extending OpenShell Research projects. -- [Privacy Guard](privacy-guard/index.md): quickstart, policy configuration, - operations, engines, architecture, and limits. +- [Privacy Guard](privacy-guard/index.md): experimental OpenShell middleware + that detects, blocks, or replaces configured sensitive entities in + provider-bound HTTP requests. From 327a97351cd1fc1c5466df2a22cea12a1dc20eeb Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Tue, 28 Jul 2026 17:31:39 -0400 Subject: [PATCH 7/8] Shorten Privacy Guard index description --- docs/documentation/index.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/documentation/index.md b/docs/documentation/index.md index 0d1d9e7..8f2d1e5 100644 --- a/docs/documentation/index.md +++ b/docs/documentation/index.md @@ -9,6 +9,4 @@ agent_markdown: true Technical documentation and references for installing, using, and extending OpenShell Research projects. -- [Privacy Guard](privacy-guard/index.md): experimental OpenShell middleware - that detects, blocks, or replaces configured sensitive entities in - provider-bound HTTP requests. +- [Privacy Guard](privacy-guard/index.md): middleware for protecting sensitive data in OpenShell. From b2eb4cd63e269451f7bc9451c66bc2d27332fbe3 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Tue, 28 Jul 2026 18:40:10 -0400 Subject: [PATCH 8/8] Add Privacy Guard engines overview --- projects/privacy-guard/docs/configuration.md | 4 +-- projects/privacy-guard/docs/engines/index.md | 26 +++++++++++++++++++ projects/privacy-guard/docs/engines/regex.md | 4 +-- projects/privacy-guard/docs/index.md | 2 +- projects/privacy-guard/docs/operations.md | 2 +- .../docs/reference/limits-and-failures.md | 2 +- zensical.toml | 3 ++- 7 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 projects/privacy-guard/docs/engines/index.md diff --git a/projects/privacy-guard/docs/configuration.md b/projects/privacy-guard/docs/configuration.md index 393ac7d..357769a 100644 --- a/projects/privacy-guard/docs/configuration.md +++ b/projects/privacy-guard/docs/configuration.md @@ -199,7 +199,7 @@ end in `.yaml` or `.yml`. Absolute paths, `..` traversal, and symlinks are rejected. Start Privacy Guard from the directory that contains the referenced catalog, or use a path relative to that directory. -See [Use RegexEngine](engines/regex.md) for the complete catalog schema. +See [RegexEngine](engines/regex.md) for the complete catalog schema. ## Inspect and validate configuration @@ -275,7 +275,7 @@ Privacy Guard process. ## Next steps -- [Use RegexEngine](engines/regex.md) +- [RegexEngine](engines/regex.md) - [Add a custom engine](engines/custom.md) - [Run and operate Privacy Guard](operations.md) - [Limits and failure behavior](reference/limits-and-failures.md) diff --git a/projects/privacy-guard/docs/engines/index.md b/projects/privacy-guard/docs/engines/index.md new file mode 100644 index 0000000..a20d970 --- /dev/null +++ b/projects/privacy-guard/docs/engines/index.md @@ -0,0 +1,26 @@ +--- +title: Engines +description: Understand how Privacy Guard engines detect and replace sensitive entities. +agent_markdown: true +--- + +# Engines + +Engines are pluggable processors that inspect request text for configured +entities. Each policy stage selects an engine, supplies its configuration, and +receives detections plus replacement text when replacement is enabled. + +Engines do not decide whether Privacy Guard allows or denies a request. The +request processor runs the configured stages in order, enforces shared safety +bounds, and applies the policy's final action to their combined results. + +Privacy Guard includes two integration paths: + +- [RegexEngine](regex.md) provides deterministic detection and replacement + using a deployment-defined pattern catalog. +- [Custom engines](custom.md) integrate another detector, model, SDK, or + service through Privacy Guard's engine contract and registry. + +Use the regex engine when the sensitive values have stable, testable formats. +Add a custom engine when detection requires semantics or an external system +that regular expressions cannot provide reliably. diff --git a/projects/privacy-guard/docs/engines/regex.md b/projects/privacy-guard/docs/engines/regex.md index 5458035..2c40a3a 100644 --- a/projects/privacy-guard/docs/engines/regex.md +++ b/projects/privacy-guard/docs/engines/regex.md @@ -1,10 +1,10 @@ --- -title: Use RegexEngine +title: RegexEngine description: Configure RegexEngine catalogs, matching flags, findings, and deterministic replacement. agent_markdown: true --- -# Use RegexEngine +# RegexEngine `RegexEngine` is the built-in Privacy Guard engine. It detects every configured regular-expression match and can replace a deterministic non-overlapping subset diff --git a/projects/privacy-guard/docs/index.md b/projects/privacy-guard/docs/index.md index 5b04d57..37a8e74 100644 --- a/projects/privacy-guard/docs/index.md +++ b/projects/privacy-guard/docs/index.md @@ -224,7 +224,7 @@ without returning partial replacement text or partial findings. recipes. - [Run and operate Privacy Guard](operations.md): CLI, gateway registration, server lifecycle, logging, and troubleshooting. -- [Use RegexEngine](engines/regex.md): pattern catalogs, flags, matching, and +- [RegexEngine](engines/regex.md): pattern catalogs, flags, matching, and replacement. - [Add a custom engine](engines/custom.md): engine contract, resources, registry factories, and tests. diff --git a/projects/privacy-guard/docs/operations.md b/projects/privacy-guard/docs/operations.md index 5b22004..52fe063 100644 --- a/projects/privacy-guard/docs/operations.md +++ b/projects/privacy-guard/docs/operations.md @@ -278,5 +278,5 @@ openshell status ## Next steps - [Configure policies](configuration.md) -- [Use RegexEngine](engines/regex.md) +- [RegexEngine](engines/regex.md) - [Limits and failure behavior](reference/limits-and-failures.md) diff --git a/projects/privacy-guard/docs/reference/limits-and-failures.md b/projects/privacy-guard/docs/reference/limits-and-failures.md index b7629c0..004f524 100644 --- a/projects/privacy-guard/docs/reference/limits-and-failures.md +++ b/projects/privacy-guard/docs/reference/limits-and-failures.md @@ -240,6 +240,6 @@ For maintainers: ## Related pages - [Run and operate Privacy Guard](../operations.md) -- [Use RegexEngine](../engines/regex.md) +- [RegexEngine](../engines/regex.md) - [Request lifecycle](../architecture/request-lifecycle.md) - [Service boundary](../architecture/service-boundary.md) diff --git a/zensical.toml b/zensical.toml index e3dea9e..2cdaf9b 100644 --- a/zensical.toml +++ b/zensical.toml @@ -32,7 +32,8 @@ nav = [ {"Run and operate Privacy Guard" = "documentation/privacy-guard/operations.md"} ]}, {"Engines" = [ - {"Use RegexEngine" = "documentation/privacy-guard/engines/regex.md"}, + "documentation/privacy-guard/engines/index.md", + {"RegexEngine" = "documentation/privacy-guard/engines/regex.md"}, {"Add a custom engine" = "documentation/privacy-guard/engines/custom.md"} ]}, {"Architecture" = [