diff --git a/loopx/control_plane/goals/artifact_lifecycle.py b/loopx/control_plane/goals/artifact_lifecycle.py index 8bcd783c83..14f81e9c60 100644 --- a/loopx/control_plane/goals/artifact_lifecycle.py +++ b/loopx/control_plane/goals/artifact_lifecycle.py @@ -19,9 +19,11 @@ from typing import Any -from ...public_safe_text import find_private_text_match +from ...public_safe_text import ( + ARTIFACT_LIFECYCLE_CATEGORIES, + classify_private_text, +) from ..runtime.public_safety import ( - SECRET_LIKE_SURFACE_PATTERN, public_safe_compact_text, validate_public_safe_value, ) @@ -60,9 +62,14 @@ def _compact_text(value: Any, *, limit: int = 240) -> str | None: validate_public_safe_value(value) except ValueError: return None - # Preserve the stricter existing private-text/provider-token contract too; - # these checks supplement, never replace, the shared public-safety owner. - if find_private_text_match(value) or SECRET_LIKE_SURFACE_PATTERN.search(value): + # One policy-aware call into the shared classifier replaces OR-ing the + # text-owner detector with the credential shape detector. The named policy + # (ARTIFACT_LIFECYCLE_CATEGORIES) preserves this projection's historical + # verdict exactly: every category but a raw remote location. + if ( + classify_private_text(value, categories=ARTIFACT_LIFECYCLE_CATEGORIES) + is not None + ): return None return public_safe_compact_text(value, limit=limit) diff --git a/loopx/control_plane/runtime/public_safety.py b/loopx/control_plane/runtime/public_safety.py index ee0ab1a487..211057667d 100644 --- a/loopx/control_plane/runtime/public_safety.py +++ b/loopx/control_plane/runtime/public_safety.py @@ -4,42 +4,21 @@ from collections.abc import Mapping from typing import Any, Callable, Optional +# Refs #5136: the text-shape definitions live in one owner now +# (loopx/public_safe_text.py). This module consumes them for recursive payload +# validation and public-output policy instead of restating a competing set. The +# redundant-alias form makes each an explicit re-export (house style under +# --no-implicit-reexport), so the existing importers of these names from this +# module are unchanged. +from ...public_safe_text import ( + LOCAL_PATH_SURFACE_PATTERN as LOCAL_PATH_SURFACE_PATTERN, + REMOTE_LOCATION_SURFACE_PATTERN as REMOTE_LOCATION_SURFACE_PATTERN, + SECRET_LIKE_SURFACE_PATTERN as SECRET_LIKE_SURFACE_PATTERN, +) NormalizeText = Callable[..., str] CompactText = Callable[..., Optional[str]] DEFAULT_PUBLIC_SAFE_LIST_LIMIT = 4 -LOCAL_PATH_SURFACE_PATTERN = re.compile( - r"(?]+|" - r"[A-Za-z]:[\\/][^\s`'\"<>]+|" - r"\\\\[A-Za-z0-9_.-]+\\[^\s`'\"<>]+" - r")", - re.IGNORECASE, -) -# Refs #5136: one definition for "this string carries a raw remote location". -# Three validators each restated the same scheme list, and the canonical -# public-safety owner had no counterpart, so a fourth caller had to invent one. -REMOTE_LOCATION_SURFACE_PATTERN = re.compile(r"(?i)\b(?:https?|file|s3|gs|tos|hdfs)://") -SECRET_LIKE_SURFACE_PATTERN = re.compile( - r"(?i)(?:\bbearer\s+[a-z0-9._~+/=-]{16,}|" - r"\b(?:access|api|secret)[_-]?key[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]+|" - r"\b(?:ak|sk)[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]+|" - r"(?]{12,}|" - r"\b(?:password|secret)[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]{12,}|" - r"-{3,}\s*BEGIN (?:[A-Z]+ )?PRIVATE KEY|" - r"\btoken[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]{12,})" -) _CREDENTIAL_FIELD_FAMILIES = frozenset( { "accesskey", diff --git a/loopx/public_safe_text.py b/loopx/public_safe_text.py index 71b6d5efbf..47f3836f5a 100644 --- a/loopx/public_safe_text.py +++ b/loopx/public_safe_text.py @@ -1,6 +1,17 @@ """Canonical private-looking-text rules for public-safe control-plane fields. -Four real validator owners enforce the same contract: `feedback`, +This module is the single owner of "does this string look private?" for the +control plane. It answers two questions that used to be conflated: + +* **Detection** -- recognize a credential shape, a local path, a raw remote + location, or an internal organizational marker, and return an explicit + *category* plus a *reason* so a caller can decide what to do. +* **Permission** -- a named *policy* selects which categories a given surface + rejects. Detection recognizing a value never implies every surface must + reject it: owner-private operational state and repository/PR publication have + different disclosure boundaries (Refs #5136). + +Four real validator owners enforce the same text contract: `feedback`, `authority`, `boundary_authority`, and the TypeScript Vision checkpoint. The rule set used to be copied into each owner, and the copies drifted: one still rejected the ordinary English word "authorization" while another accepted a @@ -9,6 +20,12 @@ `tests/fixtures/public_safe_text_corpus.json` pins both runtimes to one contract. +`control_plane/runtime/public_safety.py` also consumes the shape definitions +here (`SECRET_LIKE_SURFACE_PATTERN`, `LOCAL_PATH_SURFACE_PATTERN`, +`REMOTE_LOCATION_SURFACE_PATTERN`) instead of owning a competing set, so a +caller such as `artifact_lifecycle` can make one policy-aware call rather than +OR-ing independent detectors. + Each owner keeps its own error message and guidance, because those describe the owning surface, not the shared rule. """ @@ -16,6 +33,26 @@ from __future__ import annotations import re +from dataclasses import dataclass + +# --------------------------------------------------------------------------- +# Explicit categories. A caller names a policy (a set of categories) rather +# than reaching for a bare regex, so "recognized" and "rejected here" stay +# separate decisions (Refs #5136, direction 2). +# --------------------------------------------------------------------------- +CATEGORY_CREDENTIAL = "credential" +CATEGORY_LOCAL_PATH = "local_path" +CATEGORY_REMOTE_LOCATION = "remote_location" +CATEGORY_ORG_MARKER = "org_marker" + +ALL_CATEGORIES: frozenset[str] = frozenset( + { + CATEGORY_CREDENTIAL, + CATEGORY_LOCAL_PATH, + CATEGORY_REMOTE_LOCATION, + CATEGORY_ORG_MARKER, + } +) # Credential shape, not the plain English word. LoopX governance prose says @@ -38,23 +75,170 @@ r"[A-Za-z0-9+/=]{16,}", ) -PRIVATE_TEXT_PATTERNS: tuple[re.Pattern[str], ...] = ( - re.compile(r"/" + r"Users/"), - re.compile(r"/" + r"ext_data/"), - re.compile("la" + "rk" + "office", re.I), - re.compile("docs" + r"\." + "internal", re.I), - re.compile(r"\bt-20\d{12}-[a-z0-9]+\b"), - re.compile(r"\b" + "Bear" + r"er\b", re.I), - _AUTHORIZATION_CREDENTIAL_SHAPE, - _BASIC_CREDENTIAL_VALUE, - re.compile(r"\b" + "tok" + r"en\s*=", re.I), - re.compile(r"\b" + "pass" + r"word\b", re.I), - re.compile(r"\b" + "sec" + r"ret\b", re.I), +# Refs #5136: relocated here from control_plane/runtime/public_safety.py so a +# single owner defines each shape. public_safety re-exports these names, so its +# ~8 direct importers and 30+ recursive-validation callers are unchanged. This +# pattern is byte-identical to the one public_safety enforced before the move: +# slice A is a behavior-preserving consolidation, so no existing consumer's +# verdict changes. +LOCAL_PATH_SURFACE_PATTERN = re.compile( + r"(?]+|" + r"[A-Za-z]:[\\/][^\s`'\"<>]+|" + r"\\\\[A-Za-z0-9_.-]+\\[^\s`'\"<>]+" + r")", + re.IGNORECASE, +) +# Refs #5136, direction 3: the shared classifier can *recognize* the local-path +# shapes the legacy surface pattern misses -- a home-relative `~/...` path and a +# local path behind an explicit `path:` prefix. Recognition is opt-in +# (`include_path_gaps`) so this consolidation does not silently tighten the 30+ +# consumers of LOCAL_PATH_SURFACE_PATTERN; wiring these into a surface's +# enforcement policy is the disclosed behavior change tracked separately. +# `file://` is not added to the gap set here: direction 3 does classify it as a +# local path, but acting on that means a public projection stops carrying a +# location it accepts today, which is a disclosed tightening rather than part of +# this relocation. It is applied with the enforcement policy in the follow-up. +HOME_RELATIVE_PATH_PATTERN = re.compile(r"(?]+") +PATH_PREFIX_LOCAL_PATTERN = re.compile( + r"(?]+", re.IGNORECASE +) +LOCAL_PATH_GAP_PATTERNS: tuple[re.Pattern[str], ...] = ( + HOME_RELATIVE_PATH_PATTERN, + PATH_PREFIX_LOCAL_PATTERN, +) +# Refs #5136: one definition for "this string carries a raw remote location". +# Three validators each restated the same scheme list, and the canonical +# public-safety owner had no counterpart, so a fourth caller had to invent one. +REMOTE_LOCATION_SURFACE_PATTERN = re.compile(r"(?i)\b(?:https?|file|s3|gs|tos|hdfs)://") +SECRET_LIKE_SURFACE_PATTERN = re.compile( + r"(?i)(?:\bbearer\s+[a-z0-9._~+/=-]{16,}|" + r"\b(?:access|api|secret)[_-]?key[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]+|" + r"\b(?:ak|sk)[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]+|" + r"(?]{12,}|" + r"\b(?:password|secret)[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]{12,}|" + r"-{3,}\s*BEGIN (?:[A-Z]+ )?PRIVATE KEY|" + r"\btoken[\"']?\s*[=:]\s*[\"']?[^\s`'\"<>]{12,})" +) + + +# The text-owner rule set (feedback / authority / boundary_authority / the +# TypeScript Vision checkpoint). Each entry carries an explicit category and a +# stable reason so `classify_private_text` can hand a caller a named verdict +# instead of a bare regex object. Order is significant: `find_private_text_match` +# returns the first match, and the shared corpus pins that first-match contract. +@dataclass(frozen=True) +class _CategorizedPattern: + pattern: re.Pattern[str] + category: str + reason: str + + +_CATEGORIZED_PRIVATE_TEXT_PATTERNS: tuple[_CategorizedPattern, ...] = ( + _CategorizedPattern( + re.compile(r"/" + r"Users/"), CATEGORY_LOCAL_PATH, "absolute home-directory path" + ), + _CategorizedPattern( + re.compile(r"/" + r"ext_data/"), CATEGORY_ORG_MARKER, "internal ext_data path" + ), + _CategorizedPattern( + re.compile("la" + "rk" + "office", re.I), + CATEGORY_ORG_MARKER, + "internal Lark/Feishu office marker", + ), + _CategorizedPattern( + re.compile("docs" + r"\." + "internal", re.I), + CATEGORY_ORG_MARKER, + "internal docs host marker", + ), + _CategorizedPattern( + re.compile(r"\bt-20\d{12}-[a-z0-9]+\b"), + CATEGORY_ORG_MARKER, + "internal ticket identifier", + ), + _CategorizedPattern( + re.compile(r"\b" + "Bear" + r"er\b", re.I), + CATEGORY_CREDENTIAL, + "bearer auth scheme word", + ), + _CategorizedPattern( + _AUTHORIZATION_CREDENTIAL_SHAPE, + CATEGORY_CREDENTIAL, + "authorization header/assignment shape", + ), + _CategorizedPattern( + _BASIC_CREDENTIAL_VALUE, CATEGORY_CREDENTIAL, "basic-auth credential value" + ), + _CategorizedPattern( + re.compile(r"\b" + "tok" + r"en\s*=", re.I), + CATEGORY_CREDENTIAL, + "token assignment shape", + ), + _CategorizedPattern( + re.compile(r"\b" + "pass" + r"word\b", re.I), + CATEGORY_CREDENTIAL, + "password word", + ), + _CategorizedPattern( + re.compile(r"\b" + "sec" + r"ret\b", re.I), + CATEGORY_CREDENTIAL, + "secret word", + ), +) + +# Kept as the plain pattern tuple so `find_private_text_match` and every +# existing importer see byte-identical behavior (same patterns, same order). +PRIVATE_TEXT_PATTERNS: tuple[re.Pattern[str], ...] = tuple( + entry.pattern for entry in _CATEGORIZED_PRIVATE_TEXT_PATTERNS +) + + +# --------------------------------------------------------------------------- +# Named policies. A policy is the set of categories a surface rejects. Keeping +# these named (rather than inline regex ORs at each caller) is what lets one +# detection owner serve surfaces with different disclosure boundaries. +# --------------------------------------------------------------------------- +# The four text owners reject every recognized category (their historical +# behavior): credential shapes, local paths, remote locations, org markers. +TEXT_OWNER_CATEGORIES: frozenset[str] = ALL_CATEGORIES +# artifact_lifecycle historically OR-ed find_private_text_match (the text-owner +# set) with SECRET_LIKE_SURFACE_PATTERN, downstream of a validate_public_safe_value +# call that already rejected the local-path and credential shapes. That union +# covers every category *except* a raw remote location: this projection has +# always let an ordinary http(s) URL through. The policy preserves that exactly +# rather than silently tightening it; widening it to remote_location is a +# separate, disclosed decision (Refs #5136, direction 2). +ARTIFACT_LIFECYCLE_CATEGORIES: frozenset[str] = frozenset( + {CATEGORY_CREDENTIAL, CATEGORY_LOCAL_PATH, CATEGORY_ORG_MARKER} ) +@dataclass(frozen=True) +class PrivateTextMatch: + """An explicit, categorized private-text detection result.""" + + category: str + reason: str + pattern: re.Pattern[str] + + def find_private_text_match(value: str | None) -> re.Pattern[str] | None: - """Return the first matching private-text pattern, or None when clean.""" + """Return the first matching private-text pattern, or None when clean. + + Preserved verbatim for the four text owners and the shared corpus parity + test; `classify_private_text` is the category-aware successor. + """ if not value: return None @@ -62,3 +246,70 @@ def find_private_text_match(value: str | None) -> re.Pattern[str] | None: if pattern.search(value): return pattern return None + + +# The shape-based detectors, categorized. These supplement the text-owner +# patterns so a single call can cover both owners that artifact_lifecycle used +# to OR together. +_SHAPE_DETECTORS: tuple[tuple[re.Pattern[str], str, str], ...] = ( + (SECRET_LIKE_SURFACE_PATTERN, CATEGORY_CREDENTIAL, "credential-like value shape"), + (LOCAL_PATH_SURFACE_PATTERN, CATEGORY_LOCAL_PATH, "local filesystem path"), + ( + REMOTE_LOCATION_SURFACE_PATTERN, + CATEGORY_REMOTE_LOCATION, + "raw remote location URL", + ), +) + + +def classify_private_text( + value: str | None, + *, + categories: frozenset[str] = ALL_CATEGORIES, + include_path_gaps: bool = False, +) -> PrivateTextMatch | None: + """Return the first recognized private-text match within ``categories``. + + Detection only: recognizing a value does not decide whether a given surface + may publish it. Callers pass the named policy (category set) for their + destination. The text-owner patterns are checked first, in their pinned + order, then the relocated shape detectors, so a value that both owners used + to flag still resolves to a single explicit category and reason. + + ``include_path_gaps`` opts a surface into the direction-3 recognition of + home-relative (``~/``) and ``path:``-prefixed local references. It defaults + to False so this consolidation does not silently tighten any surface that + has not chosen the wider policy. + """ + + if not value: + return None + for entry in _CATEGORIZED_PRIVATE_TEXT_PATTERNS: + if entry.category in categories and entry.pattern.search(value): + return PrivateTextMatch(entry.category, entry.reason, entry.pattern) + for pattern, category, reason in _SHAPE_DETECTORS: + if category in categories and pattern.search(value): + return PrivateTextMatch(category, reason, pattern) + if include_path_gaps and CATEGORY_LOCAL_PATH in categories: + for pattern in LOCAL_PATH_GAP_PATTERNS: + if pattern.search(value): + return PrivateTextMatch( + CATEGORY_LOCAL_PATH, "local path behind a relative/prefixed form", pattern + ) + return None + + +def matches_private_text_policy( + value: str | None, + *, + categories: frozenset[str] = ALL_CATEGORIES, + include_path_gaps: bool = False, +) -> bool: + """True when ``value`` is recognized within the named policy's categories.""" + + return ( + classify_private_text( + value, categories=categories, include_path_gaps=include_path_gaps + ) + is not None + ) diff --git a/tests/control_plane/test_public_safe_text_classifier.py b/tests/control_plane/test_public_safe_text_classifier.py new file mode 100644 index 0000000000..d2e6514beb --- /dev/null +++ b/tests/control_plane/test_public_safe_text_classifier.py @@ -0,0 +1,249 @@ +"""Refs #5136 direction 1: the consolidated text-classification owner. + +`loopx/public_safe_text.py` became the single home for "does this string look +private?". These tests pin the part the relocation added on top of the existing +corpus parity contract: + +* detection returns an explicit *category* and a stable *reason*, not a bare + regex object; +* a named *policy* (a set of categories) decides what a given surface rejects, + so recognizing a value never implies every surface must reject it; +* the direction-3 path-gap recognition (``~/`` and ``path:``-prefixed local + references) is opt-in, so this consolidation does not silently tighten any + surface that has not chosen it; +* the relocation is behavior-preserving: ``find_private_text_match`` and the + ``PRIVATE_TEXT_PATTERNS`` tuple are unchanged, and ``classify_private_text``'s + first text-pattern match is the *identical* pattern object the legacy helper + returns; +* `ml_experiment`'s leading-``/``-or-``~`` rule stays a distinct, stricter + per-field *alias* constraint that the shared classifier does not subsume. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest + +from loopx.control_plane.goals.artifact_lifecycle import ( + _compact_text as artifact_lifecycle_compact, +) +from loopx.control_plane.runtime import public_safety +from loopx.domain_packs.ml_experiment import ( + _compact_public_text as ml_experiment_alias, +) +from loopx.public_safe_text import ( + ALL_CATEGORIES, + ARTIFACT_LIFECYCLE_CATEGORIES, + CATEGORY_CREDENTIAL, + CATEGORY_LOCAL_PATH, + CATEGORY_ORG_MARKER, + CATEGORY_REMOTE_LOCATION, + LOCAL_PATH_SURFACE_PATTERN as OWNER_LOCAL_PATH, + PRIVATE_TEXT_PATTERNS, + REMOTE_LOCATION_SURFACE_PATTERN as OWNER_REMOTE_LOCATION, + SECRET_LIKE_SURFACE_PATTERN as OWNER_SECRET_LIKE, + TEXT_OWNER_CATEGORIES, + _CATEGORIZED_PRIVATE_TEXT_PATTERNS, + classify_private_text, + find_private_text_match, + matches_private_text_policy, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +CORPUS_PATH = REPOSITORY_ROOT / "tests" / "fixtures" / "public_safe_text_corpus.json" +_PLACEHOLDER_RE = re.compile(r"\{([A-Z][A-Z_]*)\}") + +# Sensitive literals are assembled at call time so this file carries no literal +# credential-looking string, matching the corpus fixture's own discipline. +_GITHUB_TOKEN = "ghp_" + "a" * 36 +_AUTHZ_HEADER = "authorization" + ": " + "Basic " + "QWxhZGRpbjpvcGVu" + + +def test_categories_are_the_four_named_decisions() -> None: + assert ALL_CATEGORIES == frozenset( + { + CATEGORY_CREDENTIAL, + CATEGORY_LOCAL_PATH, + CATEGORY_REMOTE_LOCATION, + CATEGORY_ORG_MARKER, + } + ) + + +def test_text_owner_policy_rejects_every_recognized_category() -> None: + # The four text owners historically rejected every shape; their policy is + # the full set so the relocation changes no owner verdict. + assert TEXT_OWNER_CATEGORIES == ALL_CATEGORIES + + +def test_artifact_lifecycle_policy_excludes_only_remote_location() -> None: + # artifact_lifecycle has always let an ordinary http(s) URL through, so its + # policy is every category *except* a raw remote location. Widening it is a + # separate, disclosed decision (Refs #5136 direction 2), not part of the + # behavior-preserving relocation. + assert ARTIFACT_LIFECYCLE_CATEGORIES == frozenset( + {CATEGORY_CREDENTIAL, CATEGORY_LOCAL_PATH, CATEGORY_ORG_MARKER} + ) + assert CATEGORY_REMOTE_LOCATION not in ARTIFACT_LIFECYCLE_CATEGORIES + + +@pytest.mark.parametrize( + "value,category,reason", + [ + (_AUTHZ_HEADER, CATEGORY_CREDENTIAL, "authorization header/assignment shape"), + (_GITHUB_TOKEN, CATEGORY_CREDENTIAL, "credential-like value shape"), + ("/home/dev/x.json", CATEGORY_LOCAL_PATH, "local filesystem path"), + ("https://example.com/a", CATEGORY_REMOTE_LOCATION, "raw remote location URL"), + ("/ext_data/run/x", CATEGORY_ORG_MARKER, "internal ext_data path"), + ], +) +def test_classify_returns_an_explicit_category_and_reason( + value: str, category: str, reason: str +) -> None: + match = classify_private_text(value) + assert match is not None + assert match.category == category + assert match.reason == reason + + +@pytest.mark.parametrize( + "value", + [ + "needs owner authorization before delivery", + "weekly cadence digest rendered for goal_42", + "the operator rotated the deploy credentials yesterday", + ], +) +def test_classify_leaves_ordinary_governance_prose_alone(value: str) -> None: + assert classify_private_text(value) is None + + +def test_artifact_lifecycle_policy_lets_a_raw_remote_location_through() -> None: + # The named policy, not an inline regex OR, is what preserves the historical + # verdict: a remote location is recognized but out of policy for this surface. + url = "https://example.com/run-7/metrics.json" + assert classify_private_text(url, categories=ALL_CATEGORIES) is not None + assert classify_private_text(url, categories=ARTIFACT_LIFECYCLE_CATEGORIES) is None + # Through the real entry point the URL still compacts to itself. + assert artifact_lifecycle_compact(url) == url + # A credential is in policy and is dropped by the same entry point. + assert artifact_lifecycle_compact(_GITHUB_TOKEN) is None + + +@pytest.mark.parametrize( + "value,policy,expected", + [ + (_GITHUB_TOKEN, ALL_CATEGORIES, True), + (_GITHUB_TOKEN, ARTIFACT_LIFECYCLE_CATEGORIES, True), + ("https://example.com/a", ALL_CATEGORIES, True), + ("https://example.com/a", ARTIFACT_LIFECYCLE_CATEGORIES, False), + ("weekly digest for goal_42", ALL_CATEGORIES, False), + ], +) +def test_matches_private_text_policy_mirrors_classify( + value: str, policy: frozenset[str], expected: bool +) -> None: + assert matches_private_text_policy(value, categories=policy) is expected + assert (classify_private_text(value, categories=policy) is not None) is expected + + +def test_a_policy_narrows_the_text_owner_patterns_not_only_the_shapes() -> None: + # The category filter has to gate the text-owner patterns as well as the + # relocated shape detectors. Each value below is matched only by a text + # pattern, so a policy that drops that pattern's category must accept it -- + # and an empty policy must recognize nothing at all. + bearer = "the Bearer token expired" + ext_data = "/ext_data/run/x" + assert classify_private_text(bearer).category == CATEGORY_CREDENTIAL + assert classify_private_text(ext_data).category == CATEGORY_ORG_MARKER + assert classify_private_text(bearer, categories=frozenset({CATEGORY_LOCAL_PATH})) is None + assert classify_private_text(ext_data, categories=frozenset({CATEGORY_CREDENTIAL})) is None + assert classify_private_text(bearer, categories=frozenset()) is None + assert classify_private_text(ext_data, categories=frozenset()) is None + assert classify_private_text(_GITHUB_TOKEN, categories=frozenset()) is None + + +@pytest.mark.parametrize("value", ["~/work/model.bin", "path:/srv/data/train.json"]) +def test_path_gap_recognition_is_opt_in(value: str) -> None: + # Direction 3 adds recognition of the local-path shapes the legacy surface + # pattern misses, but defaults off so no surface tightens until it chooses + # the wider policy in a separate, disclosed change. + assert classify_private_text(value) is None + gap = classify_private_text(value, include_path_gaps=True) + assert gap is not None + assert gap.category == CATEGORY_LOCAL_PATH + + +def test_path_gap_recognition_does_not_subsume_the_alias_rule() -> None: + # `~username` has no `/` after the tilde, so the shared gap pattern does not + # flag it; ml_experiment's alias rule does. They are different decisions. + assert classify_private_text("~username/notes", include_path_gaps=True) is None + + +def test_classify_first_text_match_is_identical_to_find_private_text_match() -> None: + # Behavior-preserving pin over the real shared corpus: whenever the legacy + # helper returns a pattern, the classifier's first match is that same object + # (text patterns are checked first, in the pinned order). The classifier may + # additionally flag shape-only values the legacy helper never saw. + corpus = json.loads(CORPUS_PATH.read_text(encoding="utf-8")) + assert corpus["schema_version"] == "public_safe_text_corpus_v0" + tokens = corpus["tokens"] + + def render(template: str) -> str: + def replace(match: re.Match[str]) -> str: + name = match.group(1) + if name in tokens: + return "".join(tokens[name]) + if name.endswith("_LOWER") and name[: -len("_LOWER")] in tokens: + return "".join(tokens[name[: -len("_LOWER")]]).lower() + raise AssertionError(f"corpus placeholder {name} has no token") + + return _PLACEHOLDER_RE.sub(replace, template) + + checked = 0 + for group in ("public_safe", "private_looking"): + for sample in corpus[group]: + value = render(sample["template"]) + checked += 1 + legacy = find_private_text_match(value) + classified = classify_private_text(value, categories=ALL_CATEGORIES) + if legacy is None: + continue + assert classified is not None, sample["id"] + assert classified.pattern is legacy, sample["id"] + assert checked > 0 + + +def test_private_text_patterns_are_the_categorized_patterns_in_order() -> None: + # The compat tuple every existing importer reads is derived from the + # categorized list, same patterns, same order, so find_private_text_match is + # byte-identical to before the relocation. + assert PRIVATE_TEXT_PATTERNS == tuple( + entry.pattern for entry in _CATEGORIZED_PRIVATE_TEXT_PATTERNS + ) + assert len(PRIVATE_TEXT_PATTERNS) == 11 + + +def test_public_safety_reexports_the_same_owner_objects() -> None: + # public_safety consumes the shapes instead of restating them; the objects it + # re-exports are the very ones the owner compiles, so its ~8 importers and + # recursive payload validation see one decision. + assert public_safety.SECRET_LIKE_SURFACE_PATTERN is OWNER_SECRET_LIKE + assert public_safety.LOCAL_PATH_SURFACE_PATTERN is OWNER_LOCAL_PATH + assert public_safety.REMOTE_LOCATION_SURFACE_PATTERN is OWNER_REMOTE_LOCATION + + +def test_ml_experiment_alias_constraint_is_stricter_than_the_shared_classifier() -> None: + # Direction 3: keep ml_experiment's leading-`/`-or-`~` rule as an explicit + # alias constraint. A future single-owner pass must not fold it into the + # shared classifier, because the classifier does not flag these values even + # with path-gap recognition on -- folding would silently lose alias coverage. + for value in ("~username/notes", "/just-a-leading-slash"): + assert classify_private_text(value, include_path_gaps=True) is None + with pytest.raises(ValueError, match="must use a public alias"): + ml_experiment_alias(value, field="dataset_ref") + # Positive control: a bare alias passes the same field. + assert ml_experiment_alias("public-alias-v3", field="dataset_ref") == "public-alias-v3" diff --git a/tests/control_plane/test_public_safety_credential_shape_owner.py b/tests/control_plane/test_public_safety_credential_shape_owner.py index cf76d78fa4..4551af7672 100644 --- a/tests/control_plane/test_public_safety_credential_shape_owner.py +++ b/tests/control_plane/test_public_safety_credential_shape_owner.py @@ -37,7 +37,11 @@ from loopx.extensions.presentation import _plain_text as presentation_text REPOSITORY_ROOT = pathlib.Path(__file__).resolve().parents[2] -OWNER_MODULE = "loopx/control_plane/runtime/public_safety.py" +# Refs #5136 direction 1: the credential-shape owner moved into the shared +# text-classification home. public_safety re-exports SECRET_LIKE_SURFACE_PATTERN +# for its recursive payload validation, so the identity shapes below are still +# declared literally in exactly one module. +OWNER_MODULE = "loopx/public_safe_text.py" # One consumer decides a different question: it rejects a goal id whose *whole* # value is a provider token, so its list is anchored and cannot be reused as a # surface scan. Named here so a new surface copy still fails this test. diff --git a/tests/control_plane/test_remote_location_shape_owner.py b/tests/control_plane/test_remote_location_shape_owner.py index 508f163c2e..3a6d8e16ef 100644 --- a/tests/control_plane/test_remote_location_shape_owner.py +++ b/tests/control_plane/test_remote_location_shape_owner.py @@ -9,10 +9,13 @@ a fifth spelling, and any new object-store scheme had to be found in three places that nothing linked together. -The owner now holds the single pattern and each site keeps its own error text and its -own threshold policy - the sites reject at different lengths and one of them adds -vendor-specific markers, which is per-surface policy, not a duplicate decision. The -literal scan below is what stops the copies from growing back. +The single owner is now the shared text-classification home +(`loopx/public_safe_text.py`, Refs #5136 direction 1); `public_safety` re-exports the +compiled pattern for its recursive payload validation, so the scheme list still lives +in one module. Each site keeps its own error text and its own threshold policy - the +sites reject at different lengths and one of them adds vendor-specific markers, which +is per-surface policy, not a duplicate decision. The literal scan below is what stops +the copies from growing back. """ from __future__ import annotations @@ -65,9 +68,13 @@ def _source_text_spelling_the_scheme_list() -> list[str]: def test_the_pattern_is_compiled_once_by_the_owner() -> None: # The negative control: a spelling that survives in any module other than the - # owner is the exact regression this file exists to catch. + # owner is the exact regression this file exists to catch. Refs #5136 + # direction 1: the single owner is now the shared text-classification home + # (loopx/public_safe_text.py). public_safety re-exports the compiled pattern + # for its recursive payload validation, so the literal scheme list still + # lives in exactly one module. assert _source_text_spelling_the_scheme_list() == [ - "loopx/control_plane/runtime/public_safety.py" + "loopx/public_safe_text.py" ]