From 5bc85a2bc54e3a12d0d636ecb4124b83198ad968 Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Fri, 24 Jul 2026 17:07:53 +0900 Subject: [PATCH 01/12] fix: harden sensitive runtime logging --- .../skills/sensitive-logging-audit/SKILL.md | 93 ++ .../agents/openai.yaml | 4 + .../references/redaction-validation.md | 78 ++ .../scripts/_inventory.py | 1138 +++++++++++++++++ .../scripts/inventory_logging.py | 156 +++ .../scripts/test_inventory.py | 810 ++++++++++++ src/agents/agent.py | 11 +- .../experimental/codex/codex_tool.py | 18 +- .../memory/advanced_sqlite_session.py | 58 +- src/agents/extensions/memory/dapr_session.py | 4 +- .../extensions/sandbox/blaxel/mounts.py | 3 +- .../extensions/sandbox/blaxel/sandbox.py | 25 +- .../extensions/sandbox/cloudflare/sandbox.py | 31 +- .../extensions/sandbox/daytona/sandbox.py | 3 +- src/agents/extensions/sandbox/e2b/sandbox.py | 37 +- .../extensions/sandbox/modal/sandbox.py | 11 +- src/agents/logger.py | 222 ++++ src/agents/mcp/_logging.py | 52 + src/agents/mcp/manager.py | 33 +- src/agents/mcp/server.py | 111 +- src/agents/mcp/util.py | 51 +- .../openai_responses_compaction_session.py | 20 +- src/agents/models/openai_chatcompletions.py | 12 +- src/agents/models/openai_responses.py | 42 +- src/agents/realtime/agent.py | 8 +- src/agents/realtime/openai_realtime.py | 4 +- src/agents/realtime/session.py | 53 +- src/agents/result.py | 8 +- src/agents/run.py | 12 +- src/agents/run_internal/model_retry.py | 4 +- src/agents/run_internal/run_loop.py | 22 +- .../run_internal/session_persistence.py | 34 +- src/agents/run_internal/tool_execution.py | 20 +- src/agents/run_internal/turn_resolution.py | 18 +- src/agents/run_state.py | 6 +- src/agents/sandbox/memory/manager.py | 9 +- src/agents/sandbox/runtime.py | 7 +- src/agents/sandbox/sandboxes/unix_local.py | 11 +- src/agents/sandbox/session/manager.py | 26 +- src/agents/tool.py | 4 +- src/agents/tracing/processors.py | 42 +- src/agents/tracing/provider.py | 68 +- src/agents/util/_error_tracing.py | 3 + src/agents/voice/pipeline.py | 12 +- src/agents/voice/result.py | 8 +- .../experiemental/codex/test_codex_tool.py | 31 +- .../memory/test_advanced_sqlite_session.py | 29 +- tests/extensions/sandbox/test_cloudflare.py | 12 +- tests/extensions/sandbox/test_e2b.py | 37 +- tests/mcp/test_mcp_server_manager.py | 96 ++ tests/mcp/test_mcp_util.py | 34 +- tests/mcp/test_tool_filtering.py | 33 + ...est_openai_responses_compaction_session.py | 9 +- tests/realtime/test_agent.py | 24 + tests/realtime/test_session.py | 30 +- tests/test_agent_as_tool.py | 11 +- tests/test_agent_runner.py | 40 + tests/test_computer_tool_lifecycle.py | 28 + tests/test_error_logging_redaction.py | 202 ++- tests/test_trace_processor.py | 41 +- tests/tracing/test_tracing_env_disable.py | 22 + tests/voice/test_pipeline.py | 80 ++ 62 files changed, 3817 insertions(+), 344 deletions(-) create mode 100644 .agents/skills/sensitive-logging-audit/SKILL.md create mode 100644 .agents/skills/sensitive-logging-audit/agents/openai.yaml create mode 100644 .agents/skills/sensitive-logging-audit/references/redaction-validation.md create mode 100644 .agents/skills/sensitive-logging-audit/scripts/_inventory.py create mode 100644 .agents/skills/sensitive-logging-audit/scripts/inventory_logging.py create mode 100644 .agents/skills/sensitive-logging-audit/scripts/test_inventory.py create mode 100644 src/agents/mcp/_logging.py diff --git a/.agents/skills/sensitive-logging-audit/SKILL.md b/.agents/skills/sensitive-logging-audit/SKILL.md new file mode 100644 index 0000000000..201ac2fdf1 --- /dev/null +++ b/.agents/skills/sensitive-logging-audit/SKILL.md @@ -0,0 +1,93 @@ +--- +name: sensitive-logging-audit +description: Audit and fix sensitive-data exposure through Python runtime logging in openai-agents-python. Use when reviewing logging, print, warnings, stderr, or traceback output; checking OPENAI_AGENTS_DONT_LOG_MODEL_DATA or OPENAI_AGENTS_DONT_LOG_TOOL_DATA coverage; investigating model, tool, Realtime, MCP, session, sandbox, tracing, or arbitrary exception data in logs; or carrying an audit through fixes, adversarial tests, and repository verification. +--- + +# Sensitive Logging Audit + +## Objective + +Complete the remediation, not only the scan. Inventory runtime output sinks, classify every dynamic value, fix every demonstrated model/tool-data leak in scope, add adversarial regression tests, and run the applicable repository gates. + +Treat the inventory as a conservative review ledger, not automatic taint analysis. Receiver provenance and policy guards improve prioritization but never prove that a dynamic value is safe. + +## Workflow + +### 1. Establish the baseline + +- Work in the current checkout and preserve unrelated changes. +- Record `git status --short --branch` and the current commit. +- Read `src/agents/_debug.py`, `src/agents/logger.py`, and existing policy-aware helpers before judging call sites. +- Treat exception messages, arguments, tracebacks, causes, contexts, notes, and arbitrary values as potentially sensitive. +- Treat URL-derived display names as structured sensitive values. MCP server names may embed endpoint credentials, query parameters, or fragments even when the log does not contain a model/tool payload object. + +Run the detector tests before relying on its output: + +```bash +uv run python .agents/skills/sensitive-logging-audit/scripts/test_inventory.py +``` + +Create a baseline from the repository root: + +```bash +uv run python .agents/skills/sensitive-logging-audit/scripts/inventory_logging.py \ + --format json --output /tmp/sensitive-logging-before.json +uv run python .agents/skills/sensitive-logging-audit/scripts/inventory_logging.py \ + --summary-only +``` + +### 2. Classify every dynamic sink + +Review the complete JSON ledger. Prioritize raw output, caught exceptions, `logger.exception`, `exc_info`, `extra`, supplemental formatting arguments, and unknown receivers. + +Assign one disposition to every dynamic fingerprint group: + +- `model`: model requests, responses, Realtime events, or derived values. +- `tool`: tool arguments, outputs, MCP data, tool events, or derived values. +- `model+tool`: either class may reach the sink. +- `operational`: proven to contain only non-sensitive SDK metadata. +- `intentional-output`: explicitly user-facing output rather than diagnostics. +- `uncertain`: source tracing is incomplete; investigate before deciding. + +Record the fingerprint, group count, disposition, evidence, and action. Do not classify from a variable name or message text alone. Trace producers, callbacks, formatters, and exception ownership. + +### 3. Fix demonstrated leaks + +Before changing runtime behavior, use `$implementation-strategy`. Implement the narrowest shared-boundary fix. + +- Apply `_debug.DONT_LOG_MODEL_DATA` and `_debug.DONT_LOG_TOOL_DATA` before formatting or inspecting sensitive values. +- Redact `model+tool` values when either relevant flag disables data logging. +- Preserve existing diagnostics when sensitive-data logging is explicitly enabled. +- In redacted mode, emit a fixed message. Do not inspect or pass the sensitive object through `msg`, `args`, `extra`, or `exc_info`. +- In tool-redacted mode, use fixed MCP lifecycle and filter messages without reading `server.name` or including tool names. In diagnostic mode, sanitize URL-derived server names by removing credentials, query parameters, and fragments while retaining ordinary names. Do not mutate the server's public `name` or connection URL. +- Keep logging failure from changing fallback, cleanup, event emission, rejection, or cancellation behavior. +- Leave operational and intentional user-facing values unchanged when their safety is demonstrated. + +### 4. Add adversarial regressions + +Read [the Python redaction validation matrix](references/redaction-validation.md) and test every changed caller boundary. Helper-only tests do not prove that callers use the helper correctly. + +### 5. Re-audit the whole tree + +Run the inventory again and compare it to the baseline: + +```bash +uv run python .agents/skills/sensitive-logging-audit/scripts/inventory_logging.py \ + --compare /tmp/sensitive-logging-before.json \ + --format json --output /tmp/sensitive-logging-after.json +``` + +Inspect every new, removed, count-changed, or classification-changed fingerprint group. Classification changes include policy, shape, sink kind, confidence, method, and caught-value status. Duplicate groups deliberately require group-level re-review instead of inheriting an order-dependent disposition. + +The completion report must state total and dynamic sink counts, confirmed leaks fixed, retained candidates and evidence, duplicate groups, verification results, and remaining uncertainty. + +### 6. Run close-out gates + +- Run the detector tests and skill validation after changing the skill. +- For runtime code, tests, examples, or build/test behavior, use `$code-change-verification` after the final fix. +- Use `$pr-draft-summary` when required by the repository instructions. +- Stop after local changes and verification unless the user explicitly requests a remote action. + +## Reporting + +Lead with whether real leaks were found and fixed. Separate confirmed leaks from conservative candidates and intentional output. Never equate a clean inventory shape or a recognized policy guard with proof that all dynamic values are non-sensitive. diff --git a/.agents/skills/sensitive-logging-audit/agents/openai.yaml b/.agents/skills/sensitive-logging-audit/agents/openai.yaml new file mode 100644 index 0000000000..1f0e17b1bd --- /dev/null +++ b/.agents/skills/sensitive-logging-audit/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Sensitive Logging Audit" + short_description: "Audit and fix sensitive Python logging paths" + default_prompt: "Use $sensitive-logging-audit to inventory, verify, and fix sensitive logging leaks in this repository." diff --git a/.agents/skills/sensitive-logging-audit/references/redaction-validation.md b/.agents/skills/sensitive-logging-audit/references/redaction-validation.md new file mode 100644 index 0000000000..7a658b0f8e --- /dev/null +++ b/.agents/skills/sensitive-logging-audit/references/redaction-validation.md @@ -0,0 +1,78 @@ +# Python sensitive logging validation + +The inventory is deliberately broader than a vulnerability detector. It finds confirmed SDK logger calls, raw output, exact policy-aware helpers, and unknown receivers with logging-like method names. Review every dynamic finding; do not treat a recognized receiver, helper, or guard as proof that its values are safe. + +## Required validation matrix + +Test every changed sensitive caller boundary in both redacted and diagnostic modes. Use a unique sentinel for each source and inspect both rendered output and the complete `LogRecord`. + +| Case | Model flag | Tool flag | Value | Required assertion | +| --- | --- | --- | --- | --- | +| Model redaction | on | off | `Exception(secret)` | No sentinel or exception object remains in the record | +| Tool redaction | off | on | `Exception(secret)` | No sentinel or exception object remains in the record | +| Both redacted | on | on | model and tool values | Neither sentinel remains anywhere in the record | +| Diagnostic mode | off | off | ordinary exception | Existing diagnostic detail and traceback behavior remain | +| Hostile string | applicable | applicable | object whose `__str__` raises or returns a secret | Logging does not fail or reveal the secret | +| Hostile repr | applicable | applicable | object whose `__repr__` raises or returns a secret | Logging does not fail or reveal the secret | +| Hostile class access | applicable | applicable | exception overriding `__getattribute__` | Redacted logging does not inspect the exception | +| Exception chain | applicable | applicable | `__cause__`, `__context__`, notes, or `ExceptionGroup` containing secrets | No chained secret is attached or rendered | +| Supplemental arguments | applicable | applicable | fixed message plus secret formatting argument | Formatting arguments are omitted in redacted mode | +| Extra payload | applicable | applicable | `extra={"detail": secret}` | Secret `LogRecord` attributes are omitted | +| Traceback payload | applicable | applicable | `exc_info=True` or an exception tuple | `exc_info` and `exc_text` are absent in redacted mode | +| MCP server or tool name | tool | on | path token or custom-name sentinel | Log uses a fixed message and does not read or attach the name | +| URL-derived MCP name | tool | off | URL credentials, query, and fragment | Log retains only scheme, host, port, and path; the runtime value is unchanged | + +Also test the observable caller behavior after logging. Redaction is incorrect if it prevents a fallback result, cleanup, event emission, rejection, or cancellation from completing. + +## Inspect the full LogRecord + +Do not assert only against `caplog.text` or a mock call converted to a string. In redacted mode, inspect at least: + +- `record.msg` +- `record.args` +- `record.exc_info` +- `record.exc_text` +- values added through `record.__dict__` +- the final output of a real `logging.Formatter` + +The sensitive object itself must not remain attached even when its string representation is absent. A custom handler or exporter may inspect raw record fields. + +## Review procedure + +1. Run the inventory against all of `src/agents`. +2. Review raw output and unknown receivers first. +3. Review caught values, `logger.exception`, `exc_info`, `extra`, and formatting arguments. +4. Trace model, tool, Realtime, MCP, session, sandbox, voice, tracing, and cleanup values to their producers. +5. Classify intentional output separately from diagnostics; do not silently exempt `print` or warnings. +6. Add focused tests at every changed caller boundary. +7. Re-run the inventory and compare fingerprint groups to the baseline. +8. Re-review any duplicate group whose count changed. + +The detector supports a completeness claim for the repository's direct Python output shapes. It does not prove arbitrary runtime data flow, dynamically installed handlers, monkey-patched logger methods, or reflective calls. + +## Review ledger shape + +Use a temporary JSON ledger when the audit is large: + +```json +{ + "reviews": [ + { + "group_fingerprint": "0123456789ab", + "group_count": 1, + "disposition": "tool", + "evidence": "The value originates from the function tool input JSON.", + "action": "Guard formatting and arguments with DONT_LOG_TOOL_DATA." + } + ] +} +``` + +Validate it with: + +```bash +uv run python .agents/skills/sensitive-logging-audit/scripts/inventory_logging.py \ + --validate-review /tmp/sensitive-logging-review.json --summary-only +``` + +Allowed dispositions are `model`, `tool`, `model+tool`, `operational`, `intentional-output`, and `uncertain`. diff --git a/.agents/skills/sensitive-logging-audit/scripts/_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/_inventory.py new file mode 100644 index 0000000000..12a39f6d75 --- /dev/null +++ b/.agents/skills/sensitive-logging-audit/scripts/_inventory.py @@ -0,0 +1,1138 @@ +from __future__ import annotations + +import ast +import hashlib +import importlib.util +import re +from collections import Counter, defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +LOG_METHODS = { + "critical", + "debug", + "error", + "exception", + "fatal", + "info", + "log", + "warn", + "warning", +} +SDK_LOGGER_MODULES = {"agents.logger", "agents.tracing.logger"} +POLICY_MODULE = "agents._debug" +POLICY_NAMES = { + "DONT_LOG_MODEL_DATA": "model", + "DONT_LOG_TOOL_DATA": "tool", +} +KNOWN_HELPERS = { + "agents.logger.log_model_action_debug": "model", + "agents.logger.log_model_action_error": "model", + "agents.logger.log_model_action_warning": "model", + "agents.logger.log_model_and_tool_action_debug": "model+tool", + "agents.logger.log_model_and_tool_action_error": "model+tool", + "agents.logger.log_model_and_tool_action_warning": "model+tool", + "agents.logger.log_tool_action_debug": "tool", + "agents.logger.log_tool_action_error": "tool", + "agents.logger.log_tool_action_warning": "tool", + "agents.run_internal.tool_execution.log_tool_action_error": "tool", +} +SENSITIVE_HELPER_METHODS = {name.rsplit(".", 1)[-1] for name in KNOWN_HELPERS} +RAW_MODULE_METHODS = { + "pprint": {"pprint"}, + "traceback": {"print_exc", "print_exception"}, + "warnings": {"warn", "warn_explicit"}, +} +DISPOSITIONS = { + "intentional-output", + "model", + "model+tool", + "operational", + "tool", + "uncertain", +} +COMPARISON_CLASSIFICATION_FIELDS = ( + "kind", + "confidence", + "method", + "shape", + "policy", + "catch_value", +) + + +def _is_sdk_logger_module_or_parent(module: str) -> bool: + return any( + candidate == module or candidate.startswith(f"{module}.") + for candidate in SDK_LOGGER_MODULES + ) + + +@dataclass +class Finding: + group_fingerprint: str + fingerprint: str + file: str + line: int + column: int + kind: str + confidence: str + method: str + shape: str + policy: str + catch_value: str | None + context: str + signals: list[str] + call: str + site_index: int = 0 + group_count: int = 1 + identity_quality: str = "unique" + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def normalize_path(path: str | Path) -> str: + return str(path).replace("\\", "/") + + +def collect_source_files(roots: Sequence[str | Path]) -> list[Path]: + files: set[Path] = set() + for root_value in roots: + root = Path(root_value).resolve() + if root.is_file(): + if root.suffix == ".py": + files.add(root) + continue + if not root.is_dir(): + raise FileNotFoundError(f"Inventory root does not exist: {root_value}") + for path in root.rglob("*.py"): + relative_parts = path.relative_to(root).parts + if any(part.startswith(".") or part == "__pycache__" for part in relative_parts): + continue + files.add(path.resolve()) + return sorted(files) + + +def module_identity(file_path: str) -> tuple[str, str]: + path = normalize_path(file_path) + marker = "/src/" + relative = path.split(marker, 1)[1] if marker in path else path.removeprefix("src/") + parts = relative.split("/") + if parts[-1] == "__init__.py": + module = ".".join(parts[:-1]) + return module, module + parts[-1] = parts[-1].removesuffix(".py") + module = ".".join(parts) + package = module.rsplit(".", 1)[0] if "." in module else module + return module, package + + +def resolve_import(module: str | None, level: int, package: str) -> str: + if level == 0: + return module or "" + name = "." * level + (module or "") + try: + return importlib.util.resolve_name(name, package) + except (ImportError, ValueError): + return name + + +def expression_key(node: ast.AST) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + receiver = expression_key(node.value) + return f"{receiver}.{node.attr}" if receiver else None + if isinstance(node, ast.Subscript) and isinstance(node.slice, ast.Constant): + receiver = expression_key(node.value) + if receiver and isinstance(node.slice.value, str | int): + return f"{receiver}[{node.slice.value!r}]" + return None + + +def target_keys(node: ast.AST) -> list[str]: + key = expression_key(node) + if key: + return [key] + if isinstance(node, ast.List | ast.Tuple): + return [key for element in node.elts for key in target_keys(element)] + return [] + + +def target_value_pairs(target: ast.AST, value: ast.AST) -> list[tuple[str, ast.AST | None]]: + key = expression_key(target) + if key: + return [(key, value)] + if isinstance(target, ast.List | ast.Tuple): + if isinstance(value, ast.List | ast.Tuple) and len(target.elts) == len(value.elts): + return [ + pair + for target_element, value_element in zip(target.elts, value.elts, strict=True) + for pair in target_value_pairs(target_element, value_element) + ] + return [(key, None) for key in target_keys(target)] + return [] + + +def iter_definitions( + tree: ast.AST, +) -> Iterable[tuple[ast.AST, list[tuple[str, ast.AST | None]]]]: + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + yield node, target_value_pairs(target, node.value) + elif isinstance(node, ast.AnnAssign) and node.value is not None: + yield node, target_value_pairs(node.target, node.value) + elif isinstance(node, ast.AugAssign): + yield node, [(key, None) for key in target_keys(node.target)] + elif isinstance(node, ast.NamedExpr): + yield node, target_value_pairs(node.target, node.value) + elif isinstance(node, ast.For | ast.AsyncFor): + yield node, [(key, None) for key in target_keys(node.target)] + elif isinstance(node, ast.With | ast.AsyncWith): + targets = [ + (key, None) + for item in node.items + if item.optional_vars is not None + for key in target_keys(item.optional_vars) + ] + if targets: + yield node, targets + elif isinstance(node, ast.ExceptHandler) and node.name: + yield node, [(node.name, None)] + elif isinstance(node, ast.Delete): + yield node, [(key, None) for target in node.targets for key in target_keys(target)] + + +def bound_names(node: ast.AST) -> set[str]: + if isinstance(node, ast.Name): + return {node.id} + if isinstance(node, ast.List | ast.Tuple): + return {name for element in node.elts for name in bound_names(element)} + return set() + + +class Facts: + def __init__(self, tree: ast.Module, file_path: str): + self.values: dict[ast.AST, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set)) + self.bindings: dict[ast.AST, set[str]] = defaultdict(set) + self.node_scopes: dict[ast.AST, ast.AST] = {} + self.scope_parents: dict[ast.AST, ast.AST | None] = {tree: None} + self.parents = make_parent_map(tree) + self.definitions = list(iter_definitions(tree)) + self.definition_values: dict[ast.AST, dict[str, list[tuple[ast.AST, ast.AST | None]]]] = ( + defaultdict(lambda: defaultdict(list)) + ) + self.import_policy_values: dict[ast.AST, dict[str, set[str]]] = defaultdict(dict) + self._policy_lookup_stack: set[tuple[int, str]] = set() + self.module_name, self.package_name = module_identity(file_path) + self._index_scopes(tree) + self._collect_bindings(tree) + for definition, targets in self.definitions: + scope = self._scope_for(definition) + for key, value in targets: + self.definition_values[scope][key].append((definition, value)) + self._collect_imports(tree) + self._resolve_definitions(tree) + + def _index_scopes(self, tree: ast.Module) -> None: + scope_types = ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda | ast.ClassDef + + def visit(node: ast.AST, scope: ast.AST) -> None: + self.node_scopes[node] = scope + for child in ast.iter_child_nodes(node): + if isinstance(child, scope_types): + self.node_scopes[child] = scope + parent_scope = ( + self.scope_parents[scope] if isinstance(scope, ast.ClassDef) else scope + ) + self.scope_parents[child] = parent_scope + for descendant in ast.iter_child_nodes(child): + visit(descendant, child) + else: + visit(child, scope) + + visit(tree, tree) + + def _scope_for(self, node: ast.AST) -> ast.AST: + return self.node_scopes[node] + + def _collect_bindings(self, tree: ast.Module) -> None: + for node in ast.walk(tree): + scope = self._scope_for(node) + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): + self.bindings[scope].add(node.name) + if isinstance(node, ast.arg): + self.bindings[scope].add(node.arg) + elif isinstance(node, ast.Assign): + for target in node.targets: + self.bindings[scope].update(bound_names(target)) + elif isinstance(node, ast.AnnAssign | ast.AugAssign | ast.NamedExpr): + self.bindings[scope].update(bound_names(node.target)) + elif isinstance(node, ast.For | ast.AsyncFor): + self.bindings[scope].update(bound_names(node.target)) + elif isinstance(node, ast.comprehension): + self.bindings[scope].update(bound_names(node.target)) + elif isinstance(node, ast.With | ast.AsyncWith): + for item in node.items: + if item.optional_vars is not None: + self.bindings[scope].update(bound_names(item.optional_vars)) + elif isinstance(node, ast.ExceptHandler) and node.name: + self.bindings[scope].add(node.name) + elif isinstance(node, ast.Import): + for alias in node.names: + self.bindings[scope].add(alias.asname or alias.name.split(".", 1)[0]) + elif isinstance(node, ast.ImportFrom): + for alias in node.names: + if alias.name != "*": + self.bindings[scope].add(alias.asname or alias.name) + + def add(self, scope: ast.AST, key: str, values: Iterable[str]) -> bool: + before = len(self.values[scope][key]) + self.values[scope][key].update(values) + return len(self.values[scope][key]) != before + + def _lookup(self, node: ast.AST, key: str) -> set[str]: + scope: ast.AST | None = self._scope_for(node) + root = key.split(".", 1)[0].split("[", 1)[0] + while scope is not None: + if key in self.values[scope]: + result = set(self.values[scope][key]) + result = {fact for fact in result if not fact.startswith("policy")} + result.update(self._lookup_policy(node, key)) + return result + if root in self.bindings[scope]: + return self._lookup_policy(node, key) + scope = self.scope_parents[scope] + return set() + + def _lookup_policy(self, node: ast.AST, key: str) -> set[str]: + token = (id(node), key) + if token in self._policy_lookup_stack: + return set() + + use_scope = self._scope_for(node) + scope: ast.AST | None = use_scope + root = key.split(".", 1)[0].split("[", 1)[0] + while scope is not None: + definitions = self.definition_values[scope].get(key, []) + if definitions: + if scope is not use_scope: + return set() + preceding = [ + definition + for definition in definitions + if self._node_position(definition[0]) < self._node_position(node) + ] + if not preceding: + return set() + assignment, value = max(preceding, key=lambda item: self._node_position(item[0])) + if not self._definition_precedes_in_same_block(assignment, node): + return set() + if value is None: + return set() + self._policy_lookup_stack.add(token) + try: + return { + fact + for fact in self.infer(value) + if fact.startswith(("policy:", "policy-exact:")) + } + finally: + self._policy_lookup_stack.remove(token) + + imported = self.import_policy_values[scope].get(key) + if imported is not None: + return set(imported) + if root in self.bindings[scope]: + return set() + scope = self.scope_parents[scope] + return set() + + @staticmethod + def _node_position(node: ast.AST) -> tuple[int, int]: + return (getattr(node, "lineno", -1), getattr(node, "col_offset", -1)) + + def _definition_precedes_in_same_block(self, definition: ast.AST, use: ast.AST) -> bool: + if isinstance(definition, ast.For | ast.AsyncFor | ast.With | ast.AsyncWith): + child = use + while child in self.parents and self.parents[child] is not definition: + child = self.parents[child] + if child in definition.body: + return True + if isinstance(definition, ast.For | ast.AsyncFor) and child in definition.orelse: + return True + elif isinstance(definition, ast.ExceptHandler): + child = use + while child in self.parents and self.parents[child] is not definition: + child = self.parents[child] + if child in definition.body: + return True + + parent = self.parents.get(definition) + if parent is None: + return False + for _, value in ast.iter_fields(parent): + if not isinstance(value, list) or definition not in value: + continue + definition_index = value.index(definition) + child = use + while child in self.parents and self.parents[child] is not parent: + child = self.parents[child] + return child in value and definition_index < value.index(child) + return False + + def _collect_imports(self, tree: ast.Module) -> None: + for node in ast.walk(tree): + scope = self._scope_for(node) + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + full_name = f"{self.module_name}.{node.name}" + if scope is tree and full_name in KNOWN_HELPERS: + self.add(scope, node.name, {f"helper:{KNOWN_HELPERS[full_name]}"}) + elif isinstance(node, ast.Import): + for alias in node.names: + bound = alias.asname or alias.name.split(".", 1)[0] + if alias.name in {"logging", "pprint", "sys", "traceback", "warnings"}: + self.add(scope, bound, {f"module:{alias.name}"}) + elif alias.name == POLICY_MODULE: + self.add( + scope, + alias.asname or alias.name, + {f"module:{POLICY_MODULE}"}, + ) + elif alias.name in SDK_LOGGER_MODULES: + imported_module = ( + alias.name if alias.asname else alias.name.split(".", 1)[0] + ) + self.add(scope, bound, {f"module:{imported_module}"}) + elif isinstance(node, ast.ImportFrom): + module = resolve_import(node.module, node.level, self.package_name) + for alias in node.names: + if alias.name == "*": + continue + bound = alias.asname or alias.name + full_name = f"{module}.{alias.name}" if module else alias.name + facts: set[str] = set() + if module == "logging": + if alias.name == "getLogger": + facts.add("factory:logger") + elif alias.name in LOG_METHODS: + facts.add(f"method:logger:{alias.name}") + elif alias.name == "Logger": + facts.add("type:logger") + if module == "functools" and alias.name == "partial": + facts.add("factory:partial") + if module in SDK_LOGGER_MODULES and alias.name == "logger": + facts.add("logger") + if full_name in SDK_LOGGER_MODULES: + facts.add(f"module:{full_name}") + if full_name == POLICY_MODULE: + facts.add(f"module:{POLICY_MODULE}") + if module == POLICY_MODULE and alias.name in POLICY_NAMES: + policy = POLICY_NAMES[alias.name] + policy_facts = {f"policy:{policy}", f"policy-exact:{policy}"} + facts.update(policy_facts) + self.import_policy_values[scope][bound] = policy_facts + if full_name in KNOWN_HELPERS: + facts.add(f"helper:{KNOWN_HELPERS[full_name]}") + if module in RAW_MODULE_METHODS and alias.name in RAW_MODULE_METHODS[module]: + facts.add(f"method:raw:{module}.{alias.name}") + if module == "sys" and alias.name in {"stdout", "stderr"}: + facts.add(f"stream:{alias.name}") + if facts: + self.add(scope, bound, facts) + + def infer(self, node: ast.AST) -> set[str]: + key = expression_key(node) + result = self._lookup(node, key) if key else set() + if isinstance(node, ast.Name): + if node.id == "print": + result.add("method:raw:print") + return result + if isinstance(node, ast.Attribute): + receiver_facts = self.infer(node.value) + for fact in receiver_facts: + if fact == "module:logging": + if node.attr == "getLogger": + result.add("factory:logger") + elif node.attr == "Logger": + result.add("type:logger") + elif node.attr in LOG_METHODS: + result.add(f"method:logger:{node.attr}") + elif fact == "logger" and node.attr in LOG_METHODS: + result.add(f"method:logger:{node.attr}") + elif fact == f"module:{POLICY_MODULE}" and node.attr in POLICY_NAMES: + policy = POLICY_NAMES[node.attr] + result.update({f"policy:{policy}", f"policy-exact:{policy}"}) + elif fact.startswith("module:"): + module = fact.split(":", 1)[1] + qualified_name = f"{module}.{node.attr}" + if _is_sdk_logger_module_or_parent(qualified_name): + result.add(f"module:{qualified_name}") + if module in SDK_LOGGER_MODULES: + helper_policy = KNOWN_HELPERS.get(qualified_name) + if helper_policy is not None: + result.add(f"helper:{helper_policy}") + if node.attr == "logger": + result.add("logger") + if module in RAW_MODULE_METHODS and node.attr in RAW_MODULE_METHODS[module]: + result.add(f"method:raw:{module}.{node.attr}") + if module == "sys" and node.attr in {"stdout", "stderr"}: + result.add(f"stream:{node.attr}") + elif fact.startswith("stream:"): + if node.attr == "buffer": + result.add(fact) + elif node.attr in {"write", "writelines"}: + stream = fact.split(":", 1)[1] + result.add(f"method:raw:{stream}.{node.attr}") + return result + if isinstance(node, ast.Call): + callee_facts = self.infer(node.func) + if "factory:logger" in callee_facts or "type:logger" in callee_facts: + result.add("logger") + if self._is_partial(callee_facts) and node.args: + method_facts = { + fact for fact in self.infer(node.args[0]) if fact.startswith("method:") + } + result.update(method_facts) + if len(node.args) == 1 and not node.keywords: + result.add("partial-shape:empty") + else: + bound_call = ast.Call( + func=node.args[0], + args=node.args[1:], + keywords=node.keywords, + ) + for method_fact in method_facts: + _, _, method = method_fact.split(":", 2) + result.add(f"partial-shape:{call_shape(bound_call, method)}") + return result + if isinstance(node, ast.BoolOp): + for value in node.values: + result.update(self.infer(value)) + result = {fact for fact in result if not fact.startswith("policy-exact:")} + return result + if isinstance(node, ast.IfExp): + result.update(self.infer(node.body)) + result.update(self.infer(node.orelse)) + result = {fact for fact in result if not fact.startswith("policy-exact:")} + return result + if isinstance(node, ast.UnaryOp): + result.update(self.infer(node.operand)) + return {fact for fact in result if not fact.startswith("policy-exact:")} + if isinstance(node, ast.Compare): + result.update(self.infer(node.left)) + for comparator in node.comparators: + result.update(self.infer(comparator)) + return {fact for fact in result if not fact.startswith("policy-exact:")} + if isinstance(node, ast.Tuple | ast.List | ast.Set): + for element in node.elts: + result.update(self.infer(element)) + return result + return result + + @staticmethod + def _is_partial(facts: set[str]) -> bool: + return "factory:partial" in facts + + def _resolve_definitions(self, tree: ast.Module) -> None: + self._seed_special_attributes(tree) + changed = True + while changed: + changed = False + for definition, targets in self.definitions: + scope = self._scope_for(definition) + for key, value in targets: + inferred = self.infer(value) if value is not None else set() + changed |= self.add(scope, key, inferred) + + def _seed_special_attributes(self, tree: ast.Module) -> None: + for node in ast.walk(tree): + if not isinstance(node, ast.Import): + continue + scope = self._scope_for(node) + for alias in node.names: + if alias.name == "functools": + bound = alias.asname or "functools" + self.add(scope, f"{bound}.partial", {"factory:partial"}) + + +def make_parent_map(tree: ast.AST) -> dict[ast.AST, ast.AST]: + return {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)} + + +def normalize_node(node: ast.AST, source: str) -> str: + segment = ast.get_source_segment(source, node) + if segment is None: + segment = ast.dump(node, annotate_fields=True, include_attributes=False) + return re.sub(r"\s+", " ", segment).strip() + + +def node_references_name(node: ast.AST, name: str) -> bool: + return any(isinstance(child, ast.Name) and child.id == name for child in ast.walk(node)) + + +def branch_for_child(parent: ast.AST, child: ast.AST) -> tuple[ast.AST, bool] | None: + if isinstance(parent, ast.If): + if child in parent.body: + return parent.test, True + if child in parent.orelse: + return parent.test, False + if isinstance(parent, ast.IfExp): + if child is parent.body: + return parent.test, True + if child is parent.orelse: + return parent.test, False + return None + + +def possible_boolean_results(node: ast.AST, facts: Facts, policy: str, value: bool) -> set[bool]: + if f"policy-exact:{policy}" in facts.infer(node): + key = expression_key(node) + if key or isinstance(node, ast.Name): + return {value} + if isinstance(node, ast.Constant) and isinstance(node.value, bool): + return {node.value} + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return {not item for item in possible_boolean_results(node.operand, facts, policy, value)} + if isinstance(node, ast.BoolOp): + child_results = [ + possible_boolean_results(item, facts, policy, value) for item in node.values + ] + outcomes = {False, True} + if isinstance(node.op, ast.And): + outcomes = {all(values) for values in _boolean_product(child_results)} + elif isinstance(node.op, ast.Or): + outcomes = {any(values) for values in _boolean_product(child_results)} + return outcomes + if ( + isinstance(node, ast.Compare) + and len(node.ops) == 1 + and len(node.comparators) == 1 + and isinstance(node.ops[0], ast.Eq | ast.NotEq | ast.Is | ast.IsNot) + ): + left = possible_boolean_results(node.left, facts, policy, value) + right = possible_boolean_results(node.comparators[0], facts, policy, value) + equality = isinstance(node.ops[0], ast.Eq | ast.Is) + return {(a == b) if equality else (a != b) for a in left for b in right} + return {False, True} + + +def _boolean_product(values: Sequence[set[bool]]) -> list[tuple[bool, ...]]: + products: list[tuple[bool, ...]] = [()] + for options in values: + products = [prefix + (option,) for prefix in products for option in options] + return products + + +def block_always_exits(statements: Sequence[ast.stmt]) -> bool: + if not statements: + return False + final = statements[-1] + if isinstance(final, ast.Return | ast.Raise | ast.Break | ast.Continue): + return True + if isinstance(final, ast.If): + return block_always_exits(final.body) and block_always_exits(final.orelse) + return False + + +def continuation_guards( + parent: ast.AST, + child: ast.AST, + facts: Facts, +) -> set[str]: + guards: set[str] = set() + for field_name in ("body", "orelse", "finalbody"): + block = getattr(parent, field_name, None) + if not isinstance(block, list) or child not in block: + continue + child_index = block.index(child) + for statement in block[:child_index]: + if not isinstance(statement, ast.If): + continue + body_exits = block_always_exits(statement.body) + else_exits = block_always_exits(statement.orelse) + if body_exits == else_exits: + continue + continuing_branch = not body_exits + for policy in ("model", "tool"): + has_policy = any( + f"policy:{policy}" in facts.infer(candidate) + for candidate in ast.walk(statement.test) + ) + if has_policy and continuing_branch not in possible_boolean_results( + statement.test, facts, policy, True + ): + guards.add(policy) + return guards + + +def guarded_policy(node: ast.AST, parents: Mapping[ast.AST, ast.AST], facts: Facts) -> str: + guards: set[str] = set() + child = node + current = parents.get(node) + while current is not None: + branch = branch_for_child(current, child) + if branch: + condition, branch_value = branch + for policy in ("model", "tool"): + if any( + f"policy:{policy}" in facts.infer(candidate) + for candidate in ast.walk(condition) + ) and branch_value not in possible_boolean_results(condition, facts, policy, True): + guards.add(policy) + guards.update(continuation_guards(current, child, facts)) + if isinstance(current, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda): + break + child = current + current = parents.get(current) + if guards == {"model", "tool"}: + return "model+tool-guard" + if "model" in guards: + return "model-guard" + if "tool" in guards: + return "tool-guard" + return "none" + + +def call_site_context(node: ast.AST, parents: Mapping[ast.AST, ast.AST], source: str) -> str: + parts: list[str] = [] + child = node + current = parents.get(node) + while current is not None: + if isinstance(current, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): + parts.append(f"{type(current).__name__}:{current.name}") + elif isinstance(current, ast.If): + branch = ( + "body" if child in current.body else "else" if child in current.orelse else "test" + ) + parts.append(f"if:{normalize_node(current.test, source)}:{branch}") + elif isinstance(current, ast.IfExp): + branch = ( + "body" if child is current.body else "else" if child is current.orelse else "test" + ) + parts.append(f"ifexp:{normalize_node(current.test, source)}:{branch}") + elif isinstance(current, ast.Try): + if child in current.body: + branch = "try" + elif child in current.handlers: + handler_index = next( + index for index, handler in enumerate(current.handlers) if handler is child + ) + branch = f"handler:{handler_index}" + elif child in current.orelse: + branch = "else" + elif child in current.finalbody: + branch = "finally" + else: + branch = "nested" + parts.append(f"try:{branch}") + elif isinstance(current, ast.ExceptHandler): + parts.append( + f"except:{normalize_node(current.type, source) if current.type else 'bare'}" + ) + elif isinstance(current, ast.match_case): + pattern = normalize_node(current.pattern, source) + guard = normalize_node(current.guard, source) if current.guard else "" + parts.append(f"case:{pattern}:{guard}") + elif isinstance(current, ast.Match): + parts.append(f"match:{normalize_node(current.subject, source)}") + elif isinstance(current, ast.For | ast.AsyncFor | ast.While): + branch = ( + "body" if child in current.body else "else" if child in current.orelse else "header" + ) + parts.append(f"{type(current).__name__}:{branch}") + elif isinstance(current, ast.Call): + if child in current.args: + argument_index = next( + index for index, argument in enumerate(current.args) if argument is child + ) + parts.append(f"callback:{normalize_node(current.func, source)}:{argument_index}") + child = current + current = parents.get(current) + return ">".join(reversed(parts)) or "" + + +def enclosing_catch_names(node: ast.AST, parents: Mapping[ast.AST, ast.AST]) -> list[str]: + names: list[str] = [] + current = parents.get(node) + while current is not None: + if isinstance(current, ast.ExceptHandler) and current.name: + names.append(current.name) + current = parents.get(current) + return names + + +def call_shape(call: ast.Call, method: str) -> str: + if any(keyword.arg is None for keyword in call.keywords): + return "payload" + keywords = {keyword.arg: keyword.value for keyword in call.keywords if keyword.arg is not None} + exc_info = keywords.get("exc_info") + has_exc_info = exc_info is not None and not ( + isinstance(exc_info, ast.Constant) and exc_info.value in (None, False) + ) + if method in {"exception", "traceback.print_exc"} or has_exc_info: + return "exception-payload" + if len(call.args) > 1 or "extra" in keywords: + return "payload" + if method in SENSITIVE_HELPER_METHODS and any( + name not in {"logger", "target_logger", "message", "msg"} for name in keywords + ): + return "payload" + message = call.args[0] if call.args else keywords.get("msg", keywords.get("message")) + if message is None: + if call.keywords: + return "dynamic-message" + return "static-message" + if isinstance(message, ast.Constant) and isinstance(message.value, str): + return "static-message" + return "dynamic-message" + + +def call_shape_with_partial(call: ast.Call, method: str, callee_facts: set[str]) -> str: + shape = call_shape(call, method) + bound_shapes = { + fact.split(":", 1)[1] for fact in callee_facts if fact.startswith("partial-shape:") + } + if not bound_shapes or bound_shapes == {"empty"}: + return shape + if "exception-payload" in bound_shapes or shape == "exception-payload": + return "exception-payload" + if "payload" in bound_shapes or shape == "payload": + return "payload" + + has_call_arguments = bool(call.args or call.keywords) + if "dynamic-message" in bound_shapes: + return "payload" if has_call_arguments else "dynamic-message" + if "static-message" in bound_shapes: + return "payload" if has_call_arguments else "static-message" + return shape + + +def signals_for(text: str) -> list[str]: + normalized = text.lower() + groups = [ + ("model", r"\b(model|response|request|completion|llm|realtime event)\b"), + ( + "tool", + r"\b(tool|function call|arguments|computer action|shell action|apply_patch|mcp)\b", + ), + ("error", r"\b(error|err|exception|failure|failed|reason|traceback)\b"), + ("payload", r"\b(input|output|item|event|payload|data|trace|span|extra)\b"), + ] + return [name for name, pattern in groups if re.search(pattern, normalized)] + + +def inventory_source(source: str, file_path: str = "fixture.py") -> list[Finding]: + normalized_path = normalize_path(file_path) + tree = ast.parse(source, filename=normalized_path) + parents = make_parent_map(tree) + facts = Facts(tree, normalized_path) + findings: list[Finding] = [] + recorded_nodes: set[tuple[int, str, str]] = set() + + def record( + node: ast.AST, + call_text: str, + kind: str, + confidence: str, + method: str, + shape: str, + policy: str, + catch_value: str | None, + ) -> None: + key = (id(node), kind, method) + if key in recorded_nodes: + return + recorded_nodes.add(key) + context = call_site_context(node, parents, source) + group = hashlib.sha256(f"{normalized_path}\0{context}\0{call_text}".encode()).hexdigest()[ + :12 + ] + findings.append( + Finding( + group_fingerprint=group, + fingerprint=group, + file=normalized_path, + line=getattr(node, "lineno", 1), + column=getattr(node, "col_offset", 0) + 1, + kind=kind, + confidence=confidence, + method=method, + shape=shape, + policy=policy, + catch_value=catch_value, + context=context, + signals=signals_for(call_text), + call=call_text, + ) + ) + + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + call_text = normalize_node(node, source) + callee_facts = facts.infer(node.func) + sink_facts = sorted( + fact for fact in callee_facts if fact.startswith(("method:", "helper:")) + ) + catch_names = enclosing_catch_names(node, parents) + referenced = [ + name + for name in catch_names + if any( + node_references_name(value, name) + for value in [*node.args, *(kw.value for kw in node.keywords)] + ) + ] + catch_value = ", ".join(referenced) or None + policy = guarded_policy(node, parents, facts) + + if sink_facts: + for fact in sink_facts: + category, sink_kind, sink_method = _split_sink_fact(fact) + if category == "helper": + helper_policy = f"{sink_kind}-helper" + helper_method = ( + node.func.id if isinstance(node.func, ast.Name) else node.func.attr + ) + record( + node, + call_text, + "sensitive-helper", + "confirmed", + helper_method, + call_shape(node, helper_method), + helper_policy, + catch_value, + ) + else: + if sink_kind == "logger": + shape = call_shape_with_partial(node, sink_method, callee_facts) + if shape == "exception-payload" and catch_value is None and catch_names: + catch_value = "active exception" + record( + node, + call_text, + "logger", + "confirmed", + sink_method, + shape, + policy, + catch_value, + ) + else: + record( + node, + call_text, + "raw-output", + "confirmed", + sink_method, + call_shape_with_partial(node, sink_method, callee_facts), + policy, + catch_value, + ) + elif isinstance(node.func, ast.Attribute) and node.func.attr in LOG_METHODS: + shape = call_shape(node, node.func.attr) + if shape == "exception-payload" and catch_value is None and catch_names: + catch_value = "active exception" + record( + node, + call_text, + "logging-candidate", + "unknown", + node.func.attr, + shape, + policy, + catch_value, + ) + elif isinstance(node.func, ast.Name) and node.func.id in { + "log_model_action_error", + "log_tool_action_error", + }: + record( + node, + call_text, + "helper-candidate", + "unknown", + node.func.id, + call_shape(node, node.func.id), + "none", + catch_value, + ) + + if not _is_partial_call(callee_facts): + callback_arguments = [ + *node.args, + *(keyword.value for keyword in node.keywords if keyword.arg is not None), + ] + for argument in callback_arguments: + for fact in facts.infer(argument): + if not fact.startswith("method:"): + continue + _, sink_kind, sink_method = fact.split(":", 2) + record( + argument, + normalize_node(argument, source), + "logger-callback" if sink_kind == "logger" else "raw-output-callback", + "confirmed", + sink_method, + "dynamic-message", + guarded_policy(argument, parents, facts) + if sink_kind == "logger" + else "none", + "callback value", + ) + + findings.sort(key=lambda finding: (finding.file, finding.line, finding.column, finding.kind)) + counts = Counter(finding.group_fingerprint for finding in findings) + indexes: dict[str, int] = defaultdict(int) + for finding in findings: + index = indexes[finding.group_fingerprint] + indexes[finding.group_fingerprint] += 1 + count = counts[finding.group_fingerprint] + finding.site_index = index + finding.group_count = count + finding.identity_quality = "duplicate" if count > 1 else "unique" + finding.fingerprint = ( + finding.group_fingerprint if count == 1 else f"{finding.group_fingerprint}:{index}" + ) + return findings + + +def _split_sink_fact(fact: str) -> tuple[str, str, str]: + if fact.startswith("helper:"): + policy = fact.split(":", 1)[1] + method = ( + "log_model_and_tool_action_error" + if policy == "model+tool" + else f"log_{policy}_action_error" + ) + return "helper", policy, method + _, sink_kind, sink_method = fact.split(":", 2) + return "method", sink_kind, sink_method + + +def _is_partial_call(callee_facts: set[str]) -> bool: + return "factory:partial" in callee_facts + + +def summarize(findings: Sequence[Finding]) -> dict[str, int]: + dynamic = [finding for finding in findings if finding.shape != "static-message"] + duplicate_groups = { + finding.group_fingerprint for finding in findings if finding.identity_quality == "duplicate" + } + return { + "total": len(findings), + "dynamic": len(dynamic), + "unclassifiedDynamic": sum(finding.policy == "none" for finding in dynamic), + "catchValueLogs": sum(finding.catch_value is not None for finding in findings), + "unclassifiedCatchValueLogs": sum( + finding.catch_value is not None and finding.policy == "none" for finding in findings + ), + "rawOutputCalls": sum(finding.kind.startswith("raw-output") for finding in findings), + "unknownReceiverCalls": sum(finding.confidence == "unknown" for finding in findings), + "duplicateGroups": len(duplicate_groups), + } + + +def compare_findings( + baseline: Sequence[Mapping[str, Any]], findings: Sequence[Finding] +) -> dict[str, Any]: + before = Counter(str(item["group_fingerprint"]) for item in baseline) + after = Counter(finding.group_fingerprint for finding in findings) + before_classifications = _comparison_classifications(baseline) + after_classifications = _comparison_classifications(findings) + return { + "new": sorted(after.keys() - before.keys()), + "removed": sorted(before.keys() - after.keys()), + "countChanged": [ + {"group_fingerprint": key, "before": before[key], "after": after[key]} + for key in sorted(before.keys() & after.keys()) + if before[key] != after[key] + ], + "classificationChanged": [ + { + "group_fingerprint": key, + "before": _render_classification_counts(before_classifications[key]), + "after": _render_classification_counts(after_classifications[key]), + } + for key in sorted(before.keys() & after.keys()) + if before_classifications[key] != after_classifications[key] + ], + } + + +def _comparison_classifications( + findings: Sequence[Mapping[str, Any] | Finding], +) -> dict[str, Counter[tuple[Any, ...]]]: + result: dict[str, Counter[tuple[Any, ...]]] = defaultdict(Counter) + for finding in findings: + if isinstance(finding, Finding): + group = finding.group_fingerprint + classification = tuple( + getattr(finding, field) for field in COMPARISON_CLASSIFICATION_FIELDS + ) + else: + group = str(finding["group_fingerprint"]) + classification = tuple(finding.get(field) for field in COMPARISON_CLASSIFICATION_FIELDS) + result[group][classification] += 1 + return result + + +def _render_classification_counts( + counts: Counter[tuple[Any, ...]], +) -> list[dict[str, Any]]: + return [ + { + **dict(zip(COMPARISON_CLASSIFICATION_FIELDS, classification, strict=True)), + "count": count, + } + for classification, count in sorted(counts.items(), key=lambda item: repr(item[0])) + ] + + +def validate_review_ledger( + ledger: Mapping[str, Any], findings: Sequence[Finding] +) -> dict[str, Any]: + reviews = ledger.get("reviews") + if not isinstance(reviews, list): + return {"valid": False, "errors": ["Review ledger must contain a reviews list."]} + dynamic_counts = Counter( + finding.group_fingerprint for finding in findings if finding.shape != "static-message" + ) + review_groups = [ + str(review.get("group_fingerprint")) + for review in reviews + if isinstance(review, dict) and review.get("group_fingerprint") + ] + duplicate_review_groups = sorted( + group for group, count in Counter(review_groups).items() if count > 1 + ) + errors = [f"Duplicate review entries for {group}" for group in duplicate_review_groups] + review_by_group: dict[str, Mapping[str, Any]] = {} + for review in reviews: + if not isinstance(review, dict) or not review.get("group_fingerprint"): + continue + review_by_group.setdefault(str(review["group_fingerprint"]), review) + for group, count in sorted(dynamic_counts.items()): + review = review_by_group.get(group) + if review is None: + errors.append(f"Missing review for dynamic group {group}.") + continue + if review.get("disposition") not in DISPOSITIONS: + errors.append(f"Invalid disposition for {group}.") + if not str(review.get("evidence", "")).strip(): + errors.append(f"Missing evidence for {group}.") + if not str(review.get("action", "")).strip(): + errors.append(f"Missing action for {group}.") + if review.get("group_count") != count: + actual = review.get("group_count") + errors.append(f"Group count mismatch for {group}: expected {count}, got {actual}.") + extras = sorted(review_by_group.keys() - dynamic_counts.keys()) + if extras: + errors.append(f"Review ledger contains stale groups: {', '.join(extras)}.") + return {"valid": not errors, "errors": errors} diff --git a/.agents/skills/sensitive-logging-audit/scripts/inventory_logging.py b/.agents/skills/sensitive-logging-audit/scripts/inventory_logging.py new file mode 100644 index 0000000000..4b3bfc916e --- /dev/null +++ b/.agents/skills/sensitive-logging-audit/scripts/inventory_logging.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +from _inventory import ( + Finding, + collect_source_files, + compare_findings, + inventory_source, + summarize, + validate_review_ledger, +) + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Inventory Python runtime logging and raw-output sinks for sensitive-data review." + ) + ) + parser.add_argument("roots", nargs="*", default=["src/agents"]) + parser.add_argument("--format", choices=("json", "markdown"), default="markdown") + parser.add_argument("--summary-only", action="store_true") + parser.add_argument("--output", type=Path) + parser.add_argument("--compare", type=Path, metavar="BASELINE_JSON") + parser.add_argument("--validate-review", type=Path, metavar="REVIEW_JSON") + return parser.parse_args(argv) + + +def build_report(args: argparse.Namespace) -> tuple[dict[str, Any], bool]: + cwd = Path.cwd().resolve() + paths = collect_source_files(args.roots) + findings: list[Finding] = [] + for path in paths: + try: + display_path = path.relative_to(cwd) + except ValueError: + display_path = path + source = path.read_text(encoding="utf-8") + try: + findings.extend(inventory_source(source, str(display_path))) + except SyntaxError as error: + raise SyntaxError( + f"Failed to parse {display_path}:{error.lineno}: {error.msg}" + ) from error + + report: dict[str, Any] = { + "summary": summarize(findings), + } + if not args.summary_only: + report["findings"] = [finding.to_dict() for finding in findings] + + valid = True + if args.compare: + baseline = json.loads(args.compare.read_text(encoding="utf-8")) + baseline_findings = baseline.get("findings") + if not isinstance(baseline_findings, list): + raise ValueError("Baseline JSON must contain a findings list.") + report["comparison"] = compare_findings(baseline_findings, findings) + + if args.validate_review: + ledger = json.loads(args.validate_review.read_text(encoding="utf-8")) + if not isinstance(ledger, dict): + raise ValueError("Review ledger must be a JSON object.") + validation = validate_review_ledger(ledger, findings) + report["reviewValidation"] = validation + valid = bool(validation["valid"]) + + return report, valid + + +def render_markdown(report: dict[str, Any], summary_only: bool) -> str: + summary = report["summary"] + lines = [ + "# Sensitive logging inventory", + "", + f"- Total output calls: {summary['total']}", + f"- Dynamic calls: {summary['dynamic']}", + f"- Dynamic calls without an explicit model/tool policy: {summary['unclassifiedDynamic']}", + f"- Calls that log a caught or active exception: {summary['catchValueLogs']}", + f"- Unclassified caught-value calls: {summary['unclassifiedCatchValueLogs']}", + f"- Raw output calls: {summary['rawOutputCalls']}", + f"- Unknown receiver calls: {summary['unknownReceiverCalls']}", + f"- Duplicate fingerprint groups: {summary['duplicateGroups']}", + ] + if not summary_only: + lines.extend( + [ + "", + "| Location | Kind | Shape | Policy | Catch value | Fingerprint |", + "| --- | --- | --- | --- | --- | --- |", + ] + ) + for finding in report.get("findings", []): + location = f"{finding['file']}:{finding['line']}" + sink = f"{finding['kind']}.{finding['method']}" + lines.append( + f"| {location} | {sink} | {finding['shape']} | {finding['policy']} | " + f"{finding['catch_value'] or ''} | {finding['fingerprint']} |" + ) + + comparison = report.get("comparison") + if comparison is not None: + lines.extend( + [ + "", + "## Baseline comparison", + "", + f"- New groups: {len(comparison['new'])}", + f"- Removed groups: {len(comparison['removed'])}", + f"- Count-changed groups: {len(comparison['countChanged'])}", + f"- Classification-changed groups: {len(comparison['classificationChanged'])}", + ] + ) + + validation = report.get("reviewValidation") + if validation is not None: + lines.extend( + [ + "", + "## Review ledger validation", + "", + f"- Valid: {'yes' if validation['valid'] else 'no'}", + ] + ) + lines.extend(f"- {error}" for error in validation["errors"]) + return "\n".join(lines) + "\n" + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + try: + report, valid = build_report(args) + output = ( + json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.format == "json" + else render_markdown(report, args.summary_only) + ) + if args.output: + args.output.write_text(output, encoding="utf-8") + else: + sys.stdout.write(output) + return 0 if valid else 1 + except (OSError, SyntaxError, ValueError, json.JSONDecodeError) as error: + print(f"Sensitive logging inventory failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py new file mode 100644 index 0000000000..d1ab14efc2 --- /dev/null +++ b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py @@ -0,0 +1,810 @@ +from __future__ import annotations + +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +from _inventory import ( + collect_source_files, + compare_findings, + inventory_source, + summarize, + validate_review_ledger, +) + + +class InventoryTests(unittest.TestCase): + def test_inventories_static_dynamic_and_exception_logger_calls(self) -> None: + findings = inventory_source( + """ +from ..logger import logger + +logger.debug("ready") +logger.warning(f"Request {request_id} failed") +logger.error("Request failed: %s", error) +logger.exception("Unhandled failure") +""", + "src/agents/run_internal/fixture.py", + ) + + self.assertEqual( + [(item.method, item.shape, item.confidence) for item in findings], + [ + ("debug", "static-message", "confirmed"), + ("warning", "dynamic-message", "confirmed"), + ("error", "payload", "confirmed"), + ("exception", "exception-payload", "confirmed"), + ], + ) + + def test_inventories_keyword_messages_and_helper_payloads(self) -> None: + findings = inventory_source( + """ +from agents.logger import logger, log_tool_action_error + +logger.error(msg=secret) +logger.info(msg="ready") +log_tool_action_error(target_logger=logger, message=secret, exc=error) +""" + ) + + self.assertEqual( + [(item.kind, item.shape) for item in findings], + [ + ("logger", "dynamic-message"), + ("logger", "static-message"), + ("sensitive-helper", "payload"), + ], + ) + self.assertEqual(summarize(findings)["dynamic"], 2) + + def test_treats_expanded_logging_keywords_as_payloads(self) -> None: + findings = inventory_source( + """ +from agents.logger import logger + +logger.error("failed", **{"exc_info": error}) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [("logger", "error", "payload")], + ) + + def test_classifies_no_argument_print_exc_as_exception_payload(self) -> None: + findings = inventory_source( + """ +import traceback + +try: + run() +except Exception: + traceback.print_exc() +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [("raw-output", "traceback.print_exc", "exception-payload")], + ) + + def test_recognizes_logger_factories_aliases_methods_and_partial(self) -> None: + findings = inventory_source( + """ +import functools +import logging + +base = logging.getLogger(__name__) +alias = base +emit = alias.error +emit("failed", secret) +handler = functools.partial(alias.warning, "failed: %s") +handler(secret) +schedule(alias.info) +""" + ) + + self.assertEqual( + [(item.kind, item.method) for item in findings], + [ + ("logger", "error"), + ("logger", "warning"), + ("logger-callback", "info"), + ], + ) + + def test_imported_partial_is_resolved(self) -> None: + findings = inventory_source( + """ +from functools import partial +from agents.logger import logger + +report = partial(logger.error, "failed: %s") +report(secret) +""" + ) + + self.assertEqual([(item.kind, item.method) for item in findings], [("logger", "error")]) + + def test_recognizes_directly_constructed_logger_aliases(self) -> None: + findings = inventory_source( + """ +import logging +from logging import Logger + +direct = Logger("direct") +direct_emit = direct.error +direct_emit(secret) +qualified = logging.Logger("qualified") +qualified_emit = qualified.warning +qualified_emit(secret) +""" + ) + + self.assertEqual( + [(item.kind, item.method) for item in findings], + [("logger", "error"), ("logger", "warning")], + ) + + def test_resolves_helpers_through_qualified_sdk_logger_imports(self) -> None: + findings = inventory_source( + """ +import agents.logger +import agents.logger as audit_logger +import agents.tracing.logger +from agents import logger as logger_module +from agents.logger import logger + +agents.logger.log_model_action_error(logger, secret, error) +audit_logger.log_tool_action_error(logger, secret, error) +logger_module.log_model_and_tool_action_warning(logger, secret, error) +agents.tracing.logger.logger.error(secret) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.policy) for item in findings], + [ + ("sensitive-helper", "log_model_action_error", "model-helper"), + ("sensitive-helper", "log_tool_action_error", "tool-helper"), + ( + "sensitive-helper", + "log_model_and_tool_action_warning", + "model+tool-helper", + ), + ("logger", "error", "none"), + ], + ) + + def test_partial_preserves_bound_payload_shape(self) -> None: + findings = inventory_source( + """ +from functools import partial +from agents.logger import logger + +emit = partial(logger.error, secret) +emit() +formatted = partial(logger.warning, "failed: %s") +formatted(secret) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [ + ("logger", "error", "dynamic-message"), + ("logger", "warning", "payload"), + ], + ) + + def test_inventories_raw_output_and_unknown_receivers(self) -> None: + findings = inventory_source( + """ +import sys +import traceback +import warnings + +print(secret) +warnings.warn(secret) +sys.stderr.write(secret) +traceback.print_exception(error) +task.exception() +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.confidence) for item in findings], + [ + ("raw-output", "print", "confirmed"), + ("raw-output", "warnings.warn", "confirmed"), + ("raw-output", "stderr.write", "confirmed"), + ("raw-output", "traceback.print_exception", "confirmed"), + ("logging-candidate", "exception", "unknown"), + ], + ) + + def test_inventories_directly_imported_output_streams(self) -> None: + findings = inventory_source( + """ +from sys import stderr, stdout as out + +stderr.write(secret) +out.write(secret) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [ + ("raw-output", "stderr.write", "dynamic-message"), + ("raw-output", "stdout.write", "dynamic-message"), + ], + ) + + def test_inventories_buffered_output_streams(self) -> None: + findings = inventory_source( + """ +import sys +from sys import stdout as out + +sys.stderr.buffer.write(secret_bytes) +writer = out.buffer +writer.write(secret_bytes) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [ + ("raw-output", "stderr.write", "dynamic-message"), + ("raw-output", "stdout.write", "dynamic-message"), + ], + ) + + def test_inventories_stream_writelines(self) -> None: + findings = inventory_source( + """ +import sys +from sys import stdout as out + +sys.stderr.writelines([secret]) +out.writelines([secret]) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [ + ("raw-output", "stderr.writelines", "dynamic-message"), + ("raw-output", "stdout.writelines", "dynamic-message"), + ], + ) + + def test_inventories_warn_explicit_module_and_direct_imports(self) -> None: + findings = inventory_source( + """ +import warnings +from warnings import warn_explicit as emit_warning + +warnings.warn_explicit(secret, UserWarning, "x.py", 1) +emit_warning(secret, UserWarning, "x.py", 1) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [ + ("raw-output", "warnings.warn_explicit", "payload"), + ("raw-output", "warnings.warn_explicit", "payload"), + ], + ) + + def test_inventories_fatal_logger_aliases(self) -> None: + findings = inventory_source( + """ +import logging +from agents.logger import logger +from logging import fatal as die + +logger.fatal(secret) +logging.fatal(secret) +die(secret) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [ + ("logger", "fatal", "dynamic-message"), + ("logger", "fatal", "dynamic-message"), + ("logger", "fatal", "dynamic-message"), + ], + ) + + def test_requires_exact_policy_provenance_and_correct_polarity(self) -> None: + findings = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +if not _debug.DONT_LOG_TOOL_DATA: + logger.error("diagnostic: %s", secret) +if _debug.DONT_LOG_TOOL_DATA: + logger.error("leak: %s", secret) +if not request.DONT_LOG_TOOL_DATA: + logger.error("unrelated: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["tool-guard", "none", "none"]) + + def test_scopes_policy_facts_to_lexical_bindings(self) -> None: + findings = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +def configure(): + flag = _debug.DONT_LOG_TOOL_DATA + return flag + +def report(flag, secret): + if not flag: + logger.error("unguarded: %s", secret) +""" + ) + + self.assertEqual(findings[0].policy, "none") + + def test_preserves_direct_policy_aliases_but_not_composite_boolean_aliases(self) -> None: + findings = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +def direct(secret): + flag = _debug.DONT_LOG_TOOL_DATA + if not flag: + logger.error("guarded: %s", secret) + +def composite(enabled, secret): + guard = _debug.DONT_LOG_TOOL_DATA and enabled + if not guard: + logger.error("unguarded: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["tool-guard", "none"]) + + def test_policy_assignments_do_not_flow_backward_or_through_conditional_rebinding( + self, + ) -> None: + findings = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +def assigned_later(flag, secret): + if not flag: + logger.error("unguarded before assignment: %s", secret) + flag = _debug.DONT_LOG_TOOL_DATA + +def conditionally_reassigned(flag, condition, secret): + if condition: + flag = _debug.DONT_LOG_TOOL_DATA + if not flag: + logger.error("unguarded after conditional assignment: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["none", "none"]) + + def test_destructuring_matches_policy_facts_to_individual_values(self) -> None: + findings = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +def report(user_value, secret): + redact, enabled = _debug.DONT_LOG_TOOL_DATA, user_value + if not enabled: + logger.error("unguarded: %s", secret) + if not redact: + logger.error("guarded: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["none", "tool-guard"]) + + def test_augmented_assignment_kills_policy_alias_provenance(self) -> None: + findings = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +def report(enabled, secret): + flag = _debug.DONT_LOG_TOOL_DATA + if not flag: + logger.error("guarded: %s", secret) + flag &= enabled + if not flag: + logger.error("unguarded: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["tool-guard", "none"]) + + def test_runtime_binders_kill_imported_policy_alias_provenance(self) -> None: + findings = inventory_source( + """ +from agents.logger import logger + +def report(values, manager, secret): + from agents._debug import DONT_LOG_TOOL_DATA as flag + for flag in values: + if not flag: + logger.error("for binder: %s", secret) + with manager() as flag: + if not flag: + logger.error("with binder: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["none", "none"]) + + def test_combines_exact_model_and_tool_guards(self) -> None: + findings = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +if not _debug.DONT_LOG_MODEL_DATA and not _debug.DONT_LOG_TOOL_DATA: + logger.error("diagnostic: %s", secret) +""" + ) + + self.assertEqual(findings[0].policy, "model+tool-guard") + + def test_recognizes_early_return_policy_guards(self) -> None: + findings = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +def report(secret): + if _debug.DONT_LOG_TOOL_DATA: + logger.debug("Tool failed") + return + logger.error("Tool failed: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["none", "tool-guard"]) + + def test_recognizes_early_continue_policy_guards(self) -> None: + findings = inventory_source( + """ +from agents import _debug + +for item in items: + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + print("Data is redacted") + continue + print(item) +""", + "src/agents/fixture.py", + ) + + self.assertEqual( + [(item.shape, item.policy) for item in findings], + [("static-message", "none"), ("dynamic-message", "model+tool-guard")], + ) + + def test_trusts_only_the_exact_known_helper_import(self) -> None: + trusted = inventory_source( + """ +from agents.run_internal.tool_execution import log_tool_action_error as report_failure + +report_failure("Tool failed", error) +""" + ) + untrusted = inventory_source( + """ +from analytics.logger import log_tool_action_error + +log_tool_action_error("Tool failed", error) +""" + ) + + self.assertEqual( + (trusted[0].kind, trusted[0].policy, trusted[0].confidence), + ("sensitive-helper", "tool-helper", "confirmed"), + ) + self.assertEqual( + (untrusted[0].kind, untrusted[0].policy, untrusted[0].confidence), + ("helper-candidate", "none", "unknown"), + ) + + def test_recognizes_the_known_helper_in_its_defining_module(self) -> None: + findings = inventory_source( + """ +def log_tool_action_error(message, error): + pass + +log_tool_action_error("Tool failed", error) +""", + "src/agents/run_internal/tool_execution.py", + ) + + self.assertEqual( + (findings[0].kind, findings[0].policy, findings[0].confidence), + ("sensitive-helper", "tool-helper", "confirmed"), + ) + + def test_recognizes_shared_model_and_mixed_helpers(self) -> None: + findings = inventory_source( + """ +from agents.logger import ( + log_model_action_debug, + log_model_action_error, + log_model_and_tool_action_error, + log_model_and_tool_action_warning, +) + +log_model_action_debug(logger, "Model cleanup failed", error) +log_model_action_error(logger, "Model failed", error) +log_model_and_tool_action_error(logger, "Trace failed", error) +log_model_and_tool_action_warning(logger, "Session failed", error) +""" + ) + + self.assertEqual( + [(item.method, item.policy) for item in findings], + [ + ("log_model_action_debug", "model-helper"), + ("log_model_action_error", "model-helper"), + ("log_model_and_tool_action_error", "model+tool-helper"), + ("log_model_and_tool_action_warning", "model+tool-helper"), + ], + ) + + def test_recognizes_shared_tool_helpers_at_multiple_levels(self) -> None: + findings = inventory_source( + """ +from agents.logger import log_tool_action_debug, log_tool_action_warning + +log_tool_action_debug(logger, "Cleanup cancelled", error) +log_tool_action_warning(logger, "Cleanup failed", error) +""" + ) + + self.assertEqual( + [(item.method, item.policy) for item in findings], + [ + ("log_tool_action_debug", "tool-helper"), + ("log_tool_action_warning", "tool-helper"), + ], + ) + + def test_flags_caught_values_and_implicit_exception_payloads(self) -> None: + findings = inventory_source( + """ +from agents.logger import logger + +try: + run() +except Exception as exc: + logger.error("Run failed: %s", exc) + logger.error("Run failed", exc_info=True) + logger.exception("Run failed") +""" + ) + + self.assertEqual( + [(item.shape, item.catch_value) for item in findings], + [ + ("payload", "exc"), + ("exception-payload", "active exception"), + ("exception-payload", "active exception"), + ], + ) + + def test_fingerprints_survive_line_shifts_and_distinguish_branches(self) -> None: + before = inventory_source( + """ +from agents.logger import logger +def report(secret): + if enabled: + logger.error("failed: %s", secret) + else: + logger.error("failed: %s", secret) +""" + ) + after = inventory_source( + """ +from agents.logger import logger + + +def report(secret): + unused = None + if enabled: + logger.error("failed: %s", secret) + else: + logger.error("failed: %s", secret) +""" + ) + + self.assertEqual( + [item.group_fingerprint for item in before], + [item.group_fingerprint for item in after], + ) + self.assertNotEqual(before[0].group_fingerprint, before[1].group_fingerprint) + + def test_identical_sites_form_a_duplicate_group(self) -> None: + findings = inventory_source( + """ +from agents.logger import logger +def report(secret): + logger.error("failed: %s", secret) + logger.error("failed: %s", secret) +""" + ) + + self.assertEqual(findings[0].group_fingerprint, findings[1].group_fingerprint) + self.assertEqual([item.group_count for item in findings], [2, 2]) + self.assertEqual([item.identity_quality for item in findings], ["duplicate", "duplicate"]) + self.assertEqual([item.fingerprint.rsplit(":", 1)[-1] for item in findings], ["0", "1"]) + + def test_discovers_keyword_callback_sinks(self) -> None: + findings = inventory_source( + """ +import sys +from agents.logger import logger + +register(on_error=logger.error, writer=sys.stderr.write) +""" + ) + + self.assertEqual( + [(item.kind, item.method) for item in findings], + [("logger-callback", "error"), ("raw-output-callback", "stderr.write")], + ) + + def test_source_collection_ignores_hidden_paths_only_below_the_scan_root(self) -> None: + with TemporaryDirectory() as temp_dir: + root = Path(temp_dir) / ".cache" / "worktree" + visible = root / "src" / "visible.py" + hidden = root / ".generated" / "hidden.py" + visible.parent.mkdir(parents=True) + hidden.parent.mkdir() + visible.touch() + hidden.touch() + + self.assertEqual(collect_source_files([root]), [visible.resolve()]) + + def test_normalizes_path_separators_before_hashing(self) -> None: + source = "from agents.logger import logger\nlogger.error('failed', secret)\n" + posix = inventory_source(source, "src/agents/fixture.py") + windows = inventory_source(source, "src\\agents\\fixture.py") + self.assertEqual(posix[0].group_fingerprint, windows[0].group_fingerprint) + + def test_compare_reports_new_removed_and_duplicate_count_changes(self) -> None: + baseline = inventory_source("from agents.logger import logger\nlogger.error('x', secret)\n") + current = inventory_source( + """ +from agents.logger import logger +logger.error('x', secret) +logger.error('x', secret) +logger.warning('y', secret) +""" + ) + comparison = compare_findings([item.to_dict() for item in baseline], current) + + self.assertEqual(len(comparison["new"]), 1) + self.assertEqual(comparison["removed"], []) + self.assertEqual(len(comparison["countChanged"]), 1) + self.assertEqual(comparison["countChanged"][0]["before"], 1) + self.assertEqual(comparison["countChanged"][0]["after"], 2) + + def test_compare_reports_security_classification_changes(self) -> None: + baseline = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +def report(secret): + if _debug.DONT_LOG_TOOL_DATA: + return + logger.error("failed: %s", secret) +""" + ) + current = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +def report(secret): + if False: + return + logger.error("failed: %s", secret) +""" + ) + + comparison = compare_findings([item.to_dict() for item in baseline], current) + + self.assertEqual(comparison["new"], []) + self.assertEqual(comparison["removed"], []) + self.assertEqual(comparison["countChanged"], []) + self.assertEqual(len(comparison["classificationChanged"]), 1) + change = comparison["classificationChanged"][0] + self.assertEqual(change["group_fingerprint"], baseline[0].group_fingerprint) + self.assertEqual(change["before"][0]["policy"], "tool-guard") + self.assertEqual(change["after"][0]["policy"], "none") + + def test_review_ledger_requires_every_dynamic_group_and_exact_count(self) -> None: + findings = inventory_source( + """ +from agents.logger import logger +logger.error('x', secret) +logger.error('x', secret) +logger.info('ready') +""" + ) + dynamic_group = findings[0].group_fingerprint + valid = validate_review_ledger( + { + "reviews": [ + { + "group_fingerprint": dynamic_group, + "group_count": 2, + "disposition": "tool", + "evidence": "The argument is the tool input.", + "action": "Guard with DONT_LOG_TOOL_DATA.", + } + ] + }, + findings, + ) + invalid = validate_review_ledger({"reviews": []}, findings) + + self.assertTrue(valid["valid"]) + self.assertFalse(invalid["valid"]) + self.assertIn("Missing review", invalid["errors"][0]) + + def test_review_ledger_rejects_duplicate_group_entries(self) -> None: + findings = inventory_source("from agents.logger import logger\nlogger.error('x', secret)\n") + group = findings[0].group_fingerprint + review = { + "group_fingerprint": group, + "group_count": 1, + "disposition": "tool", + "evidence": "The argument is the tool input.", + "action": "Guard with DONT_LOG_TOOL_DATA.", + } + + validation = validate_review_ledger({"reviews": [review, dict(review)]}, findings) + + self.assertFalse(validation["valid"]) + self.assertIn(f"Duplicate review entries for {group}", validation["errors"]) + + def test_summary_separates_unknown_and_raw_candidates(self) -> None: + findings = inventory_source( + """ +from agents.logger import logger +logger.info('ready') +logger.error('failed', secret) +print(secret) +task.exception() +""" + ) + summary = summarize(findings) + + self.assertEqual(summary["total"], 4) + self.assertEqual(summary["dynamic"], 3) + self.assertEqual(summary["rawOutputCalls"], 1) + self.assertEqual(summary["unknownReceiverCalls"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/agents/agent.py b/src/agents/agent.py index 4f3c54a074..ecc1291d1d 100644 --- a/src/agents/agent.py +++ b/src/agents/agent.py @@ -29,7 +29,7 @@ from .exceptions import ModelBehaviorError, UserError from .guardrail import InputGuardrail, OutputGuardrail from .handoffs import Handoff -from .logger import logger +from .logger import log_model_and_tool_action_error, logger from .mcp import MCPUtil from .model_settings import ModelSettings, _coerce_model_settings, _declared_model_settings_type from .models.default_models import ( @@ -854,10 +854,11 @@ async def _run_handler(payload: AgentToolStreamEvent) -> None: maybe_result = stream_handler(payload) if inspect.isawaitable(maybe_result): await maybe_result - except Exception: - logger.exception( - "Error while handling on_stream event for agent tool %s.", - self.name, + except Exception as exc: + log_model_and_tool_action_error( + logger, + "Error while handling an agent tool on_stream event", + exc, ) async def dispatch_stream_events() -> None: diff --git a/src/agents/extensions/experimental/codex/codex_tool.py b/src/agents/extensions/experimental/codex/codex_tool.py index 534245dbd1..2c252e3d00 100644 --- a/src/agents/extensions/experimental/codex/codex_tool.py +++ b/src/agents/extensions/experimental/codex/codex_tool.py @@ -17,7 +17,7 @@ from agents import _debug from agents.exceptions import ModelBehaviorError, UserError -from agents.logger import logger +from agents.logger import log_model_and_tool_action_error, log_tool_action_error, logger from agents.models import _openai_shared from agents.run_context import RunContextWrapper from agents.strict_schema import ensure_strict_json_schema @@ -952,8 +952,12 @@ def _try_store_thread_id_in_run_context_after_error( try: _store_thread_id_in_run_context(ctx, key, thread_id) - except Exception: - logger.exception("Failed to store Codex thread id in run context after error.") + except Exception as exc: + log_tool_action_error( + logger, + "Failed to store Codex thread id in run context after error", + exc, + ) def _set_pydantic_context_value(context: BaseModel, key: str, value: str) -> bool: @@ -1047,8 +1051,12 @@ async def _run_handler(payload: CodexToolStreamEvent) -> None: maybe_result = on_stream(payload) if inspect.isawaitable(maybe_result): await maybe_result - except Exception: - logger.exception("Error while handling Codex on_stream event.") + except Exception as exc: + log_model_and_tool_action_error( + logger, + "Error while handling Codex on_stream event", + exc, + ) async def _dispatch() -> None: assert event_queue is not None diff --git a/src/agents/extensions/memory/advanced_sqlite_session.py b/src/agents/extensions/memory/advanced_sqlite_session.py index 2dd1e947fe..088ac8bcac 100644 --- a/src/agents/extensions/memory/advanced_sqlite_session.py +++ b/src/agents/extensions/memory/advanced_sqlite_session.py @@ -11,8 +11,14 @@ from agents.result import RunResult from agents.usage import Usage +from ... import _debug from ..._tool_identity import is_reserved_synthetic_tool_namespace, tool_qualified_name from ...items import TResponseInputItem +from ...logger import ( + log_model_action_error, + log_model_action_warning, + log_model_and_tool_action_error, +) from ...memory import SQLiteSession from ...memory.session_settings import SessionSettings, resolve_session_limit @@ -172,9 +178,11 @@ def _add_items_sync(): self._insert_items(conn, items) self._insert_structure_metadata(conn, items) conn.commit() - except Exception: + except Exception as exc: conn.rollback() - self._logger.exception("Failed to add items for session %s", self.session_id) + log_model_and_tool_action_error( + self._logger, "Failed to add session items", exc + ) raise await asyncio.to_thread(_add_items_sync) @@ -462,7 +470,7 @@ async def store_run_usage(self, result: RunResult) -> None: turn_anchor=turn_anchor, ) except Exception as e: - self._logger.error("Failed to store usage for session %s: %s", self.session_id, e) + log_model_action_error(self._logger, "Failed to store session usage", e) def _capture_current_turn(self) -> tuple[int, str, int | None]: """Return (current_turn, branch_id, turn_anchor) in one locked read. @@ -581,15 +589,19 @@ def _add_structure_sync(): try: await asyncio.to_thread(_add_structure_sync) - except Exception: - self._logger.exception( - "Failed to add structure metadata for session %s", self.session_id + except Exception as exc: + log_model_and_tool_action_error( + self._logger, + "Failed to add session structure metadata", + exc, ) # Try to clean up any orphaned messages to maintain consistency. try: await self._cleanup_orphaned_messages() - except Exception: - self._logger.exception("Failed to cleanup orphaned messages") + except Exception as cleanup_exc: + log_model_and_tool_action_error( + self._logger, "Failed to cleanup orphaned session messages", cleanup_exc + ) raise def _insert_structure_metadata( @@ -870,13 +882,21 @@ def _validate_turn(): old_branch = self._current_branch_id await asyncio.to_thread(self._commit_branch_pointer, branch_name, generation) - self._logger.debug( - "Created branch '%s' from turn %s ('%s') in '%s'", - branch_name, - turn_number, - turn_content, - old_branch, - ) + if _debug.DONT_LOG_MODEL_DATA: + self._logger.debug( + "Created branch '%s' from turn %s in '%s'", + branch_name, + turn_number, + old_branch, + ) + else: + self._logger.debug( + "Created branch '%s' from turn %s ('%s') in '%s'", + branch_name, + turn_number, + turn_content, + old_branch, + ) return branch_name async def create_branch_from_content( @@ -1580,7 +1600,9 @@ def _update_sync(): try: input_details_json = json.dumps(usage_data.input_tokens_details.__dict__) except (TypeError, ValueError) as e: - self._logger.warning("Failed to serialize input tokens details: %s", e) + log_model_action_warning( + self._logger, "Failed to serialize input token details", e + ) input_details_json = None if ( @@ -1590,7 +1612,9 @@ def _update_sync(): try: output_details_json = json.dumps(usage_data.output_tokens_details.__dict__) except (TypeError, ValueError) as e: - self._logger.warning("Failed to serialize output tokens details: %s", e) + log_model_action_warning( + self._logger, "Failed to serialize output token details", e + ) output_details_json = None with closing(conn.cursor()) as cursor: diff --git a/src/agents/extensions/memory/dapr_session.py b/src/agents/extensions/memory/dapr_session.py index eaed2574f5..e923940f11 100644 --- a/src/agents/extensions/memory/dapr_session.py +++ b/src/agents/extensions/memory/dapr_session.py @@ -43,7 +43,7 @@ ) from ...items import TResponseInputItem -from ...logger import logger +from ...logger import log_model_and_tool_action_error, logger from ...memory.session import SessionABC from ...memory.session_settings import ( SessionSettings, @@ -461,5 +461,5 @@ async def ping(self) -> bool: ) return True except Exception: - logger.error("Dapr connection failed: %s", initial_error) + log_model_and_tool_action_error(logger, "Dapr connection failed", initial_error) return False diff --git a/src/agents/extensions/sandbox/blaxel/mounts.py b/src/agents/extensions/sandbox/blaxel/mounts.py index 1476a1eb5d..d2cefac3d2 100644 --- a/src/agents/extensions/sandbox/blaxel/mounts.py +++ b/src/agents/extensions/sandbox/blaxel/mounts.py @@ -25,6 +25,7 @@ from pathlib import Path from typing import Any, Literal +from ....logger import log_tool_action_warning from ....sandbox.entries import GCSMount, Mount, R2Mount, S3Mount from ....sandbox.entries.mounts.base import MountStrategyBase from ....sandbox.errors import MountConfigError @@ -668,7 +669,7 @@ async def _detach_drive(sandbox: Any, mount_path: str) -> None: try: await drives.unmount(mount_path) except Exception as e: - logger.warning("drive detach failed for %s (non-fatal): %s", mount_path, e) + log_tool_action_warning(logger, "Drive detach failed (non-fatal)", e) __all__ = [ diff --git a/src/agents/extensions/sandbox/blaxel/sandbox.py b/src/agents/extensions/sandbox/blaxel/sandbox.py index 02c8e87b38..02d41a9712 100644 --- a/src/agents/extensions/sandbox/blaxel/sandbox.py +++ b/src/agents/extensions/sandbox/blaxel/sandbox.py @@ -29,6 +29,7 @@ from pydantic import BaseModel, Field +from ....logger import log_tool_action_debug, log_tool_action_warning from ....sandbox.entries import Mount from ....sandbox.errors import ( ExecTimeoutError, @@ -453,7 +454,9 @@ async def start(self) -> None: } ) except Exception as e: - logger.debug("workspace root mkdir failed (will retry during materialization): %s", e) + log_tool_action_debug( + logger, "Workspace root mkdir failed; retrying during materialization", e + ) await super().start() async def stop(self) -> None: @@ -467,7 +470,7 @@ async def shutdown(self) -> None: # When pause_on_exit is True the sandbox is kept alive. Blaxel # automatically resumes it on the next connection. except Exception as e: - logger.warning("sandbox delete failed during shutdown: %s", e) + log_tool_action_warning(logger, "Sandbox delete failed during shutdown", e) async def _validate_path_access(self, path: Path | str, *, for_write: bool = False) -> Path: return await self._validate_remote_path_access(path, for_write=for_write) @@ -626,7 +629,7 @@ async def running(self) -> bool: await asyncio.wait_for(self._sandbox.fs.ls("/"), timeout=10.0) return True except Exception as e: - logger.debug("sandbox health check failed: %s", e) + log_tool_action_debug(logger, "Sandbox health check failed", e) return False # -- workspace persistence ----------------------------------------------- @@ -690,7 +693,7 @@ async def persist_workspace(self) -> io.IOBase: "rm", "-f", "--", tar_path, timeout=self.state.timeouts.cleanup_s ) except Exception as e: - logger.debug("persist cleanup rm failed (non-fatal): %s", e) + log_tool_action_debug(logger, "Persist cleanup failed (non-fatal)", e) remount_error: WorkspaceArchiveReadError | None = None for mount_entry, mount_path in reversed(unmounted_mounts): @@ -764,7 +767,7 @@ async def hydrate_workspace(self, data: io.IOBase) -> None: "rm", "-f", "--", tar_path, timeout=self.state.timeouts.cleanup_s ) except Exception as e: - logger.debug("hydrate cleanup rm failed (non-fatal): %s", e) + log_tool_action_debug(logger, "Hydrate cleanup failed (non-fatal)", e) # -- PTY ----------------------------------------------------------------- @@ -947,7 +950,7 @@ async def _pty_ws_reader(self, entry: _BlaxelPtySessionEntry) -> None: ): break except Exception as e: - logger.debug("PTY ws reader terminated with error: %s", e) + log_tool_action_debug(logger, "PTY WebSocket reader terminated with an error", e) finally: entry.done = True entry.output_notify.set() @@ -1018,14 +1021,14 @@ async def _terminate_pty_entry(self, entry: _BlaxelPtySessionEntry) -> None: try: await entry.ws.close() except Exception as e: - logger.debug("PTY ws close error (non-fatal): %s", e) + log_tool_action_debug(logger, "PTY WebSocket close failed (non-fatal)", e) if entry.http_session is not None: try: await entry.http_session.close() except Exception as e: - logger.debug("PTY http session close error (non-fatal): %s", e) + log_tool_action_debug(logger, "PTY HTTP session close failed (non-fatal)", e) except Exception as e: - logger.debug("PTY entry termination error (non-fatal): %s", e) + log_tool_action_debug(logger, "PTY entry termination failed (non-fatal)", e) # --------------------------------------------------------------------------- @@ -1126,7 +1129,7 @@ async def delete(self, session: SandboxSession) -> SandboxSession: try: await inner.shutdown() except Exception as e: - logger.warning("shutdown error during delete (non-fatal): %s", e) + log_tool_action_warning(logger, "Shutdown failed during delete (non-fatal)", e) return session async def resume( @@ -1152,7 +1155,7 @@ async def resume( blaxel_sandbox = await SandboxInstance.get(state.sandbox_name) reconnected = True except Exception as e: - logger.debug("sandbox get() failed, will recreate: %s", e) + log_tool_action_debug(logger, "Sandbox lookup failed; recreating", e) if not reconnected or blaxel_sandbox is None: create_config = _build_create_config( diff --git a/src/agents/extensions/sandbox/cloudflare/sandbox.py b/src/agents/extensions/sandbox/cloudflare/sandbox.py index a3f94ec591..bbaf4138e5 100644 --- a/src/agents/extensions/sandbox/cloudflare/sandbox.py +++ b/src/agents/extensions/sandbox/cloudflare/sandbox.py @@ -29,6 +29,8 @@ import aiohttp +from .... import _debug +from ....logger import log_tool_action_debug from ....sandbox.errors import ( ConfigurationError, ErrorCode, @@ -694,12 +696,15 @@ async def _shutdown_backend(self) -> None: if resp.status < 400 or resp.status == 404: return detail = await _read_cloudflare_response_body(resp) - logger.debug( - "Failed to delete Cloudflare sandbox on shutdown: %s", - _cloudflare_http_error_message("DELETE /sandbox", resp.status, detail), - ) - except Exception: - logger.debug("Failed to delete Cloudflare sandbox on shutdown", exc_info=True) + if _debug.DONT_LOG_TOOL_DATA: + logger.debug("Failed to delete Cloudflare sandbox on shutdown") + else: + logger.debug( + "Failed to delete Cloudflare sandbox on shutdown: %s", + _cloudflare_http_error_message("DELETE /sandbox", resp.status, detail), + ) + except Exception as exc: + log_tool_action_debug(logger, "Failed to delete Cloudflare sandbox on shutdown", exc) async def _after_shutdown(self) -> None: await self._close_http() @@ -846,7 +851,10 @@ async def _pump_ws_output(self, entry: _CloudflarePtyProcessEntry) -> None: try: payload = json.loads(msg.data) except json.JSONDecodeError: - logger.debug("Ignoring non-JSON PTY text frame: %s", msg.data) + if _debug.DONT_LOG_TOOL_DATA: + logger.debug("Ignoring non-JSON PTY text frame") + else: + logger.debug("Ignoring non-JSON PTY text frame: %s", msg.data) continue msg_type = payload.get("type") @@ -859,7 +867,10 @@ async def _pump_ws_output(self, entry: _CloudflarePtyProcessEntry) -> None: entry.output_notify.set() break if msg_type == "error": - logger.warning("Cloudflare PTY error frame: %s", payload.get("message")) + if _debug.DONT_LOG_TOOL_DATA: + logger.warning("Cloudflare PTY error frame") + else: + logger.warning("Cloudflare PTY error frame: %s", payload.get("message")) entry.output_closed.set() entry.output_notify.set() break @@ -875,8 +886,8 @@ async def _pump_ws_output(self, entry: _CloudflarePtyProcessEntry) -> None: break except asyncio.CancelledError: raise - except Exception: - logger.debug("Cloudflare PTY pump ended with an exception", exc_info=True) + except Exception as exc: + log_tool_action_debug(logger, "Cloudflare PTY pump ended with an exception", exc) entry.output_closed.set() entry.output_notify.set() diff --git a/src/agents/extensions/sandbox/daytona/sandbox.py b/src/agents/extensions/sandbox/daytona/sandbox.py index 294afc6cf3..988d5a2778 100644 --- a/src/agents/extensions/sandbox/daytona/sandbox.py +++ b/src/agents/extensions/sandbox/daytona/sandbox.py @@ -26,6 +26,7 @@ from pydantic import BaseModel, Field +from ....logger import log_tool_action_debug from ....sandbox.entries import Mount from ....sandbox.errors import ( ExecTimeoutError, @@ -1335,7 +1336,7 @@ async def resume( await daytona_sandbox.start(timeout=state.start_timeout) reconnected = True except Exception as e: - logger.debug("daytona sandbox get() failed, will recreate: %s", e) + log_tool_action_debug(logger, "Daytona sandbox lookup failed; recreating", e) if not reconnected or daytona_sandbox is None: params = await self._build_create_params( diff --git a/src/agents/extensions/sandbox/e2b/sandbox.py b/src/agents/extensions/sandbox/e2b/sandbox.py index 324d08bfdf..ecbe8bc0bf 100644 --- a/src/agents/extensions/sandbox/e2b/sandbox.py +++ b/src/agents/extensions/sandbox/e2b/sandbox.py @@ -34,6 +34,7 @@ from pydantic import BaseModel, Field +from ....logger import log_tool_action_warning from ....sandbox.entries import Mount from ....sandbox.errors import ( ExecNonZeroError, @@ -858,6 +859,12 @@ async def _after_start_failed(self) -> None: async def _shutdown_backend(self) -> None: # Best-effort kill of the remote sandbox. + def diagnostic_extra() -> dict[str, object]: + return { + "sandbox_id": self.state.sandbox_id, + "pause_on_exit": self.state.pause_on_exit, + } + try: if self.state.pause_on_exit: await _sandbox_pause(self._sandbox) @@ -865,33 +872,27 @@ async def _shutdown_backend(self) -> None: await _sandbox_kill(self._sandbox) except Exception as e: if self.state.pause_on_exit: - logger.warning( + log_tool_action_warning( + logger, "Failed to pause E2B sandbox on shutdown; falling back to kill.", - extra={ - "sandbox_id": self.state.sandbox_id, - "pause_on_exit": self.state.pause_on_exit, - }, - exc_info=e, + e, + diagnostic_extra=diagnostic_extra, ) try: await _sandbox_kill(self._sandbox) except Exception as kill_exc: - logger.warning( + log_tool_action_warning( + logger, "Failed to kill E2B sandbox after pause fallback failure.", - extra={ - "sandbox_id": self.state.sandbox_id, - "pause_on_exit": self.state.pause_on_exit, - }, - exc_info=kill_exc, + kill_exc, + diagnostic_extra=diagnostic_extra, ) else: - logger.warning( + log_tool_action_warning( + logger, "Failed to kill E2B sandbox on shutdown.", - extra={ - "sandbox_id": self.state.sandbox_id, - "pause_on_exit": self.state.pause_on_exit, - }, - exc_info=e, + e, + diagnostic_extra=diagnostic_extra, ) async def _exec_internal( diff --git a/src/agents/extensions/sandbox/modal/sandbox.py b/src/agents/extensions/sandbox/modal/sandbox.py index 930d9f8b59..b70f840119 100644 --- a/src/agents/extensions/sandbox/modal/sandbox.py +++ b/src/agents/extensions/sandbox/modal/sandbox.py @@ -33,6 +33,7 @@ from modal.config import config as modal_config from modal.container_process import ContainerProcess +from ....logger import log_tool_action_warning from ....sandbox.config import DEFAULT_PYTHON_SANDBOX_IMAGE from ....sandbox.entries import Mount from ....sandbox.errors import ( @@ -1327,8 +1328,9 @@ async def restore_ephemeral_paths() -> WorkspaceArchiveReadError | None: if not rm_out.ok(): cleanup_restore_error = await restore_ephemeral_paths() if cleanup_restore_error is not None: - logger.warning( - "Failed to restore Modal ephemeral paths after cleanup failure: %s", + log_tool_action_warning( + logger, + "Failed to restore Modal ephemeral paths after cleanup failure", cleanup_restore_error, ) raise WorkspaceArchiveReadError( @@ -1352,8 +1354,9 @@ async def restore_ephemeral_paths() -> WorkspaceArchiveReadError | None: except Exception as e: restore_error = await restore_ephemeral_paths() if restore_error is not None: - logger.warning( - "Failed to restore Modal ephemeral paths after snapshot failure: %s", + log_tool_action_warning( + logger, + "Failed to restore Modal ephemeral paths after snapshot failure", restore_error, ) raise WorkspaceArchiveReadError( diff --git a/src/agents/logger.py b/src/agents/logger.py index bd81a82716..14cf61e04d 100644 --- a/src/agents/logger.py +++ b/src/agents/logger.py @@ -1,3 +1,225 @@ import logging +from collections.abc import Callable, Mapping + +from . import _debug logger = logging.getLogger("openai.agents") + +_DiagnosticExtra = Callable[[], Mapping[str, object]] + + +def _log_action_error( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + redact: bool, + stacklevel: int, + diagnostic_extra: _DiagnosticExtra | None, +) -> None: + """Log an action failure without inspecting a redacted exception.""" + if redact: + target_logger.error("%s", message, stacklevel=stacklevel) + else: + target_logger.error( + "%s: %s", + message, + exc, + exc_info=exc, + extra=diagnostic_extra() if diagnostic_extra is not None else None, + stacklevel=stacklevel, + ) + + +def _log_action_at_level( + log_method: Callable[..., None], + message: str, + exc: BaseException, + *, + redact: bool, + stacklevel: int, + diagnostic_extra: _DiagnosticExtra | None, +) -> None: + """Log an action failure at a caller-selected level.""" + if redact: + log_method("%s", message, stacklevel=stacklevel) + else: + log_method( + "%s: %s", + message, + exc, + exc_info=exc, + extra=diagnostic_extra() if diagnostic_extra is not None else None, + stacklevel=stacklevel, + ) + + +def log_model_action_error( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Log a model-data failure according to the model logging policy.""" + _log_action_error( + target_logger, + message, + exc, + redact=_debug.DONT_LOG_MODEL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_model_action_debug( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Debug-log a model-data failure according to the model logging policy.""" + _log_action_at_level( + target_logger.debug, + message, + exc, + redact=_debug.DONT_LOG_MODEL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_model_action_warning( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Warning-log a model-data failure according to the model logging policy.""" + _log_action_at_level( + target_logger.warning, + message, + exc, + redact=_debug.DONT_LOG_MODEL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_tool_action_error( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Log a tool-data failure according to the tool logging policy.""" + _log_action_error( + target_logger, + message, + exc, + redact=_debug.DONT_LOG_TOOL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_tool_action_debug( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Debug-log a tool-data failure according to the tool logging policy.""" + _log_action_at_level( + target_logger.debug, + message, + exc, + redact=_debug.DONT_LOG_TOOL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_tool_action_warning( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Warning-log a tool-data failure according to the tool logging policy.""" + _log_action_at_level( + target_logger.warning, + message, + exc, + redact=_debug.DONT_LOG_TOOL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_model_and_tool_action_error( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Log a mixed model/tool-data failure only when both data policies allow it.""" + _log_action_error( + target_logger, + message, + exc, + redact=_debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_model_and_tool_action_debug( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Debug-log a mixed-data failure only when both data policies allow it.""" + _log_action_at_level( + target_logger.debug, + message, + exc, + redact=_debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) + + +def log_model_and_tool_action_warning( + target_logger: logging.Logger, + message: str, + exc: BaseException, + *, + stacklevel: int = 3, + diagnostic_extra: _DiagnosticExtra | None = None, +) -> None: + """Warning-log a mixed-data failure only when both data policies allow it.""" + _log_action_at_level( + target_logger.warning, + message, + exc, + redact=_debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA, + stacklevel=stacklevel, + diagnostic_extra=diagnostic_extra, + ) diff --git a/src/agents/mcp/_logging.py b/src/agents/mcp/_logging.py new file mode 100644 index 0000000000..ffb6df97b5 --- /dev/null +++ b/src/agents/mcp/_logging.py @@ -0,0 +1,52 @@ +from typing import Protocol +from urllib.parse import urlsplit, urlunsplit + +from .. import _debug + +_URL_DERIVED_NAME_PREFIXES = ("sse: ", "streamable_http: ", "streamable-http: ") + + +class _MCPServerNameSource(Protocol): + @property + def name(self) -> str: ... + + +def get_mcp_server_log_name(name: str) -> str: + """Remove URL credentials, query parameters, and fragments from MCP log names.""" + prefix = next( + (candidate for candidate in _URL_DERIVED_NAME_PREFIXES if name.startswith(candidate)), + "", + ) + candidate = name[len(prefix) :] if prefix else name + + try: + parsed = urlsplit(candidate) + except ValueError: + if prefix or candidate.lower().startswith(("http://", "https://")): + return f"{prefix}" + return name + + if parsed.scheme not in {"http", "https"}: + return name + + try: + hostname = parsed.hostname + port = parsed.port + except ValueError: + return f"{prefix}" + + if not parsed.netloc or not hostname or any(character.isspace() for character in hostname): + return f"{prefix}" + + host = f"[{hostname}]" if ":" in hostname else hostname + if port is not None: + host = f"{host}:{port}" + sanitized = urlunsplit((parsed.scheme, host, parsed.path, "", "")) + return f"{prefix}{sanitized}" + + +def get_mcp_server_log_message(message: str, server: _MCPServerNameSource) -> str: + """Build an MCP log message without reading the server name in redacted mode.""" + if _debug.DONT_LOG_TOOL_DATA: + return message + return f"{message} '{get_mcp_server_log_name(server.name)}'" diff --git a/src/agents/mcp/manager.py b/src/agents/mcp/manager.py index 235816819d..b8838be3e3 100644 --- a/src/agents/mcp/manager.py +++ b/src/agents/mcp/manager.py @@ -6,7 +6,8 @@ from dataclasses import dataclass from typing import Any -from ..logger import logger +from ..logger import log_tool_action_debug, log_tool_action_error, logger +from ._logging import get_mcp_server_log_message from .server import MCPServer @@ -260,10 +261,18 @@ async def cleanup_all(self) -> None: except asyncio.CancelledError as exc: if not self.suppress_cancelled_error: raise - logger.debug("Cleanup cancelled for MCP server '%s': %s", server.name, exc) + log_tool_action_debug( + logger, + get_mcp_server_log_message("Cleanup cancelled for MCP server", server), + exc, + ) self.errors[server] = exc except Exception as exc: - logger.exception("Failed to cleanup MCP server '%s': %s", server.name, exc) + log_tool_action_error( + logger, + get_mcp_server_log_message("Failed to cleanup MCP server", server), + exc, + ) self.errors[server] = exc async def _run_with_timeout( @@ -305,7 +314,11 @@ def _refresh_active_servers(self) -> None: self._active_servers = list(self._all_servers) def _record_failure(self, server: MCPServer, exc: BaseException, phase: str) -> None: - logger.exception("Failed to %s MCP server '%s': %s", phase, server.name, exc) + log_tool_action_error( + logger, + get_mcp_server_log_message(f"Failed to {phase} MCP server", server), + exc, + ) if server not in self._failed_server_set: self.failed_servers.append(server) self._failed_server_set.add(server) @@ -343,10 +356,18 @@ async def _cleanup_servers(self, servers: Iterable[MCPServer]) -> None: except asyncio.CancelledError as exc: if not self.suppress_cancelled_error: raise - logger.debug("Cleanup cancelled for MCP server '%s': %s", server.name, exc) + log_tool_action_debug( + logger, + get_mcp_server_log_message("Cleanup cancelled for MCP server", server), + exc, + ) self.errors[server] = exc except Exception as exc: - logger.exception("Failed to cleanup MCP server '%s': %s", server.name, exc) + log_tool_action_error( + logger, + get_mcp_server_log_message("Failed to cleanup MCP server", server), + exc, + ) self.errors[server] = exc async def _connect_all_parallel(self, servers: list[MCPServer]) -> None: diff --git a/src/agents/mcp/server.py b/src/agents/mcp/server.py index 2681606b5e..4f8f89c293 100644 --- a/src/agents/mcp/server.py +++ b/src/agents/mcp/server.py @@ -38,11 +38,18 @@ ) from typing_extensions import NotRequired, TypedDict +from .. import _debug from ..exceptions import UserError -from ..logger import logger +from ..logger import ( + log_tool_action_debug, + log_tool_action_error, + log_tool_action_warning, + logger, +) from ..run_context import RunContextWrapper from ..tool import ToolErrorFunction from ..util._types import MaybeAwaitable +from ._logging import get_mcp_server_log_message, get_mcp_server_log_name from .util import ( HttpClientFactory, MCPToolCustomDataExtractor, @@ -119,10 +126,11 @@ async def _handle_post_request(self, ctx: Any) -> None: try: await super()._handle_post_request(ctx) - except httpx.HTTPError: - logger.warning( + except httpx.HTTPError as exc: + log_tool_action_warning( + logger, "Ignoring initialized notification HTTP failure", - exc_info=True, + exc, ) return @@ -161,7 +169,13 @@ async def _streamablehttp_client_with_transport( async with client: async with anyio.create_task_group() as tg: try: - logger.debug("Connecting to StreamableHTTP endpoint: %s", url) + if _debug.DONT_LOG_TOOL_DATA: + logger.debug("Connecting to StreamableHTTP endpoint") + else: + logger.debug( + "Connecting to StreamableHTTP endpoint: %s", + get_mcp_server_log_name(url), + ) def start_get_stream() -> None: tg.start_soon(transport.handle_get_stream, client, read_stream_writer) @@ -691,12 +705,15 @@ async def _apply_dynamic_tool_filter( if should_include: filtered_tools.append(tool) except Exception as e: - logger.error( - "Error applying tool filter to tool '%s' on server '%s': %s", - tool.name, - self.name, - e, - ) + if _debug.DONT_LOG_TOOL_DATA: + message = "Error applying MCP tool filter" + else: + server_name = get_mcp_server_log_name(self.name) + message = ( + f"Error applying MCP tool filter to tool '{tool.name}' " + f"on server '{server_name}'" + ) + log_tool_action_error(logger, message, e) # On error, exclude the tool for safety continue @@ -818,16 +835,20 @@ async def connect(self): if isinstance(cleanup_error, RuntimeError) and "cancel scope" in str( cleanup_error ): - logger.debug( - "Ignoring cancel scope error during cleanup of MCP server '%s': %s", - self.name, + log_tool_action_debug( + logger, + get_mcp_server_log_message( + "Ignoring cancel scope error during cleanup of MCP server", self + ), cleanup_error, ) else: # Log other cleanup errors but don't raise - original error is more # important - logger.warning( - "Error during cleanup of MCP server '%s': %s", self.name, cleanup_error + log_tool_action_warning( + logger, + get_mcp_server_log_message("Error during cleanup of MCP server", self), + cleanup_error, ) async def list_tools( @@ -1005,7 +1026,11 @@ async def cleanup(self): try: await self.exit_stack.aclose() except asyncio.CancelledError as e: - logger.debug("Cleanup cancelled for MCP server '%s': %s", self.name, e) + log_tool_action_debug( + logger, + get_mcp_server_log_message("Cleanup cancelled for MCP server", self), + e, + ) raise except BaseExceptionGroup as eg: # Extract HTTP errors from ExceptionGroup raised during cleanup @@ -1031,9 +1056,11 @@ async def cleanup(self): raise UserError(error_message) from http_error else: # Normal teardown - log but don't raise - logger.warning( - "HTTP error during cleanup of MCP server '%s': %s", - self.name, + log_tool_action_warning( + logger, + get_mcp_server_log_message( + "HTTP error during cleanup of MCP server", self + ), http_error, ) elif connect_error: @@ -1041,9 +1068,11 @@ async def cleanup(self): error_message += "Could not reach the server." raise UserError(error_message) from connect_error else: - logger.warning( - "Connection error during cleanup of MCP server '%s': %s", - self.name, + log_tool_action_warning( + logger, + get_mcp_server_log_message( + "Connection error during cleanup of MCP server", self + ), connect_error, ) elif timeout_error: @@ -1051,9 +1080,11 @@ async def cleanup(self): error_message += "Connection timeout." raise UserError(error_message) from timeout_error else: - logger.warning( - "Timeout error during cleanup of MCP server '%s': %s", - self.name, + log_tool_action_warning( + logger, + get_mcp_server_log_message( + "Timeout error during cleanup of MCP server", self + ), timeout_error, ) else: @@ -1063,16 +1094,36 @@ async def cleanup(self): for exc in eg.exceptions ) if has_cancel_scope_error: - logger.debug("Ignoring cancel scope error during cleanup: %s", eg) + log_tool_action_debug( + logger, + get_mcp_server_log_message( + "Ignoring cancel scope error during cleanup of MCP server", self + ), + eg, + ) else: - logger.error("Error cleaning up server: %s", eg) + log_tool_action_error( + logger, + get_mcp_server_log_message("Error cleaning up MCP server", self), + eg, + ) except Exception as e: # Suppress RuntimeError about cancel scopes - this is a known issue with the MCP # library when background tasks fail during async generator cleanup if isinstance(e, RuntimeError) and "cancel scope" in str(e): - logger.debug("Ignoring cancel scope error during cleanup: %s", e) + log_tool_action_debug( + logger, + get_mcp_server_log_message( + "Ignoring cancel scope error during cleanup of MCP server", self + ), + e, + ) else: - logger.error("Error cleaning up server: %s", e) + log_tool_action_error( + logger, + get_mcp_server_log_message("Error cleaning up MCP server", self), + e, + ) finally: self.session = None self._get_session_id = None diff --git a/src/agents/mcp/util.py b/src/agents/mcp/util.py index 2cb0a5595b..049b4561ba 100644 --- a/src/agents/mcp/util.py +++ b/src/agents/mcp/util.py @@ -23,7 +23,7 @@ from mcp.shared.exceptions import McpError as _McpError except ImportError: # pragma: no cover – mcp is optional on Python < 3.10 _McpError = None # type: ignore[assignment, misc] -from ..logger import logger +from ..logger import log_tool_action_error, logger from ..run_context import RunContextWrapper from ..strict_schema import ensure_strict_json_schema from ..tool import ( @@ -42,6 +42,7 @@ from ..tracing import FunctionSpanData, get_current_span, mcp_tools_span from ..util._custom_data import maybe_extract_custom_data from ..util._types import MaybeAwaitable +from ._logging import get_mcp_server_log_message, get_mcp_server_log_name if TYPE_CHECKING: ToolOutputItem = ToolOutputTextDict | ToolOutputImageDict @@ -546,7 +547,10 @@ def to_function_tool( schema = ensure_strict_json_schema(copy.deepcopy(schema)) is_strict = True except Exception as e: - logger.info("Error converting MCP schema to strict mode: %s", e) + if _debug.DONT_LOG_TOOL_DATA: + logger.info("Error converting MCP schema to strict mode") + else: + logger.info("Error converting MCP schema to strict mode: %s", e) needs_approval: ( bool | Callable[[RunContextWrapper[Any], dict[str, Any], str], Awaitable[bool]] @@ -671,7 +675,7 @@ async def invoke_mcp_tool( if json_decode_error is not None: error_message = f"Invalid JSON input for tool {tool_name_for_display}" if _debug.DONT_LOG_TOOL_DATA: - logger.debug(error_message) + logger.debug("Invalid JSON input for MCP tool") raise ModelBehaviorError(error_message) else: error_message = f"{error_message}: {input_json}" @@ -684,7 +688,7 @@ async def invoke_mcp_tool( ) if _debug.DONT_LOG_TOOL_DATA: - logger.debug("Invoking MCP tool %s", tool_name_for_display) + logger.debug("Invoking MCP tool") else: logger.debug("Invoking MCP tool %s with input %s", tool_name_for_display, input_json) @@ -725,41 +729,30 @@ async def invoke_mcp_tool( # will surface the message as a structured error result; callers who set # failure_error_function=None will have the error raised as documented. if _debug.DONT_LOG_TOOL_DATA: - logger.warning( - "MCP tool %s on server '%s' returned an error.", - tool_name_for_display, - server.name, - ) + logger.warning("MCP tool returned an error.") else: + server_log_name = get_mcp_server_log_name(server.name) error_text = e.error.message if hasattr(e, "error") and e.error else str(e) logger.warning( "MCP tool %s on server '%s' returned an error: %s", tool_name_for_display, - server.name, + server_log_name, error_text, ) raise - if _debug.DONT_LOG_TOOL_DATA: - logger.error( - "Error invoking MCP tool %s on server '%s': %s", - tool_name_for_display, - server.name, - e.__class__.__name__, - ) - else: - logger.error( - "Error invoking MCP tool %s on server '%s': %s", - tool_name_for_display, - server.name, - e, + log_message = "Error invoking MCP tool" + if not _debug.DONT_LOG_TOOL_DATA: + log_message = get_mcp_server_log_message( + f"Error invoking MCP tool {tool_name_for_display} on server", server ) + log_tool_action_error(logger, log_message, e) raise AgentsException( f"Error invoking MCP tool {tool_name_for_display} on server '{server.name}': {e}" ) from e if _debug.DONT_LOG_TOOL_DATA: - logger.debug("MCP tool %s completed.", tool_name_for_display) + logger.debug("MCP tool completed.") else: logger.debug("MCP tool %s returned %s", tool_name_for_display, result) @@ -811,8 +804,12 @@ async def invoke_mcp_tool( "server": server.name, } else: - logger.warning( - "Current span is not a FunctionSpanData, skipping tool output: %s", current_span - ) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.warning("Current span is not a FunctionSpanData; skipping tool output") + else: + logger.warning( + "Current span is not a FunctionSpanData, skipping tool output: %s", + current_span, + ) return tool_output diff --git a/src/agents/memory/openai_responses_compaction_session.py b/src/agents/memory/openai_responses_compaction_session.py index 2ec40663db..8263a81036 100644 --- a/src/agents/memory/openai_responses_compaction_session.py +++ b/src/agents/memory/openai_responses_compaction_session.py @@ -7,6 +7,7 @@ from openai import AsyncOpenAI from ..items import TResponseInputItem +from ..logger import log_model_and_tool_action_warning from ..models._openai_shared import get_default_openai_client from ..run_internal.items import normalize_input_items_for_api from .openai_conversations_session import OpenAIConversationsSession @@ -273,10 +274,11 @@ async def _restore_underlying_session_items_after_failed_clear( ) -> None: try: current_items = await self._get_all_underlying_session_items() - except Exception: - logger.warning( + except Exception as inspection_error: + log_model_and_tool_action_warning( + logger, "Failed to inspect session history after compaction replacement clear failed.", - exc_info=True, + inspection_error, ) return @@ -299,15 +301,17 @@ async def _restore_underlying_session_items( await self.underlying_session.clear_session() if previous_items: await self.underlying_session.add_items(list(previous_items)) - except Exception: - logger.warning( + except Exception as restore_error: + log_model_and_tool_action_warning( + logger, "Failed to restore session history after compaction replacement failed.", - exc_info=True, + restore_error, ) return - logger.warning( - "Restored previous session history after compaction replacement failed: %s", + log_model_and_tool_action_warning( + logger, + "Restored previous session history after compaction replacement failed", replacement_error, ) diff --git a/src/agents/models/openai_chatcompletions.py b/src/agents/models/openai_chatcompletions.py index 408e8c8ad5..b7f9d8e00a 100644 --- a/src/agents/models/openai_chatcompletions.py +++ b/src/agents/models/openai_chatcompletions.py @@ -25,7 +25,7 @@ from ..exceptions import ModelBehaviorError, UserError from ..handoffs import Handoff from ..items import ModelResponse, TResponseInputItem, TResponseStreamEvent -from ..logger import logger +from ..logger import log_model_action_debug, logger from ..retry import ModelRetryAdvice, ModelRetryAdviceRequest from ..tool import Tool from ..tracing import generation_span @@ -149,7 +149,9 @@ def _consume_background_cleanup_task_result(task: asyncio.Task[Any]) -> None: except asyncio.CancelledError: pass except Exception as exc: - logger.debug("Background stream cleanup failed after cancellation: %s", exc) + log_model_action_debug( + logger, "Background stream cleanup failed after cancellation", exc + ) def _validate_official_openai_input_content_types( self, request_input: str | list[TResponseInputItem] @@ -387,8 +389,10 @@ async def stream_response( await self._maybe_aclose_async_iterator(stream) except Exception as exc: if yielded_terminal_event: - logger.debug( - "Ignoring stream cleanup error after terminal event: %s", exc + log_model_action_debug( + logger, + "Ignoring stream cleanup error after terminal event", + exc, ) else: raise diff --git a/src/agents/models/openai_responses.py b/src/agents/models/openai_responses.py index 55c78d0245..ff17cea4a0 100644 --- a/src/agents/models/openai_responses.py +++ b/src/agents/models/openai_responses.py @@ -50,7 +50,7 @@ from ..exceptions import ModelBehaviorError, UserError from ..handoffs import Handoff from ..items import ItemHelpers, ModelResponse, TResponseInputItem -from ..logger import logger +from ..logger import log_model_action_debug, log_model_action_error, logger from ..model_settings import MCPToolChoice from ..retry import ModelRetryAdvice, ModelRetryAdviceRequest from ..tool import ( @@ -300,7 +300,9 @@ async def _cleanup_after_exhaustion(self) -> None: await self._cleanup_once() except Exception as exc: if self._yielded_terminal_event: - logger.debug("Ignoring stream cleanup error after terminal event: %s", exc) + log_model_action_debug( + logger, "Ignoring stream cleanup error after terminal event", exc + ) return raise @@ -452,7 +454,9 @@ def _consume_background_cleanup_task_result(task: asyncio.Task[Any]) -> None: except asyncio.CancelledError: pass except Exception as exc: - logger.debug("Background stream cleanup failed after cancellation: %s", exc) + log_model_action_debug( + logger, "Background stream cleanup failed after cancellation", exc + ) async def get_response( self, @@ -506,19 +510,16 @@ async def get_response( SpanError( message="Error getting response", data={ - "error": str(e) if tracing.include_data() else e.__class__.__name__, + "error": str(e) + if tracing.include_data() + else "Error details are redacted.", }, ) ) - request_id = getattr(e, "request_id", None) - if _debug.DONT_LOG_MODEL_DATA: - logger.error( - "Error getting response: %s. (request_id: %s)", - e.__class__.__name__, - request_id, - ) - else: - logger.error("Error getting response: %s. (request_id: %s)", e, request_id) + message = "Error getting response" + if not _debug.DONT_LOG_MODEL_DATA: + message = f"{message} (request_id: {getattr(e, 'request_id', None)})" + log_model_action_error(logger, message, e) raise return ModelResponse( @@ -603,8 +604,10 @@ async def stream_response( await self._maybe_aclose_async_iterator(stream) except Exception as exc: if yielded_terminal_event: - logger.debug( - "Ignoring stream cleanup error after terminal event: %s", exc + log_model_action_debug( + logger, + "Ignoring stream cleanup error after terminal event", + exc, ) else: raise @@ -624,14 +627,13 @@ async def stream_response( SpanError( message="Error streaming response", data={ - "error": str(e) if tracing.include_data() else e.__class__.__name__, + "error": str(e) + if tracing.include_data() + else "Error details are redacted.", }, ) ) - if _debug.DONT_LOG_MODEL_DATA: - logger.error("Error streaming response: %s", e.__class__.__name__) - else: - logger.error("Error streaming response: %s", e) + log_model_action_error(logger, "Error streaming response", e) raise @overload diff --git a/src/agents/realtime/agent.py b/src/agents/realtime/agent.py index 38c77619cc..fe2f9e7200 100644 --- a/src/agents/realtime/agent.py +++ b/src/agents/realtime/agent.py @@ -8,6 +8,7 @@ from agents.prompts import Prompt +from .. import _debug from ..agent import AgentBase from ..guardrail import OutputGuardrail from ..handoffs import Handoff @@ -125,6 +126,11 @@ async def get_system_prompt(self, run_context: RunContextWrapper[TContext]) -> s else: return cast(str, self.instructions(run_context, self)) elif self.instructions is not None: - logger.error("Instructions must be a string or a function, got %s", self.instructions) + if _debug.DONT_LOG_MODEL_DATA: + logger.error("Instructions must be a string or a function") + else: + logger.error( + "Instructions must be a string or a function, got %s", self.instructions + ) return None diff --git a/src/agents/realtime/openai_realtime.py b/src/agents/realtime/openai_realtime.py index 7623c7ee18..f36aafffa4 100644 --- a/src/agents/realtime/openai_realtime.py +++ b/src/agents/realtime/openai_realtime.py @@ -199,7 +199,9 @@ def _server_event_validation_summary(error: BaseException) -> str: if isinstance(error, pydantic.ValidationError): return f"{error.error_count()} validation error(s)" - return error.__class__.__name__ + if not _debug.DONT_LOG_MODEL_DATA: + return type(error).__name__ + return "validation failed" def _server_event_identity(event: Any) -> tuple[Any, Any]: diff --git a/src/agents/realtime/session.py b/src/agents/realtime/session.py index 66dc72be5b..9781bc1a0c 100644 --- a/src/agents/realtime/session.py +++ b/src/agents/realtime/session.py @@ -10,6 +10,7 @@ from pydantic import BaseModel from typing_extensions import assert_never +from .. import _debug from .._tool_identity import ( FunctionToolLookupKey, get_function_tool_lookup_key_for_tool, @@ -19,7 +20,7 @@ from ..exceptions import ToolInputGuardrailTripwireTriggered, UserError from ..handoffs import Handoff from ..items import ToolApprovalItem -from ..logger import logger +from ..logger import log_model_action_error, log_tool_action_error, logger from ..run_config import ToolErrorFormatterArgs from ..run_context import RunContextWrapper, TContext from ..tool import DEFAULT_APPROVAL_REJECTION_MESSAGE, FunctionTool, Tool, invoke_function_tool @@ -476,8 +477,8 @@ async def on_event(self, event: RealtimeModelEvent) -> None: if new_content: incoming_item = incoming_item.model_copy(update={"content": new_content}) - except Exception: - logger.error("Error merging transcripts", exc_info=True) + except Exception as exc: + log_model_action_error(logger, "Error merging transcripts", exc) pass self._history = self._get_new_history(self._history, incoming_item) @@ -810,18 +811,21 @@ async def _resolve_approval_rejection_message(self, *, tool: FunctionTool, call_ ) message = await maybe_message if inspect.isawaitable(maybe_message) else maybe_message except Exception as exc: - logger.error("Tool error formatter failed for %s: %s", tool.name, exc) + log_tool_action_error(logger, "Tool error formatter failed", exc) return REJECTION_MESSAGE if message is None: return REJECTION_MESSAGE if not isinstance(message, str): - logger.error( - "Tool error formatter returned non-string for %s: %s", - tool.name, - type(message).__name__, - ) + if _debug.DONT_LOG_TOOL_DATA: + logger.error("Tool error formatter returned a non-string value") + else: + logger.error( + "Tool error formatter returned non-string for %s: %s", + tool.name, + type(message).__name__, + ) return REJECTION_MESSAGE return message @@ -1312,13 +1316,15 @@ async def _run_output_guardrails(self, text: str, response_id: str) -> bool: if result.output.tripwire_triggered: triggered_results.append(result) except Exception as exc: - logger.warning( - "Output guardrail %r raised %s: %s; skipping it.", - guardrail.get_name(), - type(exc).__name__, - exc, - ) - logger.debug("Output guardrail failure details.", exc_info=True) + if _debug.DONT_LOG_MODEL_DATA: + logger.warning("Output guardrail raised an exception; skipping it") + else: + logger.warning( + "Output guardrail %r raised %s; skipping it.", + guardrail.get_name(), + exc, + ) + logger.debug("Output guardrail failure details.", exc_info=exc) continue if triggered_results: @@ -1432,11 +1438,14 @@ def _on_tool_call_task_done(self, task: asyncio.Task[Any]) -> None: return if isinstance(exception, _PendingToolOutputSendError): - logger.warning( - "Realtime tool output send failed for call %s; cached output will be retried", - exception.call_id, - exc_info=exception, - ) + if _debug.DONT_LOG_TOOL_DATA: + logger.warning("Realtime tool output send failed; cached output will be retried") + else: + logger.warning( + "Realtime tool output send failed for call %s; cached output will be retried", + exception.call_id, + exc_info=exception, + ) self._put_event_nowait( RealtimeError( info=self._event_info, @@ -1449,7 +1458,7 @@ def _on_tool_call_task_done(self, task: asyncio.Task[Any]) -> None: ) return - logger.exception("Realtime tool call task failed", exc_info=exception) + log_tool_action_error(logger, "Realtime tool call task failed", exception) if self._stored_exception is None: self._stored_exception = exception diff --git a/src/agents/result.py b/src/agents/result.py index f63a8e7f68..7bccccec91 100644 --- a/src/agents/result.py +++ b/src/agents/result.py @@ -28,7 +28,7 @@ ToolApprovalItem, TResponseInputItem, ) -from .logger import logger +from .logger import log_tool_action_warning, logger from .run_context import RunContextWrapper from .run_internal.items import ( NestedHistoryOwnedItemRef, @@ -661,8 +661,10 @@ async def _cleanup_once() -> None: try: await sandbox_cleanup() except Exception as error: - logger.warning( - "Failed to clean up sandbox resources after streamed run: %s", error + log_tool_action_warning( + logger, + "Failed to clean up sandbox resources after streamed run", + error, ) task = asyncio.create_task(_cleanup_once()) diff --git a/src/agents/run.py b/src/agents/run.py index 04fe9c09a0..1473559048 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -27,7 +27,7 @@ TResponseInputItem, ) from .lifecycle import RunHooks -from .logger import logger +from .logger import log_tool_action_warning, logger from .memory import Session from .result import RunResult, RunResultStreaming from .run_config import ( @@ -1606,10 +1606,14 @@ def _finalize_result(result: RunResult) -> RunResult: terminal_metadata=terminal_metadata_for_exception(run_exception), ) except Exception as error: - logger.warning("Failed to enqueue sandbox memory after run: %s", error) + log_tool_action_warning( + logger, "Failed to enqueue sandbox memory after run", error + ) sandbox_resume_state = await sandbox_runtime.cleanup() except Exception as error: - logger.warning("Failed to clean up sandbox resources after run: %s", error) + log_tool_action_warning( + logger, "Failed to clean up sandbox resources after run", error + ) else: if completed_result is not None: completed_result._sandbox_resume_state = sandbox_resume_state @@ -1619,7 +1623,7 @@ def _finalize_result(result: RunResult) -> RunResult: try: await dispose_resolved_computers(run_context=context_wrapper) except Exception as error: - logger.warning("Failed to dispose computers after run: %s", error) + log_tool_action_warning(logger, "Failed to dispose computers after run", error) if current_span: current_span.finish(reset_current=True) if current_task_span: diff --git a/src/agents/run_internal/model_retry.py b/src/agents/run_internal/model_retry.py index aa41d07e6b..4e37139329 100644 --- a/src/agents/run_internal/model_retry.py +++ b/src/agents/run_internal/model_retry.py @@ -10,7 +10,7 @@ from openai import APIConnectionError, APITimeoutError, BadRequestError from ..items import ModelResponse, TResponseStreamEvent -from ..logger import logger +from ..logger import log_model_action_debug, logger from ..models._retry_runtime import ( get_error_code as _get_error_code, get_request_id as _get_request_id, @@ -246,7 +246,7 @@ async def _close_async_iterator_quietly(iterator: Any | None) -> None: try: await _close_async_iterator(iterator) except Exception as exc: - logger.debug("Ignoring retry stream cleanup error: %s", exc) + log_model_action_debug(logger, "Ignoring retry stream cleanup error", exc) def _get_stream_event_type(event: TResponseStreamEvent) -> str | None: diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index fa60d2299e..848bc23abd 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -56,7 +56,13 @@ coerce_tool_search_output_raw_item, ) from ..lifecycle import RunHooks -from ..logger import logger +from ..logger import ( + log_model_action_debug, + log_model_action_error, + log_model_action_warning, + log_tool_action_warning, + logger, +) from ..memory import Session from ..models._response_terminal import ( response_error_event_failure_error, @@ -270,7 +276,7 @@ async def cleanup_models_after_run(tool_use_tracker: AgentToolUseTracker) -> Non try: await model._cleanup_on_run_end(tool_use_tracker) except Exception as error: - logger.warning("Failed to clean up model resources after run: %s", error) + log_model_action_warning(logger, "Failed to clean up model resources after run", error) def _should_attach_generic_agent_error(exc: Exception) -> bool: @@ -398,8 +404,8 @@ async def _run_output_guardrails_for_stream( raise except asyncio.CancelledError: raise - except Exception: - logger.error("Unexpected error in output guardrails", exc_info=True) + except Exception as exc: + log_model_action_error(logger, "Unexpected error in output guardrails", exc) raise @@ -1302,13 +1308,15 @@ async def _save_stream_items_without_count( if first_trigger is not None: raise InputGuardrailTripwireTriggered(first_trigger) except Exception as e: - logger.debug( - "Error in streamed_result finalize for agent %s - %s", current_agent.name, e + log_model_action_debug( + logger, + f"Error finalizing streamed result for agent {current_agent.name}", + e, ) try: await dispose_resolved_computers(run_context=context_wrapper) except Exception as error: - logger.warning("Failed to dispose computers after streamed run: %s", error) + log_tool_action_warning(logger, "Failed to dispose computers after streamed run", error) if current_span: current_span.finish(reset_current=True) if current_task_span: diff --git a/src/agents/run_internal/session_persistence.py b/src/agents/run_internal/session_persistence.py index 2c599445e7..b4c98d2747 100644 --- a/src/agents/run_internal/session_persistence.py +++ b/src/agents/run_internal/session_persistence.py @@ -13,9 +13,14 @@ from collections.abc import Sequence from typing import Any, cast +from .. import _debug from ..exceptions import UserError from ..items import HandoffOutputItem, ItemHelpers, RunItem, ToolCallOutputItem, TResponseInputItem -from ..logger import logger +from ..logger import ( + log_model_and_tool_action_debug, + log_model_and_tool_action_warning, + logger, +) from ..memory import ( OpenAIResponsesCompactionArgs, Session, @@ -542,8 +547,9 @@ async def rewind_session_items( len(target_serializations), ) - for i, target in enumerate(target_serializations): - logger.debug("Rewind target %d (first 300 chars): %s", i, target[:300]) + if not (_debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA): + for i, target in enumerate(target_serializations): + logger.debug("Rewind target %d (first 300 chars): %s", i, target[:300]) snapshot_serializations = target_serializations.copy() rewound = await _rewind_session_tail_suffix( @@ -554,7 +560,7 @@ async def rewind_session_items( mismatch_warning=( "Skipping session rewind because the current tail does not match the retry-owned suffix" ), - pop_failure_warning="Failed to rewind session item: %s", + pop_failure_warning="Failed to rewind session item", ) if not rewound: return @@ -571,7 +577,7 @@ async def rewind_session_items( try: latest_items = await session.get_items(limit=1) except Exception as exc: - logger.debug("Failed to peek session items while rewinding: %s", exc) + log_model_and_tool_action_debug(logger, "Failed to peek session items while rewinding", exc) return if not latest_items: @@ -584,7 +590,9 @@ async def rewind_session_items( try: session_items = await session.get_items() except Exception as exc: - logger.debug("Failed to inspect session tail while stripping stray items: %s", exc) + log_model_and_tool_action_debug( + logger, "Failed to inspect session tail while stripping stray items", exc + ) return stray_serializations = _collect_retry_owned_tail_serializations( @@ -609,7 +617,7 @@ async def rewind_session_items( "Skipping stray session cleanup because the current tail no longer matches " "retry-owned conversation items" ), - pop_failure_warning="Failed to strip stray session item: %s", + pop_failure_warning="Failed to strip stray session item", ) @@ -633,7 +641,9 @@ async def wait_for_session_cleanup( try: tail_items = await session.get_items(limit=window) except Exception as exc: - logger.debug("Failed to verify session cleanup (attempt %d): %s", attempt + 1, exc) + log_model_and_tool_action_debug( + logger, f"Failed to verify session cleanup (attempt {attempt + 1})", exc + ) await asyncio.sleep(0.1 * (attempt + 1)) continue @@ -764,7 +774,7 @@ async def _rewind_session_tail_suffix( try: tail_items = await session.get_items(limit=len(expected_serializations)) except Exception as exc: - logger.warning(pop_failure_warning, exc) + log_model_and_tool_action_warning(logger, pop_failure_warning, exc) return False if len(tail_items) != len(expected_serializations): @@ -791,7 +801,7 @@ async def _rewind_session_tail_suffix( result = await result except Exception as exc: await _restore_popped_session_items(session, popped_items) - logger.warning(pop_failure_warning, exc) + log_model_and_tool_action_warning(logger, pop_failure_warning, exc) return False if result is None: @@ -827,7 +837,9 @@ async def _restore_popped_session_items( if inspect.isawaitable(result): await result except Exception as exc: - logger.warning("Failed to restore session items after a rewind mismatch: %s", exc) + log_model_and_tool_action_warning( + logger, "Failed to restore session items after a rewind mismatch", exc + ) def _collect_retry_owned_tail_serializations( diff --git a/src/agents/run_internal/tool_execution.py b/src/agents/run_internal/tool_execution.py index 3eea93d8d6..efcecf6a61 100644 --- a/src/agents/run_internal/tool_execution.py +++ b/src/agents/run_internal/tool_execution.py @@ -58,7 +58,7 @@ ToolApprovalItem, ToolCallOutputItem, ) -from ..logger import logger +from ..logger import log_tool_action_error as _log_tool_action_error, logger from ..model_settings import ModelSettings from ..run_config import RunConfig, ToolErrorFormatterArgs from ..run_context import RunContextWrapper @@ -1048,10 +1048,7 @@ def log_tool_action_error(message: str, exc: Exception | BaseException) -> None: redacted by default (matching ``_debug.DONT_LOG_TOOL_DATA``). The full exception and traceback are logged only when tool-data logging is explicitly enabled. """ - if _debug.DONT_LOG_TOOL_DATA: - logger.error("%s: %s", message, exc.__class__.__name__) - else: - logger.error("%s: %s", message, exc, exc_info=True) + _log_tool_action_error(logger, message, exc, stacklevel=4) async def with_tool_function_span( @@ -1208,11 +1205,14 @@ async def resolve_approval_rejection_message( return REJECTION_MESSAGE if not isinstance(message, str): - logger.error( - "Tool error formatter returned non-string for %s: %s", - tool_name, - type(message).__name__, - ) + if _debug.DONT_LOG_TOOL_DATA: + logger.error("Tool error formatter returned a non-string value") + else: + logger.error( + "Tool error formatter returned non-string for %s: %s", + tool_name, + type(message).__name__, + ) return REJECTION_MESSAGE return message diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index f275f2e857..2f54478e32 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -29,6 +29,7 @@ ) from openai.types.responses.response_reasoning_item import ResponseReasoningItem +from .. import _debug from .._mcp_tool_metadata import collect_mcp_list_tools_metadata from .._tool_identity import ( build_function_tool_lookup_map, @@ -72,7 +73,7 @@ coerce_tool_search_output_raw_item, ) from ..lifecycle import RunHooks -from ..logger import logger +from ..logger import log_tool_action_error, logger from ..run_config import RunConfig, ToolErrorFormatterArgs from ..run_context import AgentHookContext, RunContextWrapper, TContext from ..run_error_handlers import RunErrorHandlers @@ -252,18 +253,21 @@ async def _resolve_tool_not_found_message( ) message = await maybe_message if inspect.isawaitable(maybe_message) else maybe_message except Exception as exc: - logger.error("Tool error formatter failed for missing tool %s: %s", tool_name, exc) + log_tool_action_error(logger, "Tool error formatter failed for missing tool", exc) return default_message if message is None: return default_message if not isinstance(message, str): - logger.error( - "Tool error formatter returned non-string for missing tool %s: %s", - tool_name, - type(message).__name__, - ) + if _debug.DONT_LOG_TOOL_DATA: + logger.error("Tool error formatter returned a non-string value for a missing tool") + else: + logger.error( + "Tool error formatter returned non-string for missing tool %s: %s", + tool_name, + type(message).__name__, + ) return default_message return message diff --git a/src/agents/run_state.py b/src/agents/run_state.py index 41c2d6f018..ef9cdc6c76 100644 --- a/src/agents/run_state.py +++ b/src/agents/run_state.py @@ -81,7 +81,7 @@ coerce_tool_search_call_raw_item, coerce_tool_search_output_raw_item, ) -from .logger import logger +from .logger import log_model_and_tool_action_warning, logger from .run_context import RunContextWrapper from .run_internal.items import ( NestedHistoryOwnedItemRef, @@ -3752,7 +3752,9 @@ def _resolve_agent_info( except UserError: raise except Exception as e: - logger.warning("Failed to deserialize item of type %s: %s", item_type, e) + log_model_and_tool_action_warning( + logger, f"Failed to deserialize item of type {item_type}", e + ) continue return result diff --git a/src/agents/sandbox/memory/manager.py b/src/agents/sandbox/memory/manager.py index 28025466dc..9919d8035b 100644 --- a/src/agents/sandbox/memory/manager.py +++ b/src/agents/sandbox/memory/manager.py @@ -10,6 +10,7 @@ from ...exceptions import UserError from ...items import TResponseInputItem +from ...logger import log_model_and_tool_action_error from ...run_config import RunConfig, SandboxRunConfig from ..capabilities.memory import Memory from ..config import MemoryGenerateConfig @@ -150,8 +151,8 @@ async def _worker(self) -> None: if queue_item is _STOP: return await self._process_rollout_file(str(queue_item)) - except Exception: - logger.exception("Sandbox memory worker failed") + except Exception as exc: + log_model_and_tool_action_error(logger, "Sandbox memory worker failed", exc) finally: self._queue.task_done() @@ -227,8 +228,8 @@ async def _run_phase_two(self) -> None: selection=selection, run_config=self._memory_run_config(), ) - except Exception: - logger.exception("Sandbox memory phase 2 failed") + except Exception as exc: + log_model_and_tool_action_error(logger, "Sandbox memory phase 2 failed", exc) return await self._storage.write_phase_two_selection(selected_items=selection.selected) self._pending_phase_two_rollout_ids = [ diff --git a/src/agents/sandbox/runtime.py b/src/agents/sandbox/runtime.py index d273a54411..cfd1372251 100644 --- a/src/agents/sandbox/runtime.py +++ b/src/agents/sandbox/runtime.py @@ -9,6 +9,7 @@ from ..agent import Agent from ..exceptions import UserError from ..items import TResponseInputItem +from ..logger import log_tool_action_warning from ..result import RunResult, RunResultStreaming from ..run_config import RunConfig from ..run_context import RunContextWrapper, TContext @@ -106,8 +107,10 @@ async def _cleanup_and_store() -> None: input_override=_stream_memory_input_override(result), ) except Exception as error: - logger.warning( - "Failed to enqueue sandbox memory after streamed run: %s", error + log_tool_action_warning( + logger, + "Failed to enqueue sandbox memory after streamed run", + error, ) payload = await self.cleanup() result._sandbox_resume_state = payload diff --git a/src/agents/sandbox/sandboxes/unix_local.py b/src/agents/sandbox/sandboxes/unix_local.py index a6aff578b9..6115faa328 100644 --- a/src/agents/sandbox/sandboxes/unix_local.py +++ b/src/agents/sandbox/sandboxes/unix_local.py @@ -27,6 +27,7 @@ from pathlib import Path from typing import Literal, cast +from ...logger import log_tool_action_warning from ..errors import ( ExecNonZeroError, ExecTimeoutError, @@ -1129,12 +1130,12 @@ async def delete(self, session: SandboxSession) -> SandboxSession: for mount_entry, mount_path in inner.state.manifest.ephemeral_mount_targets(): try: await mount_entry.unmount(inner, mount_path, Path("/")) - except Exception: + except Exception as exc: unmount_failed = True - logger.warning( - "Failed to unmount UnixLocal workspace mount before deleting root: %s", - mount_path, - exc_info=True, + log_tool_action_warning( + logger, + "Failed to unmount UnixLocal workspace mount before deleting root", + exc, ) if unmount_failed: return session diff --git a/src/agents/sandbox/session/manager.py b/src/agents/sandbox/session/manager.py index 125765e65b..0cd1fb4298 100644 --- a/src/agents/sandbox/session/manager.py +++ b/src/agents/sandbox/session/manager.py @@ -4,6 +4,7 @@ import logging from collections.abc import Sequence +from ...logger import log_tool_action_error from ..errors import OpName from .events import EventPayloadPolicy, SandboxSessionEvent, SandboxSessionFinishEvent from .sinks import ChainedSink, EventSink @@ -104,8 +105,8 @@ async def _run() -> None: if sink.mode == "sync": try: await _run() - except Exception: - self._handle_sink_error(sink, event) + except Exception as exc: + self._handle_sink_error(sink, event, exc) elif sink.mode == "async": if sink.on_error == "raise": await _run() @@ -114,8 +115,8 @@ async def _run() -> None: async def _task() -> None: try: await _run() - except Exception: - self._handle_sink_error(sink, event) + except Exception as exc: + self._handle_sink_error(sink, event, exc) task = asyncio.create_task(_task()) # Track background deliveries so the task is kept alive and can be discarded once done. @@ -126,8 +127,8 @@ async def _task() -> None: async def _task() -> None: try: await _run() - except Exception: - self._handle_sink_error(sink, event, force_no_raise=True) + except Exception as exc: + self._handle_sink_error(sink, event, exc, force_no_raise=True) task = asyncio.create_task(_task()) # Same bookkeeping as async mode, but failures are always swallowed after logging. @@ -146,16 +147,21 @@ async def _deliver_chained(self, sink: EventSink, event: SandboxSessionEvent) -> """ try: await sink.handle(event) - except Exception: + except Exception as exc: force_no_raise = sink.mode == "best_effort" - self._handle_sink_error(sink, event, force_no_raise=force_no_raise) + self._handle_sink_error(sink, event, exc, force_no_raise=force_no_raise) def _handle_sink_error( - self, sink: EventSink, event: SandboxSessionEvent, *, force_no_raise: bool = False + self, + sink: EventSink, + event: SandboxSessionEvent, + exc: Exception, + *, + force_no_raise: bool = False, ) -> None: if force_no_raise or sink.on_error in ("log", "ignore"): if sink.on_error == "log": - logger.exception("sandbox event sink failed (ignored): %s", type(sink).__name__) + log_tool_action_error(logger, "Sandbox event sink failed (ignored)", exc) return raise RuntimeError( "sandbox event sink failed: " diff --git a/src/agents/tool.py b/src/agents/tool.py index af9a6c3c5c..eb6c0a3645 100644 --- a/src/agents/tool.py +++ b/src/agents/tool.py @@ -54,7 +54,7 @@ from .editor import ApplyPatchEditor, ApplyPatchOperation from .exceptions import ModelBehaviorError, ToolTimeoutError, UserError from .function_schema import DocstringStyle, function_schema -from .logger import logger +from .logger import log_tool_action_warning, logger from .run_context import RunContextWrapper from .strict_schema import ensure_strict_json_schema from .tool_context import ToolContext @@ -885,7 +885,7 @@ async def dispose_resolved_computers(*, run_context: RunContextWrapper[Any]) -> if inspect.isawaitable(result): await result except Exception as exc: - logger.warning("Failed to dispose computer for run context: %s", exc) + log_tool_action_warning(logger, "Failed to dispose computer for run context", exc) @dataclass diff --git a/src/agents/tracing/processors.py b/src/agents/tracing/processors.py index 776939a180..6c68a2673f 100644 --- a/src/agents/tracing/processors.py +++ b/src/agents/tracing/processors.py @@ -13,7 +13,12 @@ import httpx -from ..logger import logger +from .. import _debug +from ..logger import ( + log_model_and_tool_action_error, + log_model_and_tool_action_warning, + logger, +) from .processor_interface import TracingExporter, TracingProcessor from .spans import Span from .traces import Trace @@ -24,6 +29,12 @@ class ConsoleSpanExporter(TracingExporter): def export(self, items: list[Trace | Span[Any]]) -> None: for item in items: + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + if isinstance(item, Trace): + print("[Exporter] Export trace. Trace data is redacted.") + else: + print("[Exporter] Export span. Span data is redacted.") + continue if isinstance(item, Trace): print(f"[Exporter] Export trace_id={item.trace_id}, name={item.name}") else: @@ -173,11 +184,17 @@ def _export_with_deadline(self, items: list[Trace | Span[Any]], deadline: float # If the response is a client error (4xx), we won't retry if 400 <= response.status_code < 500: - logger.error( - "[non-fatal] Tracing client error %s: %s", - response.status_code, - response.text, - ) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.error( + "[non-fatal] Tracing client error %s. Response data is redacted.", + response.status_code, + ) + else: + logger.error( + "[non-fatal] Tracing client error %s: %s", + response.status_code, + response.text, + ) break # For 5xx or other unexpected codes, treat it as transient and retry @@ -186,7 +203,9 @@ def _export_with_deadline(self, items: list[Trace | Span[Any]], deadline: float ) except httpx.RequestError as exc: # Network or other I/O error, we'll retry - logger.warning("[non-fatal] Tracing: request failed: %s", exc) + log_model_and_tool_action_warning( + logger, "[non-fatal] Tracing request failed", exc + ) # If we reach here, we need to retry or give up if attempt >= self.max_retries: @@ -690,10 +709,13 @@ def _export_batches(self, force: bool = False, deadline: float | None = None): else: self._exporter.export(items_to_export) except Exception as exc: - logger.error( - "[non-fatal] Tracing: exporter raised %s; dropping batch of %d items", + log_model_and_tool_action_error( + logger, + ( + "[non-fatal] Tracing exporter failed; " + f"dropping batch of {len(items_to_export)} items" + ), exc, - len(items_to_export), ) diff --git a/src/agents/tracing/provider.py b/src/agents/tracing/provider.py index a5f439dceb..b2cd018c08 100644 --- a/src/agents/tracing/provider.py +++ b/src/agents/tracing/provider.py @@ -10,7 +10,8 @@ from inspect import Parameter, signature from typing import Any, cast -from ..logger import logger +from .. import _debug +from ..logger import log_model_and_tool_action_error, logger from .config import TracingConfig from .processor_interface import TracingProcessor from .scope import Scope @@ -107,7 +108,9 @@ def on_trace_start(self, trace: Trace) -> None: try: processor.on_trace_start(trace) except Exception as e: - logger.error("Error in trace processor %s during on_trace_start: %s", processor, e) + log_model_and_tool_action_error( + logger, "Error in trace processor during on_trace_start", e + ) def on_trace_end(self, trace: Trace) -> None: """ @@ -117,7 +120,9 @@ def on_trace_end(self, trace: Trace) -> None: try: processor.on_trace_end(trace) except Exception as e: - logger.error("Error in trace processor %s during on_trace_end: %s", processor, e) + log_model_and_tool_action_error( + logger, "Error in trace processor during on_trace_end", e + ) def on_span_start(self, span: Span[Any]) -> None: """ @@ -127,7 +132,9 @@ def on_span_start(self, span: Span[Any]) -> None: try: processor.on_span_start(span) except Exception as e: - logger.error("Error in trace processor %s during on_span_start: %s", processor, e) + log_model_and_tool_action_error( + logger, "Error in trace processor during on_span_start", e + ) def on_span_end(self, span: Span[Any]) -> None: """ @@ -137,7 +144,9 @@ def on_span_end(self, span: Span[Any]) -> None: try: processor.on_span_end(span) except Exception as e: - logger.error("Error in trace processor %s during on_span_end: %s", processor, e) + log_model_and_tool_action_error( + logger, "Error in trace processor during on_span_end", e + ) def shutdown(self, timeout: float | None = None) -> None: """ @@ -158,7 +167,7 @@ def shutdown(self, timeout: float | None = None) -> None: else: processor.shutdown() except Exception as e: - logger.error("Error shutting down trace processor %s: %s", processor, e) + log_model_and_tool_action_error(logger, "Error shutting down trace processor", e) def force_flush(self): """ @@ -168,7 +177,7 @@ def force_flush(self): try: processor.force_flush() except Exception as e: - logger.error("Error flushing trace processor %s: %s", processor, e) + log_model_and_tool_action_error(logger, "Error flushing trace processor", e) class TraceProvider(ABC): @@ -337,12 +346,18 @@ def create_trace( """ self._refresh_disabled_flag() if self._disabled or disabled: - logger.debug("Tracing is disabled. Not creating trace %s", name) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Tracing is disabled. Not creating trace") + else: + logger.debug("Tracing is disabled. Not creating trace %s", name) return NoOpTrace() trace_id = trace_id or self.gen_trace_id() - logger.debug("Creating trace %s with id %s", name, trace_id) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Creating trace with id %s", trace_id) + else: + logger.debug("Creating trace %s with id %s", name, trace_id) return TraceImpl( name=name, @@ -367,7 +382,10 @@ def create_span( tracing_api_key: str | None = None trace_metadata: dict[str, Any] | None = None if self._disabled or disabled: - logger.debug("Tracing is disabled. Not creating span %s", span_data) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Tracing is disabled. Not creating span") + else: + logger.debug("Tracing is disabled. Not creating span %s", span_data) return NoOpSpan(span_data) if _is_noop_id(span_id): logger.debug("Span id is no-op, returning NoOpSpan") @@ -383,9 +401,14 @@ def create_span( ) return NoOpSpan(span_data) elif _is_noop_trace(current_trace) or _is_noop_span(current_span): - logger.debug( - "Parent %s or %s is no-op, returning NoOpSpan", current_span, current_trace - ) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Current trace parent is no-op, returning NoOpSpan") + else: + logger.debug( + "Parent %s or %s is no-op, returning NoOpSpan", + current_span, + current_trace, + ) return NoOpSpan(span_data) parent_id = current_span.span_id if current_span else None @@ -396,7 +419,10 @@ def create_span( elif isinstance(parent, Trace): if _is_noop_trace(parent): - logger.debug("Parent %s is no-op, returning NoOpSpan", parent) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Parent trace is no-op, returning NoOpSpan") + else: + logger.debug("Parent %s is no-op, returning NoOpSpan", parent) return NoOpSpan(span_data) trace_id = parent.trace_id parent_id = None @@ -405,14 +431,20 @@ def create_span( trace_metadata = getattr(parent, "metadata", None) elif isinstance(parent, Span): if _is_noop_span(parent): - logger.debug("Parent %s is no-op, returning NoOpSpan", parent) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Parent span is no-op, returning NoOpSpan") + else: + logger.debug("Parent %s is no-op, returning NoOpSpan", parent) return NoOpSpan(span_data) parent_id = parent.span_id trace_id = parent.trace_id tracing_api_key = parent.tracing_api_key trace_metadata = parent.trace_metadata - logger.debug("Creating span %s with id %s", span_data, span_id) + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.debug("Creating span with id %s", span_id) + else: + logger.debug("Creating span %s with id %s", span_data, span_id) return SpanImpl( trace_id=trace_id, @@ -433,7 +465,7 @@ def force_flush(self) -> None: try: self._multi_processor.force_flush() except Exception as e: - logger.error("Error flushing trace provider: %s", e) + log_model_and_tool_action_error(logger, "Error flushing trace provider", e) def shutdown(self, timeout: float | None = None) -> None: self._refresh_disabled_flag() @@ -444,4 +476,4 @@ def shutdown(self, timeout: float | None = None) -> None: _safe_debug("Shutting down trace provider") self._multi_processor.shutdown(timeout=timeout) except Exception as e: - logger.error("Error shutting down trace provider: %s", e) + log_model_and_tool_action_error(logger, "Error shutting down trace provider", e) diff --git a/src/agents/util/_error_tracing.py b/src/agents/util/_error_tracing.py index 0bd2d99e90..7f714482a5 100644 --- a/src/agents/util/_error_tracing.py +++ b/src/agents/util/_error_tracing.py @@ -1,5 +1,6 @@ from typing import Any +from .. import _debug from ..logger import logger from ..tracing import Span, SpanError, get_current_span @@ -24,5 +25,7 @@ def attach_error_to_current_span(error: SpanError) -> None: span = get_current_span() if span: attach_error_to_span(span, error) + elif _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + logger.warning("No active span; trace error was not attached") else: logger.warning("No span to add error %s to", error) diff --git a/src/agents/voice/pipeline.py b/src/agents/voice/pipeline.py index 21220c9921..83d1703d54 100644 --- a/src/agents/voice/pipeline.py +++ b/src/agents/voice/pipeline.py @@ -3,9 +3,10 @@ import asyncio from typing import Any +from .. import _debug from .._config_coercion import coerce_dataclass_config from ..exceptions import UserError -from ..logger import logger +from ..logger import log_model_and_tool_action_error, logger from ..tracing import TraceCtxManager from .input import AudioInput, StreamedAudioInput from .model import STTModel, TTSModel @@ -109,7 +110,7 @@ async def stream_events(): await output._turn_done() await output._done() except Exception as e: - logger.error("Error processing single turn: %s", e) + log_model_and_tool_action_error(logger, "Error processing single voice turn", e) await output._add_error(e) raise e @@ -135,7 +136,10 @@ async def process_turns(): async for intro_text in self.workflow.on_start(): await output._add_text(intro_text) except Exception as e: - logger.warning("on_start() failed: %s", e) + if _debug.DONT_LOG_MODEL_DATA: + logger.warning("Voice workflow on_start failed") + else: + logger.warning("Voice workflow on_start failed: %s", e, exc_info=e) transcription_session = await self._get_stt_model().create_session( audio_input, @@ -150,7 +154,7 @@ async def process_turns(): await output._add_text(text_event) await output._turn_done() except Exception as e: - logger.error("Error processing turns: %s", e) + log_model_and_tool_action_error(logger, "Error processing voice turns", e) await output._add_error(e) raise e finally: diff --git a/src/agents/voice/result.py b/src/agents/voice/result.py index 2f4b24433b..709c829779 100644 --- a/src/agents/voice/result.py +++ b/src/agents/voice/result.py @@ -7,7 +7,7 @@ from typing import Any from ..exceptions import UserError -from ..logger import logger +from ..logger import log_model_action_error, log_model_and_tool_action_error, logger from ..tracing import Span, SpeechGroupSpanData, speech_group_span, speech_span from ..tracing.util import time_iso from ..util._error_tracing import get_trace_error @@ -192,7 +192,7 @@ async def _stream_audio( }, } ) - logger.error("Error streaming audio: %s", e) + log_model_action_error(logger, "Error streaming voice audio", e) # Signal completion for whole session because of error await local_queue.put(VoiceStreamEventLifecycle(event="session_ended")) @@ -308,7 +308,9 @@ async def stream(self) -> AsyncIterator[VoiceStreamEvent]: break if isinstance(event, VoiceStreamEventError): self._stored_exception = event.error - logger.error("Error processing output: %s", event.error) + log_model_and_tool_action_error( + logger, "Error processing voice output", event.error + ) break if event is None: break diff --git a/tests/extensions/experiemental/codex/test_codex_tool.py b/tests/extensions/experiemental/codex/test_codex_tool.py index 9bf650816b..3aeb7db73d 100644 --- a/tests/extensions/experiemental/codex/test_codex_tool.py +++ b/tests/extensions/experiemental/codex/test_codex_tool.py @@ -14,6 +14,7 @@ from openai.types.responses import ResponseFunctionToolCall from pydantic import BaseModel, ConfigDict +import agents._debug as _debug from agents import Agent, function_tool from agents.exceptions import ModelBehaviorError, UserError from agents.extensions.experimental.codex import ( @@ -1802,7 +1803,13 @@ async def test_replaced_codex_tool_preserves_codex_collision_markers() -> None: @pytest.mark.asyncio -async def test_codex_tool_consume_events_with_on_stream_error() -> None: +async def test_codex_tool_consume_events_with_on_stream_error( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + secret = "SECRET_CODEX_STREAM_PAYLOAD" events = [ { "type": "item.started", @@ -1865,7 +1872,7 @@ async def event_stream(): def on_stream(payload: CodexToolStreamEvent) -> None: callbacks.append(payload.event.type) if payload.event.type == "item.started": - raise RuntimeError("boom") + raise RuntimeError(secret) context = ToolContext( context=None, @@ -1874,20 +1881,22 @@ def on_stream(payload: CodexToolStreamEvent) -> None: tool_arguments="{}", ) - with trace("codex-test"): - response, usage, thread_id = await codex_tool_module._consume_events( - event_stream(), - {"inputs": [{"type": "text", "text": "hello"}]}, - context, - SimpleNamespace(id="thread-1"), - on_stream, - 64, - ) + with caplog.at_level("ERROR", logger="openai.agents"): + with trace("codex-test"): + response, usage, thread_id = await codex_tool_module._consume_events( + event_stream(), + {"inputs": [{"type": "text", "text": "hello"}]}, + context, + SimpleNamespace(id="thread-1"), + on_stream, + 64, + ) assert response == "done" assert usage == Usage(input_tokens=1, cached_input_tokens=0, output_tokens=1) assert thread_id == "thread-1" assert "item.started" in callbacks + assert secret not in caplog.text @pytest.mark.asyncio diff --git a/tests/extensions/memory/test_advanced_sqlite_session.py b/tests/extensions/memory/test_advanced_sqlite_session.py index 2472a8bf12..9a9b50480e 100644 --- a/tests/extensions/memory/test_advanced_sqlite_session.py +++ b/tests/extensions/memory/test_advanced_sqlite_session.py @@ -7,13 +7,14 @@ import threading from pathlib import Path from typing import Any, cast -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest pytest.importorskip("sqlalchemy") # Skip tests if SQLAlchemy is not installed from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails +import agents._debug as _debug from agents import Agent, Runner, TResponseInputItem, function_tool from agents.extensions.memory import AdvancedSQLiteSession from agents.result import RunResult @@ -154,6 +155,32 @@ async def test_advanced_session_basic_functionality(agent: Agent): session.close() +@pytest.mark.parametrize("redacted", [True, False]) +async def test_create_branch_logging_respects_model_data_policy(monkeypatch, redacted: bool): + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + mock_logger = Mock() + session = AdvancedSQLiteSession( + session_id="advanced_branch_logging", + create_tables=True, + logger=mock_logger, + ) + secret = "SECRET_BRANCH_TURN_CONTENT" + + try: + await session.add_items( + [ + {"role": "user", "content": secret}, + {"role": "assistant", "content": "response"}, + ] + ) + await session.create_branch_from_turn(1, "branch") + + logged = str(mock_logger.debug.call_args) + assert (secret not in logged) is redacted + finally: + session.close() + + async def test_advanced_session_respects_custom_table_names(): """AdvancedSQLiteSession should consistently use configured table names.""" session = AdvancedSQLiteSession( diff --git a/tests/extensions/sandbox/test_cloudflare.py b/tests/extensions/sandbox/test_cloudflare.py index 84e5ae9f39..20d67ac6ab 100644 --- a/tests/extensions/sandbox/test_cloudflare.py +++ b/tests/extensions/sandbox/test_cloudflare.py @@ -1535,12 +1535,17 @@ def delete(self, url: str, **kwargs: Any) -> Any: @pytest.mark.asyncio -async def test_cloudflare_shutdown_logs_delete_response_details( +@pytest.mark.parametrize("redacted", [True, False]) +async def test_cloudflare_shutdown_logs_respect_tool_data_policy( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, + redacted: bool, ) -> None: - """Verify that DELETE response bodies are kept when shutdown cleanup fails.""" + """Verify that DELETE response bodies follow the tool-data logging policy.""" import logging + monkeypatch.setattr("agents._debug.DONT_LOG_TOOL_DATA", redacted) + sess = _make_session( fake_http=_FakeHttp( { @@ -1558,8 +1563,9 @@ async def test_cloudflare_shutdown_logs_delete_response_details( with caplog.at_level(logging.DEBUG, logger="agents.extensions.sandbox.cloudflare.sandbox"): await sess._shutdown_backend() - assert any( + has_detail = any( "DELETE /sandbox failed: HTTP 502: pool_error: pool error: Failed to start container" in r.message for r in caplog.records ) + assert has_detail is not redacted diff --git a/tests/extensions/sandbox/test_e2b.py b/tests/extensions/sandbox/test_e2b.py index d6a4ac33fa..1f5ab3d2fb 100644 --- a/tests/extensions/sandbox/test_e2b.py +++ b/tests/extensions/sandbox/test_e2b.py @@ -2189,11 +2189,15 @@ async def test_e2b_stop_terminates_live_pty_sessions() -> None: @pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) async def test_e2b_shutdown_logs_pause_failure_and_falls_back_to_kill( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, + redacted: bool, ) -> None: + monkeypatch.setattr("agents._debug.DONT_LOG_TOOL_DATA", redacted) sandbox = _FakeE2BSandbox() - sandbox.pause_error = RuntimeError("pause failed") + sandbox.pause_error = RuntimeError("SECRET_E2B_PAUSE_FAILURE") state = E2BSandboxSessionState( session_id=uuid.uuid4(), manifest=Manifest(root="/workspace"), @@ -2211,12 +2215,23 @@ async def test_e2b_shutdown_logs_pause_failure_and_falls_back_to_kill( assert sandbox.pause_calls == 1 assert sandbox.kill_calls == 1 assert "Failed to pause E2B sandbox on shutdown; falling back to kill." in caplog.text + assert ("SECRET_E2B_PAUSE_FAILURE" not in caplog.text) is redacted + record = caplog.records[-1] + assert ("sandbox_id" in record.__dict__) is not redacted + assert ("pause_on_exit" in record.__dict__) is not redacted + if not redacted: + assert record.__dict__["sandbox_id"] == sandbox.sandbox_id + assert record.__dict__["pause_on_exit"] is True @pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) async def test_e2b_shutdown_logs_kill_failure_after_pause_fallback( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, + redacted: bool, ) -> None: + monkeypatch.setattr("agents._debug.DONT_LOG_TOOL_DATA", redacted) sandbox = _FakeE2BSandbox() sandbox.pause_error = RuntimeError("pause failed") sandbox.kill_error = RuntimeError("kill failed") @@ -2237,10 +2252,22 @@ async def test_e2b_shutdown_logs_kill_failure_after_pause_fallback( assert sandbox.pause_calls == 1 assert sandbox.kill_calls == 1 assert "Failed to kill E2B sandbox after pause fallback failure." in caplog.text + record = caplog.records[-1] + assert ("sandbox_id" in record.__dict__) is not redacted + assert ("pause_on_exit" in record.__dict__) is not redacted + if not redacted: + assert record.__dict__["sandbox_id"] == sandbox.sandbox_id + assert record.__dict__["pause_on_exit"] is True @pytest.mark.asyncio -async def test_e2b_shutdown_logs_direct_kill_failure(caplog: pytest.LogCaptureFixture) -> None: +@pytest.mark.parametrize("redacted", [True, False]) +async def test_e2b_shutdown_logs_direct_kill_failure( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + redacted: bool, +) -> None: + monkeypatch.setattr("agents._debug.DONT_LOG_TOOL_DATA", redacted) sandbox = _FakeE2BSandbox() sandbox.kill_error = RuntimeError("kill failed") state = E2BSandboxSessionState( @@ -2260,6 +2287,12 @@ async def test_e2b_shutdown_logs_direct_kill_failure(caplog: pytest.LogCaptureFi assert sandbox.pause_calls == 0 assert sandbox.kill_calls == 1 assert "Failed to kill E2B sandbox on shutdown." in caplog.text + record = caplog.records[-1] + assert ("sandbox_id" in record.__dict__) is not redacted + assert ("pause_on_exit" in record.__dict__) is not redacted + if not redacted: + assert record.__dict__["sandbox_id"] == sandbox.sandbox_id + assert record.__dict__["pause_on_exit"] is False @pytest.mark.asyncio diff --git a/tests/mcp/test_mcp_server_manager.py b/tests/mcp/test_mcp_server_manager.py index ccf026e7ee..f1f769eb6f 100644 --- a/tests/mcp/test_mcp_server_manager.py +++ b/tests/mcp/test_mcp_server_manager.py @@ -1,4 +1,5 @@ import asyncio +import logging from typing import Any, cast import pytest @@ -12,7 +13,9 @@ Tool as MCPTool, ) +from agents import _debug from agents.mcp import MCPServer, MCPServerManager +from agents.mcp._logging import get_mcp_server_log_name from agents.run_context import RunContextWrapper @@ -121,6 +124,21 @@ async def read_resource(self, uri: str) -> ReadResourceResult: return ReadResourceResult(contents=[]) +class SensitiveNamedServer(FlakyServer): + def __init__(self, name: str) -> None: + super().__init__(failures=1) + self._name = name + self.name_reads = 0 + + @property + def name(self) -> str: + self.name_reads += 1 + return self._name + + async def connect(self) -> None: + raise RuntimeError("SECRET_MCP_CONNECT_ERROR") + + class CleanupAwareServer(MCPServer): def __init__(self) -> None: super().__init__() @@ -172,6 +190,84 @@ async def read_resource(self, uri: str) -> ReadResourceResult: return ReadResourceResult(contents=[]) +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("ordinary-server", "ordinary-server"), + ( + "sse: https://user:password@example.test/events?token=secret#fragment", + "sse: https://example.test/events", + ), + ( + "streamable_http: https://example.test/mcp?token=secret", + "streamable_http: https://example.test/mcp", + ), + ( + "streamable_http: https://user:password@example.test:8443/mcp?token=secret", + "streamable_http: https://example.test:8443/mcp", + ), + ("streamable_http: https://[::1]:8000/mcp", "streamable_http: https://[::1]:8000/mcp"), + ( + "streamable-http: https://example.test/mcp#secret", + "streamable-http: https://example.test/mcp", + ), + ( + "streamable_http: https://user:password@[invalid/mcp?token=secret", + "streamable_http: ", + ), + ( + "streamable_http: https://user:password/mcp?token=secret", + "streamable_http: ", + ), + ("https://user:password@example.test/mcp?token=secret", "https://example.test/mcp"), + ("https://user:password@[invalid/mcp?token=secret", ""), + ("stdio: python server.py?token=secret", "stdio: python server.py?token=secret"), + ], +) +def test_get_mcp_server_log_name(name: str, expected: str) -> None: + assert get_mcp_server_log_name(name) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) +@pytest.mark.parametrize( + ("server_name", "diagnostic_sentinel", "always_hidden"), + [ + ( + "streamable_http: https://SECRET_CREDENTIAL@example.test/" + "SECRET_MCP_PATH?token=SECRET_MCP_QUERY#SECRET_MCP_FRAGMENT", + "SECRET_MCP_PATH", + ("SECRET_CREDENTIAL", "SECRET_MCP_QUERY", "SECRET_MCP_FRAGMENT"), + ), + ( + "SECRET_CUSTOM_MCP_SERVER_NAME", + "SECRET_CUSTOM_MCP_SERVER_NAME", + (), + ), + ], +) +async def test_manager_sanitizes_url_derived_server_names_in_failure_logs( + monkeypatch, + caplog, + redacted: bool, + server_name: str, + diagnostic_sentinel: str, + always_hidden: tuple[str, ...], +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + server = SensitiveNamedServer(server_name) + manager = MCPServerManager([server]) + + with caplog.at_level(logging.ERROR, logger="openai.agents"): + await manager.connect_all() + + assert (diagnostic_sentinel not in caplog.text) is redacted + assert server.name_reads == (0 if redacted else 1) + for sentinel in always_hidden: + assert sentinel not in caplog.text + assert ("SECRET_MCP_CONNECT_ERROR" not in caplog.text) is redacted + + class CancelledServer(MCPServer): def __init__(self) -> None: super().__init__() diff --git a/tests/mcp/test_mcp_util.py b/tests/mcp/test_mcp_util.py index 7ddf9f9068..e1a06f17eb 100644 --- a/tests/mcp/test_mcp_util.py +++ b/tests/mcp/test_mcp_util.py @@ -677,7 +677,7 @@ async def test_mcp_invoke_bad_json_errors(caplog: pytest.LogCaptureFixture): with pytest.raises(ModelBehaviorError): await MCPUtil.invoke_mcp_tool(server, tool, ctx, "not_json") - assert "Invalid JSON input for tool test_tool_1" in caplog.text + assert "Invalid JSON input for MCP tool" in caplog.text @pytest.mark.asyncio @@ -807,7 +807,7 @@ async def test_mcp_invocation_crash_causes_error(caplog: pytest.LogCaptureFixtur with pytest.raises(AgentsException): await MCPUtil.invoke_mcp_tool(server, tool, ctx, "") - assert "Error invoking MCP tool test_tool_1" in caplog.text + assert "Error invoking MCP tool" in caplog.text class SecretCrashingFakeMCPServer(FakeMCPServer): @@ -837,15 +837,17 @@ async def test_mcp_invocation_crash_redacts_error_when_dont_log_tool_data( caplog.set_level(logging.DEBUG) monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) - server = SecretCrashingFakeMCPServer() - server.add_tool("test_tool_1", {}) + server = SecretCrashingFakeMCPServer(server_name="SECRET_CUSTOM_MCP_SERVER") + server.add_tool("SECRET_MCP_TOOL_NAME", {}) ctx = RunContextWrapper(context=None) - tool = MCPTool(name="test_tool_1", inputSchema={}) + tool = MCPTool(name="SECRET_MCP_TOOL_NAME", inputSchema={}) with pytest.raises(AgentsException): await MCPUtil.invoke_mcp_tool(server, tool, ctx, "") - assert "Error invoking MCP tool test_tool_1" in caplog.text + assert "Error invoking MCP tool" in caplog.text + assert "SECRET_CUSTOM_MCP_SERVER" not in caplog.text + assert "SECRET_MCP_TOOL_NAME" not in caplog.text assert "SECRET_CRASH_123" not in caplog.text @@ -856,7 +858,12 @@ async def test_mcp_invocation_crash_includes_error_when_tool_logging_enabled( caplog.set_level(logging.DEBUG) monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) - server = SecretCrashingFakeMCPServer() + server = SecretCrashingFakeMCPServer( + server_name=( + "streamable_http: https://SECRET_CREDENTIAL@example.test/" + "SECRET_MCP_PATH?token=SECRET_MCP_QUERY" + ) + ) server.add_tool("test_tool_1", {}) ctx = RunContextWrapper(context=None) tool = MCPTool(name="test_tool_1", inputSchema={}) @@ -865,6 +872,9 @@ async def test_mcp_invocation_crash_includes_error_when_tool_logging_enabled( await MCPUtil.invoke_mcp_tool(server, tool, ctx, "") assert "SECRET_CRASH_123" in caplog.text + assert "SECRET_MCP_PATH" in caplog.text + assert "SECRET_CREDENTIAL" not in caplog.text + assert "SECRET_MCP_QUERY" not in caplog.text @pytest.mark.asyncio @@ -874,15 +884,17 @@ async def test_mcp_tool_returned_error_redacts_message_when_dont_log_tool_data( caplog.set_level(logging.DEBUG) monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) - server = McpErrorFakeMCPServer() - server.add_tool("test_tool_1", {}) + server = McpErrorFakeMCPServer(server_name="SECRET_CUSTOM_MCP_SERVER") + server.add_tool("SECRET_MCP_TOOL_NAME", {}) ctx = RunContextWrapper(context=None) - tool = MCPTool(name="test_tool_1", inputSchema={}) + tool = MCPTool(name="SECRET_MCP_TOOL_NAME", inputSchema={}) with pytest.raises(McpError): await MCPUtil.invoke_mcp_tool(server, tool, ctx, "") - assert "MCP tool test_tool_1 on server" in caplog.text + assert "MCP tool returned an error" in caplog.text + assert "SECRET_CUSTOM_MCP_SERVER" not in caplog.text + assert "SECRET_MCP_TOOL_NAME" not in caplog.text assert "SECRET_MCP_123" not in caplog.text diff --git a/tests/mcp/test_tool_filtering.py b/tests/mcp/test_tool_filtering.py index 0127df806c..3da58756a6 100644 --- a/tests/mcp/test_tool_filtering.py +++ b/tests/mcp/test_tool_filtering.py @@ -9,6 +9,7 @@ import pytest from mcp import Tool as MCPTool +import agents._debug as _debug from agents import Agent from agents.mcp import ToolFilterContext, create_static_tool_filter from agents.run_context import RunContextWrapper @@ -181,6 +182,38 @@ def error_prone_filter(context: ToolFilterContext, tool: MCPTool) -> bool: assert {t.name for t in tools} == {"good_tool", "another_good_tool"} +@pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) +async def test_dynamic_filter_error_logging_preserves_identity_only_in_diagnostic_mode( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + server = FakeMCPServer( + server_name=( + "streamable_http: https://SECRET_CREDENTIAL@example.test/" + "SECRET_SERVER_PATH?token=SECRET_QUERY" + ) + ) + server.add_tool("SECRET_TOOL_NAME", {}) + + def failing_filter(context: ToolFilterContext, tool: MCPTool) -> bool: + raise ValueError("SECRET_FILTER_ERROR") + + server.tool_filter = failing_filter + + with caplog.at_level("ERROR", logger="openai.agents"): + tools = await server.list_tools(create_test_context(), create_test_agent()) + + assert tools == [] + assert "SECRET_CREDENTIAL" not in caplog.text + assert "SECRET_QUERY" not in caplog.text + assert ("SECRET_TOOL_NAME" in caplog.text) is not redacted + assert ("SECRET_SERVER_PATH" in caplog.text) is not redacted + assert ("SECRET_FILTER_ERROR" in caplog.text) is not redacted + + # === Integration Tests === diff --git a/tests/memory/test_openai_responses_compaction_session.py b/tests/memory/test_openai_responses_compaction_session.py index fe893cf88a..16b7ce8718 100644 --- a/tests/memory/test_openai_responses_compaction_session.py +++ b/tests/memory/test_openai_responses_compaction_session.py @@ -8,6 +8,7 @@ import pytest +import agents._debug as _debug from agents import Agent, Runner from agents.items import TResponseInputItem from agents.memory import ( @@ -706,9 +707,12 @@ async def clear_session(self) -> None: assert failing_session.add_calls == 1 @pytest.mark.asyncio + @pytest.mark.parametrize("redacted", [True, False]) async def test_run_compaction_reraises_replacement_error_when_restore_fails( - self, caplog: pytest.LogCaptureFixture + self, monkeypatch, caplog: pytest.LogCaptureFixture, redacted: bool ) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) history: list[TResponseInputItem] = [ cast(TResponseInputItem, {"type": "message", "role": "user", "content": "original"}), ] @@ -730,7 +734,7 @@ async def add_items(self, items: list[TResponseInputItem]) -> None: if self.add_calls == 1: await super().add_items(items[:1]) raise RuntimeError("replacement failed") - raise RuntimeError("restore failed") + raise RuntimeError("SECRET_COMPACTION_RESTORE_FAILURE") async def clear_session(self) -> None: self.clear_calls += 1 @@ -758,6 +762,7 @@ async def clear_session(self) -> None: assert ( "Failed to restore session history after compaction replacement failed." in caplog.text ) + assert ("SECRET_COMPACTION_RESTORE_FAILURE" not in caplog.text) is redacted assert failing_session.clear_calls == 2 assert failing_session.add_calls == 2 diff --git a/tests/realtime/test_agent.py b/tests/realtime/test_agent.py index 7ac5cbe359..bc2a4c408c 100644 --- a/tests/realtime/test_agent.py +++ b/tests/realtime/test_agent.py @@ -1,6 +1,7 @@ from __future__ import annotations from typing import Any +from unittest.mock import patch import pytest @@ -29,6 +30,29 @@ def _instructions(ctx, agt) -> str: assert instructions == "Dynamic" +@pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) +async def test_mutated_invalid_instructions_respect_model_data_policy( + monkeypatch, redacted: bool +) -> None: + class SensitiveInstructions: + def __str__(self) -> str: + return "SECRET_REALTIME_INSTRUCTIONS" + + __repr__ = __str__ + + agent = RealtimeAgent(name="test") + agent.instructions = SensitiveInstructions() # type: ignore[assignment] + monkeypatch.setattr("agents.realtime.agent._debug.DONT_LOG_MODEL_DATA", redacted) + + with patch("agents.realtime.agent.logger") as mock_logger: + prompt = await agent.get_system_prompt(RunContextWrapper(context=None)) + + assert prompt is None + logged = str(mock_logger.error.call_args) + assert ("SECRET_REALTIME_INSTRUCTIONS" not in logged) is redacted + + def test_post_init_rejects_invalid_field_types() -> None: with pytest.raises(TypeError, match="RealtimeAgent name must be a string"): RealtimeAgent(name=1) # type: ignore[arg-type] diff --git a/tests/realtime/test_session.py b/tests/realtime/test_session.py index 3211f2358d..b3b2da6da8 100644 --- a/tests/realtime/test_session.py +++ b/tests/realtime/test_session.py @@ -8,6 +8,7 @@ import pytest from pydantic import BaseModel, ConfigDict +import agents._debug as _debug from agents.exceptions import ToolTimeoutError, UserError from agents.guardrail import GuardrailFunctionOutput, OutputGuardrail from agents.handoffs import Handoff @@ -576,6 +577,7 @@ class _FakeAudio: @pytest.mark.asyncio async def test_item_updated_merge_exception_path_logs_error(monkeypatch): + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) model = _DummyModel() agent = RealtimeAgent(name="agent") session = RealtimeSession(model, agent, None) @@ -594,8 +596,7 @@ async def test_item_updated_merge_exception_path_logs_error(monkeypatch): with patch("agents.realtime.session.logger") as mock_logger: await session.on_event(RealtimeModelItemUpdatedEvent(item=incoming)) - # error branch should be hit - assert mock_logger.error.called + mock_logger.error.assert_called_once_with("%s", "Error merging transcripts", stacklevel=3) @pytest.mark.asyncio @@ -3119,6 +3120,31 @@ async def test_reject_pending_tool_call_uses_run_level_formatter( for ev in events ) + @pytest.mark.asyncio + async def test_rejection_formatter_error_is_redacted( + self, monkeypatch, mock_model, mock_agent, mock_function_tool + ): + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + + def fail_formatter(_args): + raise ValueError("SECRET_REALTIME_TOOL_FORMATTER") + + session = RealtimeSession( + mock_model, + mock_agent, + None, + run_config={"tool_error_formatter": fail_formatter}, + ) + + with patch("agents.realtime.session.logger") as mock_logger: + message = await session._resolve_approval_rejection_message( + tool=mock_function_tool, + call_id="call_reject_error", + ) + + assert message + mock_logger.error.assert_called_once_with("%s", "Tool error formatter failed", stacklevel=3) + @pytest.mark.asyncio async def test_reject_pending_tool_call_prefers_explicit_message( self, mock_model, mock_agent, mock_function_tool diff --git a/tests/test_agent_as_tool.py b/tests/test_agent_as_tool.py index c5cc123034..ab5efcce57 100644 --- a/tests/test_agent_as_tool.py +++ b/tests/test_agent_as_tool.py @@ -13,6 +13,7 @@ from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall from pydantic import BaseModel, Field +import agents._debug as _debug from agents import ( Agent, AgentBase, @@ -2406,8 +2407,12 @@ async def _invoke_tool() -> Any: @pytest.mark.asyncio async def test_agent_as_tool_streaming_handler_exception_does_not_fail_call( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) agent = Agent(name="handler_error_agent") + secret = "SECRET_AGENT_STREAM_PAYLOAD" class DummyStreamingResult: def __init__(self) -> None: @@ -2427,7 +2432,7 @@ async def stream_events(self): ) def bad_handler(event: AgentToolStreamEvent) -> None: - raise RuntimeError("boom") + raise RuntimeError(secret) tool_call = ResponseFunctionToolCall( id="call_bad", @@ -2450,9 +2455,11 @@ def bad_handler(event: AgentToolStreamEvent) -> None: tool_call=tool_call, ) - output = await tool.on_invoke_tool(tool_context, '{"input": "go"}') + with caplog.at_level("ERROR", logger="openai.agents"): + output = await tool.on_invoke_tool(tool_context, '{"input": "go"}') assert output == "ok" + assert secret not in caplog.text @pytest.mark.asyncio diff --git a/tests/test_agent_runner.py b/tests/test_agent_runner.py index 12160d886e..ab6485045d 100644 --- a/tests/test_agent_runner.py +++ b/tests/test_agent_runner.py @@ -17,6 +17,7 @@ from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary from typing_extensions import TypedDict +import agents._debug as _debug from agents import ( Agent, GuardrailFunctionOutput, @@ -2779,6 +2780,45 @@ async def test_rewind_handles_id_stripped_sessions() -> None: assert session.saved_items == [] +@pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) +async def test_rewind_debug_logging_respects_model_and_tool_policies( + monkeypatch, redacted: bool +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + secret = "SECRET_REWIND_SESSION_CONTENT" + session = IdStrippingSession() + item = cast( + TResponseInputItem, + {"id": "message-1", "type": "message", "role": "user", "content": secret}, + ) + await session.add_items([item]) + + with patch("agents.run_internal.session_persistence.logger") as mock_logger: + await rewind_session_items(session, [item]) + + logged = str(mock_logger.debug.call_args_list) + assert (secret not in logged) is redacted + + +@pytest.mark.asyncio +async def test_rewind_failure_uses_placeholder_free_shared_logger_message() -> None: + class FailingTailSession(SimpleListSession): + async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]: + raise RuntimeError("tail failure") + + item = cast(TResponseInputItem, {"type": "message", "role": "user", "content": "hi"}) + session = FailingTailSession(history=[item]) + + with patch( + "agents.run_internal.session_persistence.log_model_and_tool_action_warning" + ) as mock_warning: + await rewind_session_items(session, [item]) + + assert mock_warning.call_args.args[1] == "Failed to rewind session item" + + @pytest.mark.asyncio async def test_rewind_skips_mismatched_tail_suffix() -> None: target = cast(TResponseInputItem, {"type": "message", "role": "user", "content": "target"}) diff --git a/tests/test_computer_tool_lifecycle.py b/tests/test_computer_tool_lifecycle.py index 860dcef9b7..bbb0e04baf 100644 --- a/tests/test_computer_tool_lifecycle.py +++ b/tests/test_computer_tool_lifecycle.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import logging from typing import Any, cast from unittest.mock import AsyncMock @@ -11,6 +12,7 @@ ResponseComputerToolCall, ) +import agents._debug as _debug from agents import ( Agent, ComputerProvider, @@ -68,6 +70,32 @@ def drag(self, path: list[tuple[int, int]]) -> None: return None +@pytest.mark.asyncio +@pytest.mark.parametrize("redacted", [True, False]) +async def test_dispose_computer_failure_respects_tool_data_policy( + monkeypatch, caplog, redacted: bool +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + + async def dispose(**_kwargs: Any) -> None: + raise RuntimeError("SECRET_COMPUTER_DISPOSE_FAILURE") + + tool = ComputerTool( + computer=ComputerProvider[FakeComputer]( + create=AsyncMock(return_value=FakeComputer()), + dispose=dispose, + ) + ) + ctx = RunContextWrapper(context=None) + await resolve_computer(tool=tool, run_context=ctx) + + with caplog.at_level(logging.WARNING, logger="openai.agents"): + await dispose_resolved_computers(run_context=ctx) + + assert "Failed to dispose computer for run context" in caplog.text + assert ("SECRET_COMPUTER_DISPOSE_FAILURE" not in caplog.text) is redacted + + def _make_message(text: str) -> ResponseOutputMessage: return ResponseOutputMessage( id="msg-1", diff --git a/tests/test_error_logging_redaction.py b/tests/test_error_logging_redaction.py index 5942e6cd5b..6e0aa77f87 100644 --- a/tests/test_error_logging_redaction.py +++ b/tests/test_error_logging_redaction.py @@ -9,8 +9,10 @@ from __future__ import annotations import logging +from pathlib import Path from unittest.mock import patch +import httpx import pytest from openai import AsyncOpenAI @@ -23,6 +25,17 @@ RunContextWrapper, trace, ) +from agents.logger import ( + log_model_action_debug, + log_model_action_error, + log_model_action_warning, + log_model_and_tool_action_debug, + log_model_and_tool_action_error, + log_model_and_tool_action_warning, + log_tool_action_debug, + log_tool_action_error as log_shared_tool_action_error, + log_tool_action_warning, +) from agents.run_internal.tool_execution import ( log_tool_action_error, resolve_approval_rejection_message, @@ -31,8 +44,44 @@ _SECRET = "super secret prompt content" +class _RecordingHandler(logging.Handler): + def __init__(self) -> None: + super().__init__() + self.records: list[logging.LogRecord] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.records.append(record) + + +class _HostileException(Exception): + def __str__(self) -> str: + raise AssertionError("redacted logging inspected __str__") + + def __repr__(self) -> str: + raise AssertionError("redacted logging inspected __repr__") + + def __getattribute__(self, name: str): + if name in {"__class__", "__traceback__"}: + raise AssertionError(f"redacted logging inspected {name}") + return super().__getattribute__(name) + + +def _emit_shared_error_for_location(test_logger, helper) -> None: + helper(test_logger, "Fixed operational message", ValueError("failure")) + + +def _emit_tool_execution_error_for_location() -> None: + log_tool_action_error("Fixed operational message", ValueError("failure")) + + def _responses_model() -> OpenAIResponsesModel: - return OpenAIResponsesModel(model="test-model", openai_client=AsyncOpenAI(api_key="test")) + return OpenAIResponsesModel( + model="test-model", + openai_client=AsyncOpenAI( + api_key="test", + http_client=httpx.AsyncClient(trust_env=False), + ), + ) @pytest.mark.allow_call_model_methods @@ -63,7 +112,8 @@ async def raise_fetch(*args, **kwargs): mock_logger.error.assert_called_once() logged = str(mock_logger.error.call_args) assert _SECRET not in logged - assert "ValueError" in logged + assert "ValueError" not in logged + assert "Error getting response" in logged @pytest.mark.allow_call_model_methods @@ -124,7 +174,8 @@ async def raise_fetch(*args, **kwargs): mock_logger.error.assert_called_once() logged = str(mock_logger.error.call_args) assert _SECRET not in logged - assert "ValueError" in logged + assert "ValueError" not in logged + assert "Error streaming response" in logged def test_log_tool_action_error_redacts_by_default(monkeypatch) -> None: @@ -134,13 +185,150 @@ def test_log_tool_action_error_redacts_by_default(monkeypatch) -> None: log_tool_action_error("Shell executor failed", ValueError("rm -rf /secret/path")) mock_logger.error.assert_called_once() - logged = str(mock_logger.error.call_args) - assert "/secret/path" not in logged - assert "ValueError" in logged + assert mock_logger.error.call_args.args == ("%s", "Shell executor failed") # No traceback either, since it can embed the same sensitive data. assert mock_logger.error.call_args.kwargs.get("exc_info") in (None, False) +@pytest.mark.parametrize( + ("helper", "model_flag", "tool_flag"), + [ + (log_model_action_error, True, False), + (log_model_action_debug, True, False), + (log_model_action_warning, True, False), + (log_tool_action_debug, False, True), + (log_shared_tool_action_error, False, True), + (log_tool_action_warning, False, True), + (log_model_and_tool_action_error, True, False), + (log_model_and_tool_action_error, False, True), + (log_model_and_tool_action_debug, True, False), + (log_model_and_tool_action_warning, False, True), + ], +) +def test_shared_error_helpers_do_not_inspect_or_attach_redacted_exceptions( + monkeypatch, + helper, + model_flag: bool, + tool_flag: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_flag) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_flag) + test_logger = logging.Logger("sensitive-logging-redacted") + handler = _RecordingHandler() + test_logger.addHandler(handler) + hostile = _HostileException() + + helper(test_logger, "Fixed operational message", hostile) + + assert len(handler.records) == 1 + record = handler.records[0] + assert record.msg == "%s" + assert record.args == ("Fixed operational message",) + assert record.exc_info is None + assert record.exc_text is None + assert hostile not in record.__dict__.values() + assert logging.Formatter().format(record) == "Fixed operational message" + + +def test_shared_error_helper_preserves_diagnostics_when_enabled(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + test_logger = logging.Logger("sensitive-logging-diagnostic") + handler = _RecordingHandler() + test_logger.addHandler(handler) + error = ValueError(_SECRET) + + log_shared_tool_action_error(test_logger, "Tool failed", error) + + record = handler.records[0] + assert isinstance(record.args, tuple) + assert error in record.args + assert record.exc_info is not None + assert record.exc_info[1] is error + assert _SECRET in logging.Formatter().format(record) + + +@pytest.mark.parametrize("redacted", [True, False]) +def test_shared_error_helper_conditionally_attaches_diagnostic_extra( + monkeypatch, redacted: bool +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + test_logger = logging.Logger("sensitive-logging-diagnostic-extra") + handler = _RecordingHandler() + test_logger.addHandler(handler) + extra_calls = 0 + + def diagnostic_extra() -> dict[str, object]: + nonlocal extra_calls + extra_calls += 1 + return {"sandbox_id": _SECRET} + + log_tool_action_warning( + test_logger, + "Tool failed", + ValueError("failure"), + diagnostic_extra=diagnostic_extra, + ) + + record = handler.records[0] + assert extra_calls == (0 if redacted else 1) + assert ("sandbox_id" in record.__dict__) is not redacted + if not redacted: + assert record.__dict__["sandbox_id"] == _SECRET + + +@pytest.mark.parametrize( + "helper", + [log_shared_tool_action_error, log_tool_action_warning], +) +def test_shared_error_helpers_preserve_direct_caller_location(monkeypatch, helper) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + test_logger = logging.Logger("sensitive-logging-location") + handler = _RecordingHandler() + test_logger.addHandler(handler) + + _emit_shared_error_for_location(test_logger, helper) + + record = handler.records[0] + assert Path(record.pathname).resolve() == Path(__file__).resolve() + assert record.funcName == "_emit_shared_error_for_location" + + +def test_tool_execution_error_helper_preserves_external_caller_location(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", True) + test_logger = logging.Logger("sensitive-logging-wrapped-location") + handler = _RecordingHandler() + test_logger.addHandler(handler) + + with patch("agents.run_internal.tool_execution.logger", test_logger): + _emit_tool_execution_error_for_location() + + record = handler.records[0] + assert Path(record.pathname).resolve() == Path(__file__).resolve() + assert record.funcName == "_emit_tool_execution_error_for_location" + + +def test_shared_error_helper_drops_exception_chains_and_notes(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) + test_logger = logging.Logger("sensitive-logging-chain") + handler = _RecordingHandler() + test_logger.addHandler(handler) + cause = ValueError(f"{_SECRET} cause") + error = RuntimeError(f"{_SECRET} outer") + error.__cause__ = cause + if hasattr(error, "add_note"): + error.add_note(f"{_SECRET} note") + else: + error.__notes__ = [f"{_SECRET} note"] + + log_model_action_error(test_logger, "Model failed", error) + + record = handler.records[0] + assert record.exc_info is None + assert record.exc_text is None + assert error not in record.__dict__.values() + assert _SECRET not in logging.Formatter().format(record) + + def test_log_tool_action_error_logs_full_when_tool_data_enabled(monkeypatch) -> None: monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) @@ -150,7 +338,7 @@ def test_log_tool_action_error_logs_full_when_tool_data_enabled(monkeypatch) -> mock_logger.error.assert_called_once() logged = str(mock_logger.error.call_args) assert "/secret/path" in logged - assert mock_logger.error.call_args.kwargs.get("exc_info") is True + assert isinstance(mock_logger.error.call_args.kwargs.get("exc_info"), ValueError) @pytest.mark.asyncio diff --git a/tests/test_trace_processor.py b/tests/test_trace_processor.py index c0d8898599..ae34114a30 100644 --- a/tests/test_trace_processor.py +++ b/tests/test_trace_processor.py @@ -12,9 +12,10 @@ import httpx import pytest +import agents._debug as _debug from agents.tracing import flush_traces, get_trace_provider from agents.tracing.processor_interface import TracingExporter, TracingProcessor -from agents.tracing.processors import BackendSpanExporter, BatchTraceProcessor +from agents.tracing.processors import BackendSpanExporter, BatchTraceProcessor, ConsoleSpanExporter from agents.tracing.provider import DefaultTraceProvider, TraceProvider from agents.tracing.span_data import AgentSpanData from agents.tracing.spans import Span, SpanImpl @@ -45,6 +46,20 @@ def get_trace(processor: TracingProcessor) -> TraceImpl: ) +@pytest.mark.parametrize("redacted", [True, False]) +def test_console_span_exporter_respects_data_policy(monkeypatch, capsys, redacted: bool) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + span = get_span(mock_processor()) + span.span_data.name = "SECRET_CONSOLE_SPAN" + + ConsoleSpanExporter().export([span]) + + output = capsys.readouterr().out + assert ("SECRET_CONSOLE_SPAN" not in output) is redacted + assert "Export span" in output + + @pytest.fixture def mocked_exporter(): exporter = MagicMock() @@ -438,17 +453,31 @@ def test_backend_span_exporter_2xx_success(mock_client): @patch("httpx.Client") -def test_backend_span_exporter_4xx_client_error(mock_client): - mock_response = MagicMock() - mock_response.status_code = 400 - mock_response.text = "Bad Request" +@pytest.mark.parametrize("redacted", [True, False]) +def test_backend_span_exporter_4xx_client_error(mock_client, monkeypatch, caplog, redacted: bool): + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + + class Response: + status_code = 400 + text_reads = 0 + + @property + def text(self) -> str: + self.text_reads += 1 + return "SECRET_TRACE_RESPONSE_BODY" + + mock_response = Response() mock_client.return_value.post.return_value = mock_response exporter = BackendSpanExporter(api_key="test_key") - exporter.export([get_span(mock_processor())]) + with caplog.at_level(logging.ERROR, logger="openai.agents"): + exporter.export([get_span(mock_processor())]) # 4xx should not be retried mock_client.return_value.post.assert_called_once() + assert ("SECRET_TRACE_RESPONSE_BODY" not in caplog.text) is redacted + assert mock_response.text_reads == (0 if redacted else 1) exporter.close() diff --git a/tests/tracing/test_tracing_env_disable.py b/tests/tracing/test_tracing_env_disable.py index aa2fd93f20..e49b11ea2f 100644 --- a/tests/tracing/test_tracing_env_disable.py +++ b/tests/tracing/test_tracing_env_disable.py @@ -1,5 +1,8 @@ import logging +import pytest + +import agents._debug as _debug from agents.tracing.provider import DefaultTraceProvider from agents.tracing.scope import Scope from agents.tracing.span_data import AgentSpanData @@ -17,6 +20,25 @@ def test_env_read_on_first_use(monkeypatch): assert isinstance(trace, NoOpTrace) +@pytest.mark.parametrize("redacted", [True, False]) +def test_disabled_span_logging_respects_data_policy(monkeypatch, caplog, redacted: bool): + class SensitiveAgentSpanData(AgentSpanData): + def __repr__(self) -> str: + return "SECRET_SPAN_NAME" + + monkeypatch.setenv("OPENAI_AGENTS_DISABLE_TRACING", "1") + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + provider = DefaultTraceProvider() + + with caplog.at_level(logging.DEBUG, logger="openai.agents"): + span = provider.create_span(SensitiveAgentSpanData(name="agent")) + + assert isinstance(span, NoOpSpan) + assert ("SECRET_SPAN_NAME" not in caplog.text) is redacted + assert "Tracing is disabled. Not creating span" in caplog.text + + def test_env_cached_after_first_use(monkeypatch): """Env flag is cached after the first trace and later env changes do not flip it.""" monkeypatch.setenv("OPENAI_AGENTS_DISABLE_TRACING", "0") diff --git a/tests/voice/test_pipeline.py b/tests/voice/test_pipeline.py index d6f97bb0eb..4faae9d85e 100644 --- a/tests/voice/test_pipeline.py +++ b/tests/voice/test_pipeline.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import logging from dataclasses import dataclass, field from typing import Any @@ -8,6 +9,7 @@ import numpy.typing as npt import pytest +import agents._debug as _debug from agents import trace from tests.testing_processor import fetch_events, fetch_span_errors @@ -437,6 +439,16 @@ async def run(self, _: str): yield "out_1" +class _FailingWorkflow(FakeWorkflow): + def __init__(self, error: BaseException): + super().__init__() + self.error = error + + async def run(self, _: str): + raise self.error + yield "" # pragma: no cover + + class _OnStartYieldThenFailWorkflow(FakeWorkflow): async def on_start(self): yield "intro" @@ -493,3 +505,71 @@ async def test_voicepipeline_multi_turn_on_start_exception_does_not_abort() -> N assert events[-1] == "session_ended" assert "error" not in events + + +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted", "redacted"), + [ + (True, False, True), + (False, True, True), + (False, False, False), + ], +) +async def test_voice_workflow_errors_apply_model_and_tool_logging_policies( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + streamed: bool, + model_redacted: bool, + tool_redacted: bool, + redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + error = RuntimeError("SECRET_VOICE_TOOL_PAYLOAD") + pipeline = VoicePipeline( + workflow=_FailingWorkflow(error), + stt_model=FakeSTT(["first"]), + tts_model=FakeTTS(), + ) + audio_input = ( + await FakeStreamedAudioInput.get(count=1) + if streamed + else AudioInput(buffer=np.zeros(2, dtype=np.int16)) + ) + caplog.set_level(logging.ERROR, logger="openai.agents") + + result = await pipeline.run(audio_input) + with pytest.raises(RuntimeError, match="SECRET_VOICE_TOOL_PAYLOAD"): + await extract_events(result) + assert result.text_generation_task is not None + assert result.text_generation_task.exception() is error + + expected_pipeline_message = ( + "Error processing voice turns" if streamed else "Error processing single voice turn" + ) + records = [ + record + for record in caplog.records + if record.name == "openai.agents" + and isinstance(record.args, tuple) + and record.args + and record.args[0] in {expected_pipeline_message, "Error processing voice output"} + ] + assert len(records) == 2 + for record in records: + if redacted: + assert record.msg == "%s" + assert record.args in { + (expected_pipeline_message,), + ("Error processing voice output",), + } + assert record.exc_info is None + assert "SECRET_VOICE_TOOL_PAYLOAD" not in logging.Formatter().format(record) + else: + assert record.msg == "%s: %s" + assert isinstance(record.args, tuple) + assert error in record.args + assert record.exc_info is not None + assert record.exc_info[1] is error From 1aa2fcfeb30bc706245b1e3242f9b3a2ce958a8f Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Fri, 24 Jul 2026 20:59:51 +0900 Subject: [PATCH 02/12] fix --- .../scripts/_inventory.py | 38 ++++++++++ .../scripts/test_inventory.py | 30 ++++++++ src/agents/voice/pipeline.py | 14 ++-- tests/voice/test_pipeline.py | 75 ++++++++++++++++++- 4 files changed, 150 insertions(+), 7 deletions(-) diff --git a/.agents/skills/sensitive-logging-audit/scripts/_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/_inventory.py index 12a39f6d75..0feb80f13d 100644 --- a/.agents/skills/sensitive-logging-audit/scripts/_inventory.py +++ b/.agents/skills/sensitive-logging-audit/scripts/_inventory.py @@ -177,6 +177,30 @@ def target_value_pairs(target: ast.AST, value: ast.AST) -> list[tuple[str, ast.A return [] +def pattern_bound_names(pattern: ast.pattern) -> set[str]: + if isinstance(pattern, ast.MatchAs): + names = pattern_bound_names(pattern.pattern) if pattern.pattern is not None else set() + if pattern.name: + names.add(pattern.name) + return names + if isinstance(pattern, ast.MatchStar): + return {pattern.name} if pattern.name else set() + if isinstance(pattern, ast.MatchMapping): + names = {name for child in pattern.patterns for name in pattern_bound_names(child)} + if pattern.rest: + names.add(pattern.rest) + return names + if isinstance(pattern, ast.MatchClass): + return { + name + for child in [*pattern.patterns, *pattern.kwd_patterns] + for name in pattern_bound_names(child) + } + if isinstance(pattern, ast.MatchSequence | ast.MatchOr): + return {name for child in pattern.patterns for name in pattern_bound_names(child)} + return set() + + def iter_definitions( tree: ast.AST, ) -> Iterable[tuple[ast.AST, list[tuple[str, ast.AST | None]]]]: @@ -205,6 +229,10 @@ def iter_definitions( yield node, [(node.name, None)] elif isinstance(node, ast.Delete): yield node, [(key, None) for target in node.targets for key in target_keys(target)] + elif isinstance(node, ast.match_case): + names = pattern_bound_names(node.pattern) + if names: + yield node.pattern, [(name, None) for name in sorted(names)] def bound_names(node: ast.AST) -> set[str]: @@ -282,6 +310,8 @@ def _collect_bindings(self, tree: ast.Module) -> None: self.bindings[scope].update(bound_names(item.optional_vars)) elif isinstance(node, ast.ExceptHandler) and node.name: self.bindings[scope].add(node.name) + elif isinstance(node, ast.match_case): + self.bindings[scope].update(pattern_bound_names(node.pattern)) elif isinstance(node, ast.Import): for alias in node.names: self.bindings[scope].add(alias.asname or alias.name.split(".", 1)[0]) @@ -371,6 +401,14 @@ def _definition_precedes_in_same_block(self, definition: ast.AST, use: ast.AST) child = self.parents[child] if child in definition.body: return True + elif isinstance(definition, ast.pattern): + case = self.parents.get(definition) + if isinstance(case, ast.match_case): + child = use + while child in self.parents and self.parents[child] is not case: + child = self.parents[child] + if child is case.guard or child in case.body: + return True parent = self.parents.get(definition) if parent is None: diff --git a/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py index d1ab14efc2..f1b07ec27f 100644 --- a/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py +++ b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py @@ -453,6 +453,36 @@ def report(values, manager, secret): self.assertEqual([item.policy for item in findings], ["none", "none"]) + def test_match_pattern_captures_kill_policy_alias_provenance(self) -> None: + findings = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +def direct_capture(value, secret): + flag = _debug.DONT_LOG_TOOL_DATA + match value: + case flag: + if not flag: + logger.error("direct capture: %s", secret) + +def nested_capture(value, secret): + flag = _debug.DONT_LOG_TOOL_DATA + match value: + case {"enabled": flag}: + if not flag: + logger.error("nested capture: %s", secret) + +def guarded_capture(value, secret): + flag = _debug.DONT_LOG_TOOL_DATA + match value: + case flag if not flag: + logger.error("guard capture: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["none", "none", "none"]) + def test_combines_exact_model_and_tool_guards(self) -> None: findings = inventory_source( """ diff --git a/src/agents/voice/pipeline.py b/src/agents/voice/pipeline.py index 83d1703d54..da2bdceafd 100644 --- a/src/agents/voice/pipeline.py +++ b/src/agents/voice/pipeline.py @@ -3,10 +3,13 @@ import asyncio from typing import Any -from .. import _debug from .._config_coercion import coerce_dataclass_config from ..exceptions import UserError -from ..logger import log_model_and_tool_action_error, logger +from ..logger import ( + log_model_and_tool_action_error, + log_model_and_tool_action_warning, + logger, +) from ..tracing import TraceCtxManager from .input import AudioInput, StreamedAudioInput from .model import STTModel, TTSModel @@ -136,10 +139,9 @@ async def process_turns(): async for intro_text in self.workflow.on_start(): await output._add_text(intro_text) except Exception as e: - if _debug.DONT_LOG_MODEL_DATA: - logger.warning("Voice workflow on_start failed") - else: - logger.warning("Voice workflow on_start failed: %s", e, exc_info=e) + log_model_and_tool_action_warning( + logger, "Voice workflow on_start failed", e + ) transcription_session = await self._get_stt_model().create_session( audio_input, diff --git a/tests/voice/test_pipeline.py b/tests/voice/test_pipeline.py index 4faae9d85e..45db259929 100644 --- a/tests/voice/test_pipeline.py +++ b/tests/voice/test_pipeline.py @@ -450,9 +450,13 @@ async def run(self, _: str): class _OnStartYieldThenFailWorkflow(FakeWorkflow): + def __init__(self, outputs: list[list[str]], error: BaseException | None = None): + super().__init__(outputs) + self.error = error or RuntimeError("boom") + async def on_start(self): yield "intro" - raise RuntimeError("boom") + raise self.error @pytest.mark.asyncio @@ -507,6 +511,75 @@ async def test_voicepipeline_multi_turn_on_start_exception_does_not_abort() -> N assert "error" not in events +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted", "redacted"), + [ + (True, False, True), + (False, True, True), + (False, False, False), + ], +) +async def test_voice_on_start_errors_apply_model_and_tool_logging_policies( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + model_redacted: bool, + tool_redacted: bool, + redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + cause = ValueError("SECRET_VOICE_ON_START_TOOL_PAYLOAD") + error = RuntimeError("Voice startup failed") + error.__cause__ = cause + pipeline = VoicePipeline( + workflow=_OnStartYieldThenFailWorkflow([["out_1"]], error), + stt_model=FakeSTT(["first"]), + tts_model=FakeTTS(), + ) + streamed_audio_input = await FakeStreamedAudioInput.get(count=1) + caplog.set_level(logging.WARNING, logger="openai.agents") + + result = await pipeline.run(streamed_audio_input) + events, _ = await extract_events(result) + + assert events[-1] == "session_ended" + records = [ + record + for record in caplog.records + if record.name == "openai.agents" + and ( + record.msg + in { + "Voice workflow on_start failed", + "Voice workflow on_start failed: %s", + } + or ( + isinstance(record.args, tuple) + and record.args + and record.args[0] == "Voice workflow on_start failed" + ) + ) + ] + assert len(records) == 1 + record = records[0] + if redacted: + assert record.msg == "%s" + assert record.args == ("Voice workflow on_start failed",) + assert record.exc_info is None + assert record.exc_text is None + assert error not in record.__dict__.values() + assert cause not in record.__dict__.values() + assert "SECRET_VOICE_ON_START_TOOL_PAYLOAD" not in logging.Formatter().format(record) + else: + assert record.msg == "%s: %s" + assert isinstance(record.args, tuple) + assert error in record.args + assert record.exc_info is not None + assert record.exc_info[1] is error + assert "SECRET_VOICE_ON_START_TOOL_PAYLOAD" in logging.Formatter().format(record) + + @pytest.mark.asyncio @pytest.mark.parametrize("streamed", [False, True]) @pytest.mark.parametrize( From 9cff0a570d1b04d9d340f06d0de5d9bb2f9fb90e Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Fri, 24 Jul 2026 21:34:24 +0900 Subject: [PATCH 03/12] fix --- .../scripts/_inventory.py | 47 +++++++--- .../scripts/test_inventory.py | 87 +++++++++++++++++++ 2 files changed, 123 insertions(+), 11 deletions(-) diff --git a/.agents/skills/sensitive-logging-audit/scripts/_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/_inventory.py index 0feb80f13d..27342c41ce 100644 --- a/.agents/skills/sensitive-logging-audit/scripts/_inventory.py +++ b/.agents/skills/sensitive-logging-audit/scripts/_inventory.py @@ -41,6 +41,7 @@ } SENSITIVE_HELPER_METHODS = {name.rsplit(".", 1)[-1] for name in KNOWN_HELPERS} RAW_MODULE_METHODS = { + "builtins": {"print"}, "pprint": {"pprint"}, "traceback": {"print_exc", "print_exception"}, "warnings": {"warn", "warn_explicit"}, @@ -62,6 +63,8 @@ "catch_value", ) +DefinitionValue = ast.AST | frozenset[str] | None + def _is_sdk_logger_module_or_parent(module: str) -> bool: return any( @@ -250,11 +253,12 @@ def __init__(self, tree: ast.Module, file_path: str): self.node_scopes: dict[ast.AST, ast.AST] = {} self.scope_parents: dict[ast.AST, ast.AST | None] = {tree: None} self.parents = make_parent_map(tree) - self.definitions = list(iter_definitions(tree)) - self.definition_values: dict[ast.AST, dict[str, list[tuple[ast.AST, ast.AST | None]]]] = ( + self.definitions: list[tuple[ast.AST, list[tuple[str, DefinitionValue]]]] = [ + (definition, list(targets)) for definition, targets in iter_definitions(tree) + ] + self.definition_values: dict[ast.AST, dict[str, list[tuple[ast.AST, DefinitionValue]]]] = ( defaultdict(lambda: defaultdict(list)) ) - self.import_policy_values: dict[ast.AST, dict[str, set[str]]] = defaultdict(dict) self._policy_lookup_stack: set[tuple[int, str]] = set() self.module_name, self.package_name = module_identity(file_path) self._index_scopes(tree) @@ -351,6 +355,11 @@ def _lookup_policy(self, node: ast.AST, key: str) -> set[str]: definitions = self.definition_values[scope].get(key, []) if definitions: if scope is not use_scope: + if all(isinstance(value, frozenset) for _, value in definitions): + _, imported_facts = max( + definitions, key=lambda item: self._node_position(item[0]) + ) + return set(imported_facts) return set() preceding = [ definition @@ -364,6 +373,8 @@ def _lookup_policy(self, node: ast.AST, key: str) -> set[str]: return set() if value is None: return set() + if isinstance(value, frozenset): + return set(value) self._policy_lookup_stack.add(token) try: return { @@ -374,9 +385,6 @@ def _lookup_policy(self, node: ast.AST, key: str) -> set[str]: finally: self._policy_lookup_stack.remove(token) - imported = self.import_policy_values[scope].get(key) - if imported is not None: - return set(imported) if root in self.bindings[scope]: return set() scope = self.scope_parents[scope] @@ -433,7 +441,14 @@ def _collect_imports(self, tree: ast.Module) -> None: elif isinstance(node, ast.Import): for alias in node.names: bound = alias.asname or alias.name.split(".", 1)[0] - if alias.name in {"logging", "pprint", "sys", "traceback", "warnings"}: + if alias.name in { + "builtins", + "logging", + "pprint", + "sys", + "traceback", + "warnings", + }: self.add(scope, bound, {f"module:{alias.name}"}) elif alias.name == POLICY_MODULE: self.add( @@ -459,7 +474,7 @@ def _collect_imports(self, tree: ast.Module) -> None: facts.add("factory:logger") elif alias.name in LOG_METHODS: facts.add(f"method:logger:{alias.name}") - elif alias.name == "Logger": + elif alias.name in {"Logger", "LoggerAdapter"}: facts.add("type:logger") if module == "functools" and alias.name == "partial": facts.add("factory:partial") @@ -473,7 +488,9 @@ def _collect_imports(self, tree: ast.Module) -> None: policy = POLICY_NAMES[alias.name] policy_facts = {f"policy:{policy}", f"policy-exact:{policy}"} facts.update(policy_facts) - self.import_policy_values[scope][bound] = policy_facts + policy_value = frozenset(policy_facts) + self.definitions.append((node, [(bound, policy_value)])) + self.definition_values[scope][bound].append((node, policy_value)) if full_name in KNOWN_HELPERS: facts.add(f"helper:{KNOWN_HELPERS[full_name]}") if module in RAW_MODULE_METHODS and alias.name in RAW_MODULE_METHODS[module]: @@ -496,7 +513,7 @@ def infer(self, node: ast.AST) -> set[str]: if fact == "module:logging": if node.attr == "getLogger": result.add("factory:logger") - elif node.attr == "Logger": + elif node.attr in {"Logger", "LoggerAdapter"}: result.add("type:logger") elif node.attr in LOG_METHODS: result.add(f"method:logger:{node.attr}") @@ -584,7 +601,10 @@ def _resolve_definitions(self, tree: ast.Module) -> None: for definition, targets in self.definitions: scope = self._scope_for(definition) for key, value in targets: - inferred = self.infer(value) if value is not None else set() + if isinstance(value, frozenset): + inferred = value + else: + inferred = self.infer(value) if value is not None else set() changed |= self.add(scope, key, inferred) def _seed_special_attributes(self, tree: ast.Module) -> None: @@ -789,6 +809,11 @@ def call_site_context(node: ast.AST, parents: Mapping[ast.AST, ast.AST], source: index for index, argument in enumerate(current.args) if argument is child ) parts.append(f"callback:{normalize_node(current.func, source)}:{argument_index}") + elif isinstance(child, ast.keyword) and child in current.keywords: + keyword_name = child.arg if child.arg is not None else "**" + parts.append( + f"callback:{normalize_node(current.func, source)}:keyword:{keyword_name}" + ) child = current current = parents.get(current) return ">".join(reversed(parts)) or "" diff --git a/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py index f1b07ec27f..f38fd89011 100644 --- a/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py +++ b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py @@ -147,6 +147,46 @@ def test_recognizes_directly_constructed_logger_aliases(self) -> None: [("logger", "error"), ("logger", "warning")], ) + def test_recognizes_logger_adapter_sources(self) -> None: + findings = inventory_source( + """ +import logging +from logging import LoggerAdapter + +base = logging.getLogger("base") +direct = LoggerAdapter(base, {}) +direct_emit = direct.error +direct_emit(secret) +qualified = logging.LoggerAdapter(base, {}) +qualified_emit = qualified.warning +qualified_emit(secret) +""" + ) + + self.assertEqual( + [(item.kind, item.method) for item in findings], + [("logger", "error"), ("logger", "warning")], + ) + + def test_inventories_print_through_builtins_imports(self) -> None: + findings = inventory_source( + """ +import builtins +from builtins import print as emit + +builtins.print(secret) +emit(secret) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [ + ("raw-output", "builtins.print", "dynamic-message"), + ("raw-output", "builtins.print", "dynamic-message"), + ], + ) + def test_resolves_helpers_through_qualified_sdk_logger_imports(self) -> None: findings = inventory_source( """ @@ -453,6 +493,36 @@ def report(values, manager, secret): self.assertEqual([item.policy for item in findings], ["none", "none"]) + def test_policy_imports_are_ordered_definitions(self) -> None: + findings = inventory_source( + """ +from agents.logger import logger + +def report(flag, secret): + if not flag: + logger.error("before import: %s", secret) + from agents._debug import DONT_LOG_TOOL_DATA as flag + if not flag: + logger.error("after import: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["none", "tool-guard"]) + + def test_module_policy_import_still_reaches_nested_function(self) -> None: + findings = inventory_source( + """ +from agents._debug import DONT_LOG_TOOL_DATA as flag +from agents.logger import logger + +def report(secret): + if not flag: + logger.error("guarded: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["tool-guard"]) + def test_match_pattern_captures_kill_policy_alias_provenance(self) -> None: findings = inventory_source( """ @@ -701,6 +771,23 @@ def test_discovers_keyword_callback_sinks(self) -> None: [("logger-callback", "error"), ("raw-output-callback", "stderr.write")], ) + def test_keyword_callback_fingerprints_include_registration_context(self) -> None: + findings = inventory_source( + """ +from agents.logger import logger + +register_model(on_error=logger.error) +register_operational(on_error=logger.error) +register_model(on_warning=logger.error) +""" + ) + + self.assertEqual(len(findings), 3) + self.assertEqual(len({item.group_fingerprint for item in findings}), 3) + self.assertIn("callback:register_model:keyword:on_error", findings[0].context) + self.assertIn("callback:register_operational:keyword:on_error", findings[1].context) + self.assertIn("callback:register_model:keyword:on_warning", findings[2].context) + def test_source_collection_ignores_hidden_paths_only_below_the_scan_root(self) -> None: with TemporaryDirectory() as temp_dir: root = Path(temp_dir) / ".cache" / "worktree" From 520adb2081410a17cdd85f0ea174c14922b1aa84 Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Fri, 24 Jul 2026 22:28:42 +0900 Subject: [PATCH 04/12] fix --- .../references/redaction-validation.md | 1 + .../scripts/_inventory.py | 47 ++++---- .../scripts/test_inventory.py | 107 ++++++++++++++++++ 3 files changed, 132 insertions(+), 23 deletions(-) diff --git a/.agents/skills/sensitive-logging-audit/references/redaction-validation.md b/.agents/skills/sensitive-logging-audit/references/redaction-validation.md index 7a658b0f8e..a9f5dd0ee9 100644 --- a/.agents/skills/sensitive-logging-audit/references/redaction-validation.md +++ b/.agents/skills/sensitive-logging-audit/references/redaction-validation.md @@ -76,3 +76,4 @@ uv run python .agents/skills/sensitive-logging-audit/scripts/inventory_logging.p ``` Allowed dispositions are `model`, `tool`, `model+tool`, `operational`, `intentional-output`, and `uncertain`. +Both `evidence` and `action` must be non-empty JSON strings; structured values do not count as review rationale. diff --git a/.agents/skills/sensitive-logging-audit/scripts/_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/_inventory.py index 27342c41ce..5094f016d6 100644 --- a/.agents/skills/sensitive-logging-audit/scripts/_inventory.py +++ b/.agents/skills/sensitive-logging-audit/scripts/_inventory.py @@ -42,7 +42,7 @@ SENSITIVE_HELPER_METHODS = {name.rsplit(".", 1)[-1] for name in KNOWN_HELPERS} RAW_MODULE_METHODS = { "builtins": {"print"}, - "pprint": {"pprint"}, + "pprint": {"pp", "pprint"}, "traceback": {"print_exc", "print_exception"}, "warnings": {"warn", "warn_explicit"}, } @@ -66,6 +66,10 @@ DefinitionValue = ast.AST | frozenset[str] | None +def _helper_fact(full_name: str) -> str: + return f"helper:{KNOWN_HELPERS[full_name]}:{full_name.rsplit('.', 1)[-1]}" + + def _is_sdk_logger_module_or_parent(module: str) -> bool: return any( candidate == module or candidate.startswith(f"{module}.") @@ -208,7 +212,9 @@ def iter_definitions( tree: ast.AST, ) -> Iterable[tuple[ast.AST, list[tuple[str, ast.AST | None]]]]: for node in ast.walk(tree): - if isinstance(node, ast.Assign): + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): + yield node, [(node.name, None)] + elif isinstance(node, ast.Assign): for target in node.targets: yield node, target_value_pairs(target, node.value) elif isinstance(node, ast.AnnAssign) and node.value is not None: @@ -437,7 +443,7 @@ def _collect_imports(self, tree: ast.Module) -> None: if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): full_name = f"{self.module_name}.{node.name}" if scope is tree and full_name in KNOWN_HELPERS: - self.add(scope, node.name, {f"helper:{KNOWN_HELPERS[full_name]}"}) + self.add(scope, node.name, {_helper_fact(full_name)}) elif isinstance(node, ast.Import): for alias in node.names: bound = alias.asname or alias.name.split(".", 1)[0] @@ -492,7 +498,7 @@ def _collect_imports(self, tree: ast.Module) -> None: self.definitions.append((node, [(bound, policy_value)])) self.definition_values[scope][bound].append((node, policy_value)) if full_name in KNOWN_HELPERS: - facts.add(f"helper:{KNOWN_HELPERS[full_name]}") + facts.add(_helper_fact(full_name)) if module in RAW_MODULE_METHODS and alias.name in RAW_MODULE_METHODS[module]: facts.add(f"method:raw:{module}.{alias.name}") if module == "sys" and alias.name in {"stdout", "stderr"}: @@ -530,7 +536,7 @@ def infer(self, node: ast.AST) -> set[str]: if module in SDK_LOGGER_MODULES: helper_policy = KNOWN_HELPERS.get(qualified_name) if helper_policy is not None: - result.add(f"helper:{helper_policy}") + result.add(_helper_fact(qualified_name)) if node.attr == "logger": result.add("logger") if module in RAW_MODULE_METHODS and node.attr in RAW_MODULE_METHODS[module]: @@ -549,11 +555,11 @@ def infer(self, node: ast.AST) -> set[str]: if "factory:logger" in callee_facts or "type:logger" in callee_facts: result.add("logger") if self._is_partial(callee_facts) and node.args: - method_facts = { - fact for fact in self.infer(node.args[0]) if fact.startswith("method:") - } + callable_facts = self.infer(node.args[0]) + method_facts = {fact for fact in callable_facts if fact.startswith("method:")} result.update(method_facts) - if len(node.args) == 1 and not node.keywords: + inherited_shape = any(fact.startswith("partial-shape:") for fact in callable_facts) + if len(node.args) == 1 and not node.keywords and not inherited_shape: result.add("partial-shape:empty") else: bound_call = ast.Call( @@ -563,7 +569,8 @@ def infer(self, node: ast.AST) -> set[str]: ) for method_fact in method_facts: _, _, method = method_fact.split(":", 2) - result.add(f"partial-shape:{call_shape(bound_call, method)}") + shape = call_shape_with_partial(bound_call, method, callable_facts) + result.add(f"partial-shape:{shape}") return result if isinstance(node, ast.BoolOp): for value in node.values: @@ -959,16 +966,13 @@ def record( category, sink_kind, sink_method = _split_sink_fact(fact) if category == "helper": helper_policy = f"{sink_kind}-helper" - helper_method = ( - node.func.id if isinstance(node.func, ast.Name) else node.func.attr - ) record( node, call_text, "sensitive-helper", "confirmed", - helper_method, - call_shape(node, helper_method), + sink_method, + call_shape(node, sink_method), helper_policy, catch_value, ) @@ -1068,12 +1072,7 @@ def record( def _split_sink_fact(fact: str) -> tuple[str, str, str]: if fact.startswith("helper:"): - policy = fact.split(":", 1)[1] - method = ( - "log_model_and_tool_action_error" - if policy == "model+tool" - else f"log_{policy}_action_error" - ) + _, policy, method = fact.split(":", 2) return "helper", policy, method _, sink_kind, sink_method = fact.split(":", 2) return "method", sink_kind, sink_method @@ -1188,9 +1187,11 @@ def validate_review_ledger( continue if review.get("disposition") not in DISPOSITIONS: errors.append(f"Invalid disposition for {group}.") - if not str(review.get("evidence", "")).strip(): + evidence = review.get("evidence") + if not isinstance(evidence, str) or not evidence.strip(): errors.append(f"Missing evidence for {group}.") - if not str(review.get("action", "")).strip(): + action = review.get("action") + if not isinstance(action, str) or not action.strip(): errors.append(f"Missing action for {group}.") if review.get("group_count") != count: actual = review.get("group_count") diff --git a/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py index f38fd89011..4df2528515 100644 --- a/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py +++ b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py @@ -238,6 +238,23 @@ def test_partial_preserves_bound_payload_shape(self) -> None: ], ) + def test_nested_partial_preserves_bound_payload_shape(self) -> None: + findings = inventory_source( + """ +from functools import partial +from agents.logger import logger + +first = partial(logger.error, secret) +second = partial(first) +second() +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [("logger", "error", "dynamic-message")], + ) + def test_inventories_raw_output_and_unknown_receivers(self) -> None: findings = inventory_source( """ @@ -264,6 +281,25 @@ def test_inventories_raw_output_and_unknown_receivers(self) -> None: ], ) + def test_inventories_pprint_pp_aliases(self) -> None: + findings = inventory_source( + """ +import pprint +from pprint import pp as emit + +pprint.pp(secret) +emit(secret) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [ + ("raw-output", "pprint.pp", "dynamic-message"), + ("raw-output", "pprint.pp", "dynamic-message"), + ], + ) + def test_inventories_directly_imported_output_streams(self) -> None: findings = inventory_source( """ @@ -475,6 +511,36 @@ def report(enabled, secret): self.assertEqual([item.policy for item in findings], ["tool-guard", "none"]) + def test_definitions_kill_policy_alias_provenance(self) -> None: + findings = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +def function_definition(secret): + flag = _debug.DONT_LOG_TOOL_DATA + + @lambda function: False + def flag(): + pass + + if not flag: + logger.error("function definition: %s", secret) + +def class_definition(secret): + flag = _debug.DONT_LOG_TOOL_DATA + + @lambda class_: False + class flag: + pass + + if not flag: + logger.error("class definition: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["none", "none"]) + def test_runtime_binders_kill_imported_policy_alias_provenance(self) -> None: findings = inventory_source( """ @@ -687,6 +753,30 @@ def test_recognizes_shared_tool_helpers_at_multiple_levels(self) -> None: ], ) + def test_recognizes_conditional_helper_callees(self) -> None: + findings = inventory_source( + """ +from agents.logger import logger, log_tool_action_warning + +(log_tool_action_warning if enabled else log_tool_action_warning)( + logger, + "Cleanup failed", + error, +) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.policy) for item in findings], + [ + ( + "sensitive-helper", + "log_tool_action_warning", + "tool-helper", + ) + ], + ) + def test_flags_caught_values_and_implicit_exception_payloads(self) -> None: findings = inventory_source( """ @@ -905,6 +995,23 @@ def test_review_ledger_rejects_duplicate_group_entries(self) -> None: self.assertFalse(validation["valid"]) self.assertIn(f"Duplicate review entries for {group}", validation["errors"]) + def test_review_ledger_rejects_non_text_evidence_and_actions(self) -> None: + findings = inventory_source("from agents.logger import logger\nlogger.error('x', secret)\n") + group = findings[0].group_fingerprint + review = { + "group_fingerprint": group, + "group_count": 1, + "disposition": "tool", + "evidence": {}, + "action": [], + } + + validation = validate_review_ledger({"reviews": [review]}, findings) + + self.assertFalse(validation["valid"]) + self.assertIn(f"Missing evidence for {group}.", validation["errors"]) + self.assertIn(f"Missing action for {group}.", validation["errors"]) + def test_summary_separates_unknown_and_raw_candidates(self) -> None: findings = inventory_source( """ From 54cd3cf191c690b84d81cef52ece46acd3fdae88 Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Fri, 24 Jul 2026 22:53:14 +0900 Subject: [PATCH 05/12] fix --- .../scripts/_inventory.py | 105 ++++++++++++++++-- .../scripts/test_inventory.py | 99 +++++++++++++++++ src/agents/tracing/provider.py | 46 ++++++-- tests/test_error_logging_redaction.py | 87 +++++++++++++++ 4 files changed, 320 insertions(+), 17 deletions(-) diff --git a/.agents/skills/sensitive-logging-audit/scripts/_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/_inventory.py index 5094f016d6..e6da3b0b5f 100644 --- a/.agents/skills/sensitive-logging-audit/scripts/_inventory.py +++ b/.agents/skills/sensitive-logging-audit/scripts/_inventory.py @@ -42,6 +42,7 @@ SENSITIVE_HELPER_METHODS = {name.rsplit(".", 1)[-1] for name in KNOWN_HELPERS} RAW_MODULE_METHODS = { "builtins": {"print"}, + "os": {"write"}, "pprint": {"pp", "pprint"}, "traceback": {"print_exc", "print_exception"}, "warnings": {"warn", "warn_explicit"}, @@ -164,6 +165,8 @@ def target_keys(node: ast.AST) -> list[str]: key = expression_key(node) if key: return [key] + if isinstance(node, ast.Starred): + return target_keys(node.value) if isinstance(node, ast.List | ast.Tuple): return [key for element in node.elts for key in target_keys(element)] return [] @@ -173,6 +176,8 @@ def target_value_pairs(target: ast.AST, value: ast.AST) -> list[tuple[str, ast.A key = expression_key(target) if key: return [(key, value)] + if isinstance(target, ast.Starred): + return [(key, None) for key in target_keys(target.value)] if isinstance(target, ast.List | ast.Tuple): if isinstance(value, ast.List | ast.Tuple) and len(target.elts) == len(value.elts): return [ @@ -247,6 +252,8 @@ def iter_definitions( def bound_names(node: ast.AST) -> set[str]: if isinstance(node, ast.Name): return {node.id} + if isinstance(node, ast.Starred): + return bound_names(node.value) if isinstance(node, ast.List | ast.Tuple): return {name for element in node.elts for name in bound_names(element)} return set() @@ -447,9 +454,11 @@ def _collect_imports(self, tree: ast.Module) -> None: elif isinstance(node, ast.Import): for alias in node.names: bound = alias.asname or alias.name.split(".", 1)[0] + self._record_import_definition(node, scope, bound, None) if alias.name in { "builtins", "logging", + "os", "pprint", "sys", "traceback", @@ -475,6 +484,7 @@ def _collect_imports(self, tree: ast.Module) -> None: bound = alias.asname or alias.name full_name = f"{module}.{alias.name}" if module else alias.name facts: set[str] = set() + policy_value: DefinitionValue = None if module == "logging": if alias.name == "getLogger": facts.add("factory:logger") @@ -495,8 +505,7 @@ def _collect_imports(self, tree: ast.Module) -> None: policy_facts = {f"policy:{policy}", f"policy-exact:{policy}"} facts.update(policy_facts) policy_value = frozenset(policy_facts) - self.definitions.append((node, [(bound, policy_value)])) - self.definition_values[scope][bound].append((node, policy_value)) + self._record_import_definition(node, scope, bound, policy_value) if full_name in KNOWN_HELPERS: facts.add(_helper_fact(full_name)) if module in RAW_MODULE_METHODS and alias.name in RAW_MODULE_METHODS[module]: @@ -506,6 +515,16 @@ def _collect_imports(self, tree: ast.Module) -> None: if facts: self.add(scope, bound, facts) + def _record_import_definition( + self, + node: ast.Import | ast.ImportFrom, + scope: ast.AST, + bound: str, + value: DefinitionValue, + ) -> None: + self.definitions.append((node, [(bound, value)])) + self.definition_values[scope][bound].append((node, value)) + def infer(self, node: ast.AST) -> set[str]: key = expression_key(node) result = self._lookup(node, key) if key else set() @@ -554,6 +573,16 @@ def infer(self, node: ast.AST) -> set[str]: callee_facts = self.infer(node.func) if "factory:logger" in callee_facts or "type:logger" in callee_facts: result.add("logger") + if self._is_getattr_call(node) and len(node.args) >= 2: + receiver_facts = self.infer(node.args[0]) + attribute = node.args[1] + if ( + "logger" in receiver_facts + and isinstance(attribute, ast.Constant) + and isinstance(attribute.value, str) + and attribute.value in LOG_METHODS + ): + result.add(f"method:logger:{attribute.value}") if self._is_partial(callee_facts) and node.args: callable_facts = self.infer(node.args[0]) method_facts = {fact for fact in callable_facts if fact.startswith("method:")} @@ -600,6 +629,15 @@ def infer(self, node: ast.AST) -> set[str]: def _is_partial(facts: set[str]) -> bool: return "factory:partial" in facts + def _is_getattr_call(self, node: ast.Call) -> bool: + if isinstance(node.func, ast.Name): + return node.func.id == "getattr" + return ( + isinstance(node.func, ast.Attribute) + and node.func.attr == "getattr" + and "module:builtins" in self.infer(node.func.value) + ) + def _resolve_definitions(self, tree: ast.Module) -> None: self._seed_special_attributes(tree) changed = True @@ -882,6 +920,35 @@ def call_shape_with_partial(call: ast.Call, method: str, callee_facts: set[str]) return shape +def is_standard_file_descriptor_write(call: ast.Call, method: str) -> bool: + if method != "os.write": + return True + if not call.args: + return False + descriptor = call.args[0] + return ( + isinstance(descriptor, ast.Constant) + and type(descriptor.value) is int + and descriptor.value in {1, 2} + ) + + +def is_unknown_logging_callback(argument: ast.AST, keyword: str | None) -> bool: + if not isinstance(argument, ast.Attribute) or argument.attr not in LOG_METHODS: + return False + receiver = expression_key(argument.value) + receiver_name = receiver.rsplit(".", 1)[-1].lower() if receiver else "" + logger_like_receiver = receiver_name in {"log", "logger"} or receiver_name.endswith( + ("_log", "_logger") + ) + callback_keyword = keyword is not None and ( + keyword.startswith("on_") + or keyword.endswith(("_callback", "_handler")) + or keyword in {"callback", "handler"} + ) + return logger_like_receiver or callback_keyword + + def signals_for(text: str) -> list[str]: normalized = text.lower() groups = [ @@ -992,6 +1059,8 @@ def record( catch_value, ) else: + if not is_standard_file_descriptor_write(node, sink_method): + continue record( node, call_text, @@ -1033,13 +1102,18 @@ def record( if not _is_partial_call(callee_facts): callback_arguments = [ - *node.args, - *(keyword.value for keyword in node.keywords if keyword.arg is not None), + *((argument, None) for argument in node.args), + *( + (keyword.value, keyword.arg) + for keyword in node.keywords + if keyword.arg is not None + ), ] - for argument in callback_arguments: - for fact in facts.infer(argument): - if not fact.startswith("method:"): - continue + for argument, keyword in callback_arguments: + method_facts = { + fact for fact in facts.infer(argument) if fact.startswith("method:") + } + for fact in method_facts: _, sink_kind, sink_method = fact.split(":", 2) record( argument, @@ -1053,6 +1127,21 @@ def record( else "none", "callback value", ) + if ( + not method_facts + and is_unknown_logging_callback(argument, keyword) + and isinstance(argument, ast.Attribute) + ): + record( + argument, + normalize_node(argument, source), + "logging-candidate-callback", + "unknown", + argument.attr, + "dynamic-message", + guarded_policy(argument, parents, facts), + "callback value", + ) findings.sort(key=lambda finding: (finding.file, finding.line, finding.column, finding.kind)) counts = Counter(finding.group_fingerprint for finding in findings) diff --git a/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py index 4df2528515..38eadab817 100644 --- a/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py +++ b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py @@ -300,6 +300,26 @@ def test_inventories_pprint_pp_aliases(self) -> None: ], ) + def test_inventories_standard_file_descriptor_writes(self) -> None: + findings = inventory_source( + """ +import os +from os import write as emit + +os.write(2, secret_bytes) +emit(1, secret_bytes) +os.write(3, secret_bytes) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [ + ("raw-output", "os.write", "payload"), + ("raw-output", "os.write", "payload"), + ], + ) + def test_inventories_directly_imported_output_streams(self) -> None: findings = inventory_source( """ @@ -541,6 +561,49 @@ class flag: self.assertEqual([item.policy for item in findings], ["none", "none"]) + def test_non_policy_imports_kill_policy_alias_provenance(self) -> None: + findings = inventory_source( + """ +from agents.logger import logger + +def direct_import(secret): + from agents._debug import DONT_LOG_TOOL_DATA as flag + from settings import enabled as flag + if not flag: + logger.error("direct import: %s", secret) + +def module_import(secret): + from agents._debug import DONT_LOG_TOOL_DATA as flag + import settings as flag + if not flag: + logger.error("module import: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["none", "none"]) + + def test_starred_targets_kill_policy_alias_provenance(self) -> None: + findings = inventory_source( + """ +from agents import _debug +from agents.logger import logger + +def assignment(secret): + flag = _debug.DONT_LOG_TOOL_DATA + *flag, tail = [False] + if not flag: + logger.error("starred assignment: %s", secret) + +def loop(values, secret): + flag = _debug.DONT_LOG_TOOL_DATA + for *flag, tail in values: + if not flag: + logger.error("starred loop: %s", secret) +""" + ) + + self.assertEqual([item.policy for item in findings], ["none", "none"]) + def test_runtime_binders_kill_imported_policy_alias_provenance(self) -> None: findings = inventory_source( """ @@ -777,6 +840,42 @@ def test_recognizes_conditional_helper_callees(self) -> None: ], ) + def test_resolves_constant_getattr_on_known_loggers(self) -> None: + findings = inventory_source( + """ +import builtins +from agents.logger import logger + +emit = getattr(logger, "error") +emit(secret) +builtins.getattr(logger, "warning")(secret) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.shape) for item in findings], + [ + ("logger", "error", "dynamic-message"), + ("logger", "warning", "dynamic-message"), + ], + ) + + def test_inventories_callbacks_on_unknown_logger_receivers(self) -> None: + findings = inventory_source( + """ +register(log.warning) +register(on_error=log.error) +""" + ) + + self.assertEqual( + [(item.kind, item.method, item.confidence) for item in findings], + [ + ("logging-candidate-callback", "warning", "unknown"), + ("logging-candidate-callback", "error", "unknown"), + ], + ) + def test_flags_caught_values_and_implicit_exception_payloads(self) -> None: findings = inventory_source( """ diff --git a/src/agents/tracing/provider.py b/src/agents/tracing/provider.py index b2cd018c08..8f4d884e86 100644 --- a/src/agents/tracing/provider.py +++ b/src/agents/tracing/provider.py @@ -6,6 +6,7 @@ import time import uuid from abc import ABC, abstractmethod +from collections.abc import Callable from datetime import datetime, timezone from inspect import Parameter, signature from typing import Any, cast @@ -19,7 +20,7 @@ from .traces import NoOpTrace, Trace, TraceImpl -def _safe_debug(message: str) -> None: +def _safe_debug(message: str | Callable[[], str]) -> None: """Best-effort debug logging that tolerates closed streams during shutdown.""" def _has_closed_stream_handler(log: logging.Logger) -> bool: @@ -38,7 +39,7 @@ def _has_closed_stream_handler(log: logging.Logger) -> bool: # Avoid emitting debug logs when any handler already owns a closed stream. if _has_closed_stream_handler(logger): return - logger.debug(message) + logger.debug(message() if callable(message) else message) except Exception: # Avoid noisy shutdown errors when the underlying stream is already closed. return @@ -109,7 +110,10 @@ def on_trace_start(self, trace: Trace) -> None: processor.on_trace_start(trace) except Exception as e: log_model_and_tool_action_error( - logger, "Error in trace processor during on_trace_start", e + logger, + "Error in trace processor during on_trace_start", + e, + diagnostic_extra=lambda processor=processor: {"trace_processor": processor}, ) def on_trace_end(self, trace: Trace) -> None: @@ -121,7 +125,10 @@ def on_trace_end(self, trace: Trace) -> None: processor.on_trace_end(trace) except Exception as e: log_model_and_tool_action_error( - logger, "Error in trace processor during on_trace_end", e + logger, + "Error in trace processor during on_trace_end", + e, + diagnostic_extra=lambda processor=processor: {"trace_processor": processor}, ) def on_span_start(self, span: Span[Any]) -> None: @@ -133,7 +140,10 @@ def on_span_start(self, span: Span[Any]) -> None: processor.on_span_start(span) except Exception as e: log_model_and_tool_action_error( - logger, "Error in trace processor during on_span_start", e + logger, + "Error in trace processor during on_span_start", + e, + diagnostic_extra=lambda processor=processor: {"trace_processor": processor}, ) def on_span_end(self, span: Span[Any]) -> None: @@ -145,7 +155,10 @@ def on_span_end(self, span: Span[Any]) -> None: processor.on_span_end(span) except Exception as e: log_model_and_tool_action_error( - logger, "Error in trace processor during on_span_end", e + logger, + "Error in trace processor during on_span_end", + e, + diagnostic_extra=lambda processor=processor: {"trace_processor": processor}, ) def shutdown(self, timeout: float | None = None) -> None: @@ -154,7 +167,12 @@ def shutdown(self, timeout: float | None = None) -> None: """ deadline = None if timeout is None else time.monotonic() + timeout for processor in self._processors: - _safe_debug(f"Shutting down trace processor {processor}") + if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: + _safe_debug("Shutting down trace processor") + else: + _safe_debug( + lambda processor=processor: f"Shutting down trace processor {processor}" + ) try: processor_timeout = _remaining_timeout(deadline) if processor_timeout is not None and processor_timeout <= 0: @@ -167,7 +185,12 @@ def shutdown(self, timeout: float | None = None) -> None: else: processor.shutdown() except Exception as e: - log_model_and_tool_action_error(logger, "Error shutting down trace processor", e) + log_model_and_tool_action_error( + logger, + "Error shutting down trace processor", + e, + diagnostic_extra=lambda processor=processor: {"trace_processor": processor}, + ) def force_flush(self): """ @@ -177,7 +200,12 @@ def force_flush(self): try: processor.force_flush() except Exception as e: - log_model_and_tool_action_error(logger, "Error flushing trace processor", e) + log_model_and_tool_action_error( + logger, + "Error flushing trace processor", + e, + diagnostic_extra=lambda processor=processor: {"trace_processor": processor}, + ) class TraceProvider(ABC): diff --git a/tests/test_error_logging_redaction.py b/tests/test_error_logging_redaction.py index 6e0aa77f87..16f008eceb 100644 --- a/tests/test_error_logging_redaction.py +++ b/tests/test_error_logging_redaction.py @@ -10,6 +10,7 @@ import logging from pathlib import Path +from typing import Any from unittest.mock import patch import httpx @@ -40,6 +41,10 @@ log_tool_action_error, resolve_approval_rejection_message, ) +from agents.tracing.processor_interface import TracingProcessor +from agents.tracing.provider import SynchronousMultiTracingProcessor +from agents.tracing.spans import Span +from agents.tracing.traces import Trace _SECRET = "super secret prompt content" @@ -66,6 +71,36 @@ def __getattribute__(self, name: str): return super().__getattribute__(name) +class _FailingTracingProcessor(TracingProcessor): + def __init__(self) -> None: + self.str_calls = 0 + + def __str__(self) -> str: + self.str_calls += 1 + return "SECRET_TRACE_PROCESSOR_ID" + + def _fail(self) -> None: + raise ValueError(_SECRET) + + def on_trace_start(self, trace: Trace) -> None: + self._fail() + + def on_trace_end(self, trace: Trace) -> None: + self._fail() + + def on_span_start(self, span: Span[Any]) -> None: + self._fail() + + def on_span_end(self, span: Span[Any]) -> None: + self._fail() + + def shutdown(self) -> None: + self._fail() + + def force_flush(self) -> None: + self._fail() + + def _emit_shared_error_for_location(test_logger, helper) -> None: helper(test_logger, "Fixed operational message", ValueError("failure")) @@ -276,6 +311,58 @@ def diagnostic_extra() -> dict[str, object]: assert record.__dict__["sandbox_id"] == _SECRET +@pytest.mark.parametrize( + "operation", + [ + "on_trace_start", + "on_trace_end", + "on_span_start", + "on_span_end", + "force_flush", + "shutdown", + ], +) +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted"), + [(True, False), (False, True), (False, False)], +) +def test_trace_processor_failure_identity_follows_both_data_policies( + monkeypatch, + operation: str, + model_redacted: bool, + tool_redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + test_logger = logging.Logger("sensitive-logging-trace-processor", level=logging.DEBUG) + test_logger.propagate = False + handler = _RecordingHandler() + test_logger.addHandler(handler) + failing = _FailingTracingProcessor() + multi = SynchronousMultiTracingProcessor() + multi.add_tracing_processor(failing) + + with patch("agents.tracing.provider.logger", test_logger): + if operation.startswith(("on_trace", "on_span")): + getattr(multi, operation)(object()) + else: + getattr(multi, operation)() + + record = next(record for record in handler.records if record.levelno == logging.ERROR) + redacted = model_redacted or tool_redacted + if redacted: + assert "trace_processor" not in record.__dict__ + assert failing not in record.__dict__.values() + assert record.exc_info is None + assert _SECRET not in logging.Formatter().format(record) + assert failing.str_calls == 0 + else: + assert record.__dict__["trace_processor"] is failing + assert record.exc_info is not None + assert record.exc_info[1] is not None + assert _SECRET in logging.Formatter().format(record) + + @pytest.mark.parametrize( "helper", [log_shared_tool_action_error, log_tool_action_warning], From fa372d6d283858b8da9150e6ea18c82b803b994e Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Fri, 24 Jul 2026 23:05:28 +0900 Subject: [PATCH 06/12] fix --- .../skills/sensitive-logging-audit/SKILL.md | 91 +- .../references/redaction-validation.md | 62 +- .../scripts/_inventory.py | 1291 ----------------- .../scripts/inventory_logging.py | 370 +++-- .../scripts/test_inventory.py | 1140 ++------------- src/agents/tracing/provider.py | 25 +- 6 files changed, 465 insertions(+), 2514 deletions(-) delete mode 100644 .agents/skills/sensitive-logging-audit/scripts/_inventory.py diff --git a/.agents/skills/sensitive-logging-audit/SKILL.md b/.agents/skills/sensitive-logging-audit/SKILL.md index 201ac2fdf1..ca150f4240 100644 --- a/.agents/skills/sensitive-logging-audit/SKILL.md +++ b/.agents/skills/sensitive-logging-audit/SKILL.md @@ -1,93 +1,78 @@ --- name: sensitive-logging-audit -description: Audit and fix sensitive-data exposure through Python runtime logging in openai-agents-python. Use when reviewing logging, print, warnings, stderr, or traceback output; checking OPENAI_AGENTS_DONT_LOG_MODEL_DATA or OPENAI_AGENTS_DONT_LOG_TOOL_DATA coverage; investigating model, tool, Realtime, MCP, session, sandbox, tracing, or arbitrary exception data in logs; or carrying an audit through fixes, adversarial tests, and repository verification. +description: Audit and fix sensitive-data exposure through Python runtime logging in openai-agents-python. Use when reviewing logging, print, warnings, stderr, traceback, MCP names, model or tool exceptions, redaction flags, or any diagnostic path that may retain user data. --- # Sensitive Logging Audit ## Objective -Complete the remediation, not only the scan. Inventory runtime output sinks, classify every dynamic value, fix every demonstrated model/tool-data leak in scope, add adversarial regression tests, and run the applicable repository gates. +Find candidate output sinks, trace their values manually, fix demonstrated leaks at shared runtime boundaries, and prove redaction with adversarial tests. -Treat the inventory as a conservative review ledger, not automatic taint analysis. Receiver provenance and policy guards improve prioritization but never prove that a dynamic value is safe. +The collector is only a syntax-based search aid. It does not resolve Python aliases or control flow, certify policy guards, or prove that an absent candidate is safe. ## Workflow -### 1. Establish the baseline +### 1. Establish the review surface - Work in the current checkout and preserve unrelated changes. -- Record `git status --short --branch` and the current commit. -- Read `src/agents/_debug.py`, `src/agents/logger.py`, and existing policy-aware helpers before judging call sites. -- Treat exception messages, arguments, tracebacks, causes, contexts, notes, and arbitrary values as potentially sensitive. -- Treat URL-derived display names as structured sensitive values. MCP server names may embed endpoint credentials, query parameters, or fragments even when the log does not contain a model/tool payload object. +- Read `src/agents/_debug.py`, `src/agents/logger.py`, and the affected callers. +- Treat exception messages, arguments, tracebacks, causes, contexts, notes, names, URLs, and arbitrary values as potentially sensitive. +- Read [the Python redaction validation matrix](references/redaction-validation.md). -Run the detector tests before relying on its output: +Run the collector tests, then collect candidates: ```bash uv run python .agents/skills/sensitive-logging-audit/scripts/test_inventory.py +uv run python .agents/skills/sensitive-logging-audit/scripts/inventory_logging.py \ + --format json --output /tmp/sensitive-logging-candidates.json ``` -Create a baseline from the repository root: +The report intentionally contains no `policy`, `safe`, or guard classification. + +### 2. Supplement the collector with source search + +The collector does not follow assignments such as `emit = logger.error`. Search the source directly and inspect aliases, callbacks, wrappers, and reflective dispatch: ```bash -uv run python .agents/skills/sensitive-logging-audit/scripts/inventory_logging.py \ - --format json --output /tmp/sensitive-logging-before.json -uv run python .agents/skills/sensitive-logging-audit/scripts/inventory_logging.py \ - --summary-only +rg -n '\.(debug|info|warning|warn|error|exception|critical|fatal|log)\b' src/agents +rg -n '\b(print|pprint|pp|warn|warn_explicit|write|writelines|print_exc|print_exception)\b' src/agents +rg -n 'DONT_LOG_(MODEL|TOOL)_DATA|log_(model|tool|model_and_tool)_action' src/agents ``` -### 2. Classify every dynamic sink +Do not turn collector coverage or a textual guard into a security conclusion. Trace producers and callers. -Review the complete JSON ledger. Prioritize raw output, caught exceptions, `logger.exception`, `exc_info`, `extra`, supplemental formatting arguments, and unknown receivers. +### 3. Classify manually -Assign one disposition to every dynamic fingerprint group: +Assign each reviewed path one disposition: - `model`: model requests, responses, Realtime events, or derived values. - `tool`: tool arguments, outputs, MCP data, tool events, or derived values. - `model+tool`: either class may reach the sink. -- `operational`: proven to contain only non-sensitive SDK metadata. +- `operational`: demonstrated to contain only non-sensitive SDK metadata. - `intentional-output`: explicitly user-facing output rather than diagnostics. -- `uncertain`: source tracing is incomplete; investigate before deciding. - -Record the fingerprint, group count, disposition, evidence, and action. Do not classify from a variable name or message text alone. Trace producers, callbacks, formatters, and exception ownership. - -### 3. Fix demonstrated leaks +- `uncertain`: source tracing is incomplete. -Before changing runtime behavior, use `$implementation-strategy`. Implement the narrowest shared-boundary fix. +Record evidence in the audit report. The script does not validate or inherit dispositions. -- Apply `_debug.DONT_LOG_MODEL_DATA` and `_debug.DONT_LOG_TOOL_DATA` before formatting or inspecting sensitive values. -- Redact `model+tool` values when either relevant flag disables data logging. -- Preserve existing diagnostics when sensitive-data logging is explicitly enabled. -- In redacted mode, emit a fixed message. Do not inspect or pass the sensitive object through `msg`, `args`, `extra`, or `exc_info`. -- In tool-redacted mode, use fixed MCP lifecycle and filter messages without reading `server.name` or including tool names. In diagnostic mode, sanitize URL-derived server names by removing credentials, query parameters, and fragments while retaining ordinary names. Do not mutate the server's public `name` or connection URL. -- Keep logging failure from changing fallback, cleanup, event emission, rejection, or cancellation behavior. -- Leave operational and intentional user-facing values unchanged when their safety is demonstrated. +### 4. Fix runtime boundaries -### 4. Add adversarial regressions - -Read [the Python redaction validation matrix](references/redaction-validation.md) and test every changed caller boundary. Helper-only tests do not prove that callers use the helper correctly. - -### 5. Re-audit the whole tree - -Run the inventory again and compare it to the baseline: - -```bash -uv run python .agents/skills/sensitive-logging-audit/scripts/inventory_logging.py \ - --compare /tmp/sensitive-logging-before.json \ - --format json --output /tmp/sensitive-logging-after.json -``` +Before changing runtime behavior, use `$implementation-strategy`. -Inspect every new, removed, count-changed, or classification-changed fingerprint group. Classification changes include policy, shape, sink kind, confidence, method, and caught-value status. Duplicate groups deliberately require group-level re-review instead of inheriting an order-dependent disposition. +- Check the relevant `_debug.DONT_LOG_MODEL_DATA` and `_debug.DONT_LOG_TOOL_DATA` flags before formatting or inspecting sensitive values. +- Redact mixed model/tool values when either flag disables data logging. +- In redacted mode, emit a fixed message and omit sensitive `args`, `extra`, and `exc_info`. +- Build diagnostic-only context lazily so redacted mode never reads it. +- Preserve useful diagnostics when sensitive-data logging is explicitly enabled. +- Keep logging failure from changing fallback, cleanup, event, rejection, or cancellation behavior. +- For MCP URLs, remove credentials, query parameters, and fragments in diagnostic mode; never use sanitized names as a substitute for fixed redacted messages. -The completion report must state total and dynamic sink counts, confirmed leaks fixed, retained candidates and evidence, duplicate groups, verification results, and remaining uncertainty. +### 5. Prove caller behavior -### 6. Run close-out gates +Add tests at every changed caller boundary. Inspect the complete `LogRecord`, not only rendered text. Test both redacted policies, diagnostic mode, hostile objects, exception chains, and the caller's observable fallback or cleanup behavior as applicable. -- Run the detector tests and skill validation after changing the skill. -- For runtime code, tests, examples, or build/test behavior, use `$code-change-verification` after the final fix. -- Use `$pr-draft-summary` when required by the repository instructions. -- Stop after local changes and verification unless the user explicitly requests a remote action. +### 6. Re-run and close out -## Reporting +Re-run the collector, the manual searches, focused tests, and applicable repository gates. Use `$code-change-verification` for runtime or test changes and `$pr-draft-summary` when required. -Lead with whether real leaks were found and fixed. Separate confirmed leaks from conservative candidates and intentional output. Never equate a clean inventory shape or a recognized policy guard with proof that all dynamic values are non-sensitive. +Report candidate counts as search coverage only. Lead with confirmed leaks fixed, retained intentional output, reviewed uncertainty, and verification results. Never report a clean collector result as proof that no sensitive logging path exists. diff --git a/.agents/skills/sensitive-logging-audit/references/redaction-validation.md b/.agents/skills/sensitive-logging-audit/references/redaction-validation.md index a9f5dd0ee9..35b2ab6cba 100644 --- a/.agents/skills/sensitive-logging-audit/references/redaction-validation.md +++ b/.agents/skills/sensitive-logging-audit/references/redaction-validation.md @@ -1,6 +1,6 @@ # Python sensitive logging validation -The inventory is deliberately broader than a vulnerability detector. It finds confirmed SDK logger calls, raw output, exact policy-aware helpers, and unknown receivers with logging-like method names. Review every dynamic finding; do not treat a recognized receiver, helper, or guard as proof that its values are safe. +The collector reports syntactic logging and raw-output candidates. It does not resolve aliases, prove receiver types, evaluate guards, classify payloads, or support a completeness claim. Review candidates together with direct source searches and runtime tests. ## Required validation matrix @@ -39,41 +39,25 @@ The sensitive object itself must not remain attached even when its string repres ## Review procedure -1. Run the inventory against all of `src/agents`. -2. Review raw output and unknown receivers first. -3. Review caught values, `logger.exception`, `exc_info`, `extra`, and formatting arguments. -4. Trace model, tool, Realtime, MCP, session, sandbox, voice, tracing, and cleanup values to their producers. -5. Classify intentional output separately from diagnostics; do not silently exempt `print` or warnings. -6. Add focused tests at every changed caller boundary. -7. Re-run the inventory and compare fingerprint groups to the baseline. -8. Re-review any duplicate group whose count changed. - -The detector supports a completeness claim for the repository's direct Python output shapes. It does not prove arbitrary runtime data flow, dynamically installed handlers, monkey-patched logger methods, or reflective calls. - -## Review ledger shape - -Use a temporary JSON ledger when the audit is large: - -```json -{ - "reviews": [ - { - "group_fingerprint": "0123456789ab", - "group_count": 1, - "disposition": "tool", - "evidence": "The value originates from the function tool input JSON.", - "action": "Guard formatting and arguments with DONT_LOG_TOOL_DATA." - } - ] -} -``` - -Validate it with: - -```bash -uv run python .agents/skills/sensitive-logging-audit/scripts/inventory_logging.py \ - --validate-review /tmp/sensitive-logging-review.json --summary-only -``` - -Allowed dispositions are `model`, `tool`, `model+tool`, `operational`, `intentional-output`, and `uncertain`. -Both `evidence` and `action` must be non-empty JSON strings; structured values do not count as review rationale. +1. Run the collector against all of `src/agents`. +2. Run the supplemental `rg` searches from `SKILL.md` and inspect aliases and dynamic dispatch. +3. Review raw output and ambiguous receivers first. +4. Review caught values, `logger.exception`, `exc_info`, `extra`, and formatting arguments. +5. Trace model, tool, Realtime, MCP, session, sandbox, voice, tracing, and cleanup values to their producers. +6. Classify intentional output separately from diagnostics; do not silently exempt `print` or warnings. +7. Add focused tests at every changed caller boundary. +8. Re-run the collector and source searches after the fix. + +An empty or unchanged collector report is not proof of safety. Assignment aliases, monkey-patched methods, dynamically installed handlers, non-constant reflection, and arbitrary runtime data flow require manual inspection. + +## Audit report expectations + +For each confirmed or uncertain path, record: + +- The source location and value producer. +- The manual disposition: `model`, `tool`, `model+tool`, `operational`, `intentional-output`, or `uncertain`. +- Concrete evidence for the disposition. +- The fix or reason for retaining the path. +- The caller-level regression test, when behavior changed. + +Do not reuse a disposition solely because a fingerprint or call text is unchanged. diff --git a/.agents/skills/sensitive-logging-audit/scripts/_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/_inventory.py deleted file mode 100644 index e6da3b0b5f..0000000000 --- a/.agents/skills/sensitive-logging-audit/scripts/_inventory.py +++ /dev/null @@ -1,1291 +0,0 @@ -from __future__ import annotations - -import ast -import hashlib -import importlib.util -import re -from collections import Counter, defaultdict -from collections.abc import Iterable, Mapping, Sequence -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any - -LOG_METHODS = { - "critical", - "debug", - "error", - "exception", - "fatal", - "info", - "log", - "warn", - "warning", -} -SDK_LOGGER_MODULES = {"agents.logger", "agents.tracing.logger"} -POLICY_MODULE = "agents._debug" -POLICY_NAMES = { - "DONT_LOG_MODEL_DATA": "model", - "DONT_LOG_TOOL_DATA": "tool", -} -KNOWN_HELPERS = { - "agents.logger.log_model_action_debug": "model", - "agents.logger.log_model_action_error": "model", - "agents.logger.log_model_action_warning": "model", - "agents.logger.log_model_and_tool_action_debug": "model+tool", - "agents.logger.log_model_and_tool_action_error": "model+tool", - "agents.logger.log_model_and_tool_action_warning": "model+tool", - "agents.logger.log_tool_action_debug": "tool", - "agents.logger.log_tool_action_error": "tool", - "agents.logger.log_tool_action_warning": "tool", - "agents.run_internal.tool_execution.log_tool_action_error": "tool", -} -SENSITIVE_HELPER_METHODS = {name.rsplit(".", 1)[-1] for name in KNOWN_HELPERS} -RAW_MODULE_METHODS = { - "builtins": {"print"}, - "os": {"write"}, - "pprint": {"pp", "pprint"}, - "traceback": {"print_exc", "print_exception"}, - "warnings": {"warn", "warn_explicit"}, -} -DISPOSITIONS = { - "intentional-output", - "model", - "model+tool", - "operational", - "tool", - "uncertain", -} -COMPARISON_CLASSIFICATION_FIELDS = ( - "kind", - "confidence", - "method", - "shape", - "policy", - "catch_value", -) - -DefinitionValue = ast.AST | frozenset[str] | None - - -def _helper_fact(full_name: str) -> str: - return f"helper:{KNOWN_HELPERS[full_name]}:{full_name.rsplit('.', 1)[-1]}" - - -def _is_sdk_logger_module_or_parent(module: str) -> bool: - return any( - candidate == module or candidate.startswith(f"{module}.") - for candidate in SDK_LOGGER_MODULES - ) - - -@dataclass -class Finding: - group_fingerprint: str - fingerprint: str - file: str - line: int - column: int - kind: str - confidence: str - method: str - shape: str - policy: str - catch_value: str | None - context: str - signals: list[str] - call: str - site_index: int = 0 - group_count: int = 1 - identity_quality: str = "unique" - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - -def normalize_path(path: str | Path) -> str: - return str(path).replace("\\", "/") - - -def collect_source_files(roots: Sequence[str | Path]) -> list[Path]: - files: set[Path] = set() - for root_value in roots: - root = Path(root_value).resolve() - if root.is_file(): - if root.suffix == ".py": - files.add(root) - continue - if not root.is_dir(): - raise FileNotFoundError(f"Inventory root does not exist: {root_value}") - for path in root.rglob("*.py"): - relative_parts = path.relative_to(root).parts - if any(part.startswith(".") or part == "__pycache__" for part in relative_parts): - continue - files.add(path.resolve()) - return sorted(files) - - -def module_identity(file_path: str) -> tuple[str, str]: - path = normalize_path(file_path) - marker = "/src/" - relative = path.split(marker, 1)[1] if marker in path else path.removeprefix("src/") - parts = relative.split("/") - if parts[-1] == "__init__.py": - module = ".".join(parts[:-1]) - return module, module - parts[-1] = parts[-1].removesuffix(".py") - module = ".".join(parts) - package = module.rsplit(".", 1)[0] if "." in module else module - return module, package - - -def resolve_import(module: str | None, level: int, package: str) -> str: - if level == 0: - return module or "" - name = "." * level + (module or "") - try: - return importlib.util.resolve_name(name, package) - except (ImportError, ValueError): - return name - - -def expression_key(node: ast.AST) -> str | None: - if isinstance(node, ast.Name): - return node.id - if isinstance(node, ast.Attribute): - receiver = expression_key(node.value) - return f"{receiver}.{node.attr}" if receiver else None - if isinstance(node, ast.Subscript) and isinstance(node.slice, ast.Constant): - receiver = expression_key(node.value) - if receiver and isinstance(node.slice.value, str | int): - return f"{receiver}[{node.slice.value!r}]" - return None - - -def target_keys(node: ast.AST) -> list[str]: - key = expression_key(node) - if key: - return [key] - if isinstance(node, ast.Starred): - return target_keys(node.value) - if isinstance(node, ast.List | ast.Tuple): - return [key for element in node.elts for key in target_keys(element)] - return [] - - -def target_value_pairs(target: ast.AST, value: ast.AST) -> list[tuple[str, ast.AST | None]]: - key = expression_key(target) - if key: - return [(key, value)] - if isinstance(target, ast.Starred): - return [(key, None) for key in target_keys(target.value)] - if isinstance(target, ast.List | ast.Tuple): - if isinstance(value, ast.List | ast.Tuple) and len(target.elts) == len(value.elts): - return [ - pair - for target_element, value_element in zip(target.elts, value.elts, strict=True) - for pair in target_value_pairs(target_element, value_element) - ] - return [(key, None) for key in target_keys(target)] - return [] - - -def pattern_bound_names(pattern: ast.pattern) -> set[str]: - if isinstance(pattern, ast.MatchAs): - names = pattern_bound_names(pattern.pattern) if pattern.pattern is not None else set() - if pattern.name: - names.add(pattern.name) - return names - if isinstance(pattern, ast.MatchStar): - return {pattern.name} if pattern.name else set() - if isinstance(pattern, ast.MatchMapping): - names = {name for child in pattern.patterns for name in pattern_bound_names(child)} - if pattern.rest: - names.add(pattern.rest) - return names - if isinstance(pattern, ast.MatchClass): - return { - name - for child in [*pattern.patterns, *pattern.kwd_patterns] - for name in pattern_bound_names(child) - } - if isinstance(pattern, ast.MatchSequence | ast.MatchOr): - return {name for child in pattern.patterns for name in pattern_bound_names(child)} - return set() - - -def iter_definitions( - tree: ast.AST, -) -> Iterable[tuple[ast.AST, list[tuple[str, ast.AST | None]]]]: - for node in ast.walk(tree): - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): - yield node, [(node.name, None)] - elif isinstance(node, ast.Assign): - for target in node.targets: - yield node, target_value_pairs(target, node.value) - elif isinstance(node, ast.AnnAssign) and node.value is not None: - yield node, target_value_pairs(node.target, node.value) - elif isinstance(node, ast.AugAssign): - yield node, [(key, None) for key in target_keys(node.target)] - elif isinstance(node, ast.NamedExpr): - yield node, target_value_pairs(node.target, node.value) - elif isinstance(node, ast.For | ast.AsyncFor): - yield node, [(key, None) for key in target_keys(node.target)] - elif isinstance(node, ast.With | ast.AsyncWith): - targets = [ - (key, None) - for item in node.items - if item.optional_vars is not None - for key in target_keys(item.optional_vars) - ] - if targets: - yield node, targets - elif isinstance(node, ast.ExceptHandler) and node.name: - yield node, [(node.name, None)] - elif isinstance(node, ast.Delete): - yield node, [(key, None) for target in node.targets for key in target_keys(target)] - elif isinstance(node, ast.match_case): - names = pattern_bound_names(node.pattern) - if names: - yield node.pattern, [(name, None) for name in sorted(names)] - - -def bound_names(node: ast.AST) -> set[str]: - if isinstance(node, ast.Name): - return {node.id} - if isinstance(node, ast.Starred): - return bound_names(node.value) - if isinstance(node, ast.List | ast.Tuple): - return {name for element in node.elts for name in bound_names(element)} - return set() - - -class Facts: - def __init__(self, tree: ast.Module, file_path: str): - self.values: dict[ast.AST, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set)) - self.bindings: dict[ast.AST, set[str]] = defaultdict(set) - self.node_scopes: dict[ast.AST, ast.AST] = {} - self.scope_parents: dict[ast.AST, ast.AST | None] = {tree: None} - self.parents = make_parent_map(tree) - self.definitions: list[tuple[ast.AST, list[tuple[str, DefinitionValue]]]] = [ - (definition, list(targets)) for definition, targets in iter_definitions(tree) - ] - self.definition_values: dict[ast.AST, dict[str, list[tuple[ast.AST, DefinitionValue]]]] = ( - defaultdict(lambda: defaultdict(list)) - ) - self._policy_lookup_stack: set[tuple[int, str]] = set() - self.module_name, self.package_name = module_identity(file_path) - self._index_scopes(tree) - self._collect_bindings(tree) - for definition, targets in self.definitions: - scope = self._scope_for(definition) - for key, value in targets: - self.definition_values[scope][key].append((definition, value)) - self._collect_imports(tree) - self._resolve_definitions(tree) - - def _index_scopes(self, tree: ast.Module) -> None: - scope_types = ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda | ast.ClassDef - - def visit(node: ast.AST, scope: ast.AST) -> None: - self.node_scopes[node] = scope - for child in ast.iter_child_nodes(node): - if isinstance(child, scope_types): - self.node_scopes[child] = scope - parent_scope = ( - self.scope_parents[scope] if isinstance(scope, ast.ClassDef) else scope - ) - self.scope_parents[child] = parent_scope - for descendant in ast.iter_child_nodes(child): - visit(descendant, child) - else: - visit(child, scope) - - visit(tree, tree) - - def _scope_for(self, node: ast.AST) -> ast.AST: - return self.node_scopes[node] - - def _collect_bindings(self, tree: ast.Module) -> None: - for node in ast.walk(tree): - scope = self._scope_for(node) - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): - self.bindings[scope].add(node.name) - if isinstance(node, ast.arg): - self.bindings[scope].add(node.arg) - elif isinstance(node, ast.Assign): - for target in node.targets: - self.bindings[scope].update(bound_names(target)) - elif isinstance(node, ast.AnnAssign | ast.AugAssign | ast.NamedExpr): - self.bindings[scope].update(bound_names(node.target)) - elif isinstance(node, ast.For | ast.AsyncFor): - self.bindings[scope].update(bound_names(node.target)) - elif isinstance(node, ast.comprehension): - self.bindings[scope].update(bound_names(node.target)) - elif isinstance(node, ast.With | ast.AsyncWith): - for item in node.items: - if item.optional_vars is not None: - self.bindings[scope].update(bound_names(item.optional_vars)) - elif isinstance(node, ast.ExceptHandler) and node.name: - self.bindings[scope].add(node.name) - elif isinstance(node, ast.match_case): - self.bindings[scope].update(pattern_bound_names(node.pattern)) - elif isinstance(node, ast.Import): - for alias in node.names: - self.bindings[scope].add(alias.asname or alias.name.split(".", 1)[0]) - elif isinstance(node, ast.ImportFrom): - for alias in node.names: - if alias.name != "*": - self.bindings[scope].add(alias.asname or alias.name) - - def add(self, scope: ast.AST, key: str, values: Iterable[str]) -> bool: - before = len(self.values[scope][key]) - self.values[scope][key].update(values) - return len(self.values[scope][key]) != before - - def _lookup(self, node: ast.AST, key: str) -> set[str]: - scope: ast.AST | None = self._scope_for(node) - root = key.split(".", 1)[0].split("[", 1)[0] - while scope is not None: - if key in self.values[scope]: - result = set(self.values[scope][key]) - result = {fact for fact in result if not fact.startswith("policy")} - result.update(self._lookup_policy(node, key)) - return result - if root in self.bindings[scope]: - return self._lookup_policy(node, key) - scope = self.scope_parents[scope] - return set() - - def _lookup_policy(self, node: ast.AST, key: str) -> set[str]: - token = (id(node), key) - if token in self._policy_lookup_stack: - return set() - - use_scope = self._scope_for(node) - scope: ast.AST | None = use_scope - root = key.split(".", 1)[0].split("[", 1)[0] - while scope is not None: - definitions = self.definition_values[scope].get(key, []) - if definitions: - if scope is not use_scope: - if all(isinstance(value, frozenset) for _, value in definitions): - _, imported_facts = max( - definitions, key=lambda item: self._node_position(item[0]) - ) - return set(imported_facts) - return set() - preceding = [ - definition - for definition in definitions - if self._node_position(definition[0]) < self._node_position(node) - ] - if not preceding: - return set() - assignment, value = max(preceding, key=lambda item: self._node_position(item[0])) - if not self._definition_precedes_in_same_block(assignment, node): - return set() - if value is None: - return set() - if isinstance(value, frozenset): - return set(value) - self._policy_lookup_stack.add(token) - try: - return { - fact - for fact in self.infer(value) - if fact.startswith(("policy:", "policy-exact:")) - } - finally: - self._policy_lookup_stack.remove(token) - - if root in self.bindings[scope]: - return set() - scope = self.scope_parents[scope] - return set() - - @staticmethod - def _node_position(node: ast.AST) -> tuple[int, int]: - return (getattr(node, "lineno", -1), getattr(node, "col_offset", -1)) - - def _definition_precedes_in_same_block(self, definition: ast.AST, use: ast.AST) -> bool: - if isinstance(definition, ast.For | ast.AsyncFor | ast.With | ast.AsyncWith): - child = use - while child in self.parents and self.parents[child] is not definition: - child = self.parents[child] - if child in definition.body: - return True - if isinstance(definition, ast.For | ast.AsyncFor) and child in definition.orelse: - return True - elif isinstance(definition, ast.ExceptHandler): - child = use - while child in self.parents and self.parents[child] is not definition: - child = self.parents[child] - if child in definition.body: - return True - elif isinstance(definition, ast.pattern): - case = self.parents.get(definition) - if isinstance(case, ast.match_case): - child = use - while child in self.parents and self.parents[child] is not case: - child = self.parents[child] - if child is case.guard or child in case.body: - return True - - parent = self.parents.get(definition) - if parent is None: - return False - for _, value in ast.iter_fields(parent): - if not isinstance(value, list) or definition not in value: - continue - definition_index = value.index(definition) - child = use - while child in self.parents and self.parents[child] is not parent: - child = self.parents[child] - return child in value and definition_index < value.index(child) - return False - - def _collect_imports(self, tree: ast.Module) -> None: - for node in ast.walk(tree): - scope = self._scope_for(node) - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): - full_name = f"{self.module_name}.{node.name}" - if scope is tree and full_name in KNOWN_HELPERS: - self.add(scope, node.name, {_helper_fact(full_name)}) - elif isinstance(node, ast.Import): - for alias in node.names: - bound = alias.asname or alias.name.split(".", 1)[0] - self._record_import_definition(node, scope, bound, None) - if alias.name in { - "builtins", - "logging", - "os", - "pprint", - "sys", - "traceback", - "warnings", - }: - self.add(scope, bound, {f"module:{alias.name}"}) - elif alias.name == POLICY_MODULE: - self.add( - scope, - alias.asname or alias.name, - {f"module:{POLICY_MODULE}"}, - ) - elif alias.name in SDK_LOGGER_MODULES: - imported_module = ( - alias.name if alias.asname else alias.name.split(".", 1)[0] - ) - self.add(scope, bound, {f"module:{imported_module}"}) - elif isinstance(node, ast.ImportFrom): - module = resolve_import(node.module, node.level, self.package_name) - for alias in node.names: - if alias.name == "*": - continue - bound = alias.asname or alias.name - full_name = f"{module}.{alias.name}" if module else alias.name - facts: set[str] = set() - policy_value: DefinitionValue = None - if module == "logging": - if alias.name == "getLogger": - facts.add("factory:logger") - elif alias.name in LOG_METHODS: - facts.add(f"method:logger:{alias.name}") - elif alias.name in {"Logger", "LoggerAdapter"}: - facts.add("type:logger") - if module == "functools" and alias.name == "partial": - facts.add("factory:partial") - if module in SDK_LOGGER_MODULES and alias.name == "logger": - facts.add("logger") - if full_name in SDK_LOGGER_MODULES: - facts.add(f"module:{full_name}") - if full_name == POLICY_MODULE: - facts.add(f"module:{POLICY_MODULE}") - if module == POLICY_MODULE and alias.name in POLICY_NAMES: - policy = POLICY_NAMES[alias.name] - policy_facts = {f"policy:{policy}", f"policy-exact:{policy}"} - facts.update(policy_facts) - policy_value = frozenset(policy_facts) - self._record_import_definition(node, scope, bound, policy_value) - if full_name in KNOWN_HELPERS: - facts.add(_helper_fact(full_name)) - if module in RAW_MODULE_METHODS and alias.name in RAW_MODULE_METHODS[module]: - facts.add(f"method:raw:{module}.{alias.name}") - if module == "sys" and alias.name in {"stdout", "stderr"}: - facts.add(f"stream:{alias.name}") - if facts: - self.add(scope, bound, facts) - - def _record_import_definition( - self, - node: ast.Import | ast.ImportFrom, - scope: ast.AST, - bound: str, - value: DefinitionValue, - ) -> None: - self.definitions.append((node, [(bound, value)])) - self.definition_values[scope][bound].append((node, value)) - - def infer(self, node: ast.AST) -> set[str]: - key = expression_key(node) - result = self._lookup(node, key) if key else set() - if isinstance(node, ast.Name): - if node.id == "print": - result.add("method:raw:print") - return result - if isinstance(node, ast.Attribute): - receiver_facts = self.infer(node.value) - for fact in receiver_facts: - if fact == "module:logging": - if node.attr == "getLogger": - result.add("factory:logger") - elif node.attr in {"Logger", "LoggerAdapter"}: - result.add("type:logger") - elif node.attr in LOG_METHODS: - result.add(f"method:logger:{node.attr}") - elif fact == "logger" and node.attr in LOG_METHODS: - result.add(f"method:logger:{node.attr}") - elif fact == f"module:{POLICY_MODULE}" and node.attr in POLICY_NAMES: - policy = POLICY_NAMES[node.attr] - result.update({f"policy:{policy}", f"policy-exact:{policy}"}) - elif fact.startswith("module:"): - module = fact.split(":", 1)[1] - qualified_name = f"{module}.{node.attr}" - if _is_sdk_logger_module_or_parent(qualified_name): - result.add(f"module:{qualified_name}") - if module in SDK_LOGGER_MODULES: - helper_policy = KNOWN_HELPERS.get(qualified_name) - if helper_policy is not None: - result.add(_helper_fact(qualified_name)) - if node.attr == "logger": - result.add("logger") - if module in RAW_MODULE_METHODS and node.attr in RAW_MODULE_METHODS[module]: - result.add(f"method:raw:{module}.{node.attr}") - if module == "sys" and node.attr in {"stdout", "stderr"}: - result.add(f"stream:{node.attr}") - elif fact.startswith("stream:"): - if node.attr == "buffer": - result.add(fact) - elif node.attr in {"write", "writelines"}: - stream = fact.split(":", 1)[1] - result.add(f"method:raw:{stream}.{node.attr}") - return result - if isinstance(node, ast.Call): - callee_facts = self.infer(node.func) - if "factory:logger" in callee_facts or "type:logger" in callee_facts: - result.add("logger") - if self._is_getattr_call(node) and len(node.args) >= 2: - receiver_facts = self.infer(node.args[0]) - attribute = node.args[1] - if ( - "logger" in receiver_facts - and isinstance(attribute, ast.Constant) - and isinstance(attribute.value, str) - and attribute.value in LOG_METHODS - ): - result.add(f"method:logger:{attribute.value}") - if self._is_partial(callee_facts) and node.args: - callable_facts = self.infer(node.args[0]) - method_facts = {fact for fact in callable_facts if fact.startswith("method:")} - result.update(method_facts) - inherited_shape = any(fact.startswith("partial-shape:") for fact in callable_facts) - if len(node.args) == 1 and not node.keywords and not inherited_shape: - result.add("partial-shape:empty") - else: - bound_call = ast.Call( - func=node.args[0], - args=node.args[1:], - keywords=node.keywords, - ) - for method_fact in method_facts: - _, _, method = method_fact.split(":", 2) - shape = call_shape_with_partial(bound_call, method, callable_facts) - result.add(f"partial-shape:{shape}") - return result - if isinstance(node, ast.BoolOp): - for value in node.values: - result.update(self.infer(value)) - result = {fact for fact in result if not fact.startswith("policy-exact:")} - return result - if isinstance(node, ast.IfExp): - result.update(self.infer(node.body)) - result.update(self.infer(node.orelse)) - result = {fact for fact in result if not fact.startswith("policy-exact:")} - return result - if isinstance(node, ast.UnaryOp): - result.update(self.infer(node.operand)) - return {fact for fact in result if not fact.startswith("policy-exact:")} - if isinstance(node, ast.Compare): - result.update(self.infer(node.left)) - for comparator in node.comparators: - result.update(self.infer(comparator)) - return {fact for fact in result if not fact.startswith("policy-exact:")} - if isinstance(node, ast.Tuple | ast.List | ast.Set): - for element in node.elts: - result.update(self.infer(element)) - return result - return result - - @staticmethod - def _is_partial(facts: set[str]) -> bool: - return "factory:partial" in facts - - def _is_getattr_call(self, node: ast.Call) -> bool: - if isinstance(node.func, ast.Name): - return node.func.id == "getattr" - return ( - isinstance(node.func, ast.Attribute) - and node.func.attr == "getattr" - and "module:builtins" in self.infer(node.func.value) - ) - - def _resolve_definitions(self, tree: ast.Module) -> None: - self._seed_special_attributes(tree) - changed = True - while changed: - changed = False - for definition, targets in self.definitions: - scope = self._scope_for(definition) - for key, value in targets: - if isinstance(value, frozenset): - inferred = value - else: - inferred = self.infer(value) if value is not None else set() - changed |= self.add(scope, key, inferred) - - def _seed_special_attributes(self, tree: ast.Module) -> None: - for node in ast.walk(tree): - if not isinstance(node, ast.Import): - continue - scope = self._scope_for(node) - for alias in node.names: - if alias.name == "functools": - bound = alias.asname or "functools" - self.add(scope, f"{bound}.partial", {"factory:partial"}) - - -def make_parent_map(tree: ast.AST) -> dict[ast.AST, ast.AST]: - return {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)} - - -def normalize_node(node: ast.AST, source: str) -> str: - segment = ast.get_source_segment(source, node) - if segment is None: - segment = ast.dump(node, annotate_fields=True, include_attributes=False) - return re.sub(r"\s+", " ", segment).strip() - - -def node_references_name(node: ast.AST, name: str) -> bool: - return any(isinstance(child, ast.Name) and child.id == name for child in ast.walk(node)) - - -def branch_for_child(parent: ast.AST, child: ast.AST) -> tuple[ast.AST, bool] | None: - if isinstance(parent, ast.If): - if child in parent.body: - return parent.test, True - if child in parent.orelse: - return parent.test, False - if isinstance(parent, ast.IfExp): - if child is parent.body: - return parent.test, True - if child is parent.orelse: - return parent.test, False - return None - - -def possible_boolean_results(node: ast.AST, facts: Facts, policy: str, value: bool) -> set[bool]: - if f"policy-exact:{policy}" in facts.infer(node): - key = expression_key(node) - if key or isinstance(node, ast.Name): - return {value} - if isinstance(node, ast.Constant) and isinstance(node.value, bool): - return {node.value} - if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): - return {not item for item in possible_boolean_results(node.operand, facts, policy, value)} - if isinstance(node, ast.BoolOp): - child_results = [ - possible_boolean_results(item, facts, policy, value) for item in node.values - ] - outcomes = {False, True} - if isinstance(node.op, ast.And): - outcomes = {all(values) for values in _boolean_product(child_results)} - elif isinstance(node.op, ast.Or): - outcomes = {any(values) for values in _boolean_product(child_results)} - return outcomes - if ( - isinstance(node, ast.Compare) - and len(node.ops) == 1 - and len(node.comparators) == 1 - and isinstance(node.ops[0], ast.Eq | ast.NotEq | ast.Is | ast.IsNot) - ): - left = possible_boolean_results(node.left, facts, policy, value) - right = possible_boolean_results(node.comparators[0], facts, policy, value) - equality = isinstance(node.ops[0], ast.Eq | ast.Is) - return {(a == b) if equality else (a != b) for a in left for b in right} - return {False, True} - - -def _boolean_product(values: Sequence[set[bool]]) -> list[tuple[bool, ...]]: - products: list[tuple[bool, ...]] = [()] - for options in values: - products = [prefix + (option,) for prefix in products for option in options] - return products - - -def block_always_exits(statements: Sequence[ast.stmt]) -> bool: - if not statements: - return False - final = statements[-1] - if isinstance(final, ast.Return | ast.Raise | ast.Break | ast.Continue): - return True - if isinstance(final, ast.If): - return block_always_exits(final.body) and block_always_exits(final.orelse) - return False - - -def continuation_guards( - parent: ast.AST, - child: ast.AST, - facts: Facts, -) -> set[str]: - guards: set[str] = set() - for field_name in ("body", "orelse", "finalbody"): - block = getattr(parent, field_name, None) - if not isinstance(block, list) or child not in block: - continue - child_index = block.index(child) - for statement in block[:child_index]: - if not isinstance(statement, ast.If): - continue - body_exits = block_always_exits(statement.body) - else_exits = block_always_exits(statement.orelse) - if body_exits == else_exits: - continue - continuing_branch = not body_exits - for policy in ("model", "tool"): - has_policy = any( - f"policy:{policy}" in facts.infer(candidate) - for candidate in ast.walk(statement.test) - ) - if has_policy and continuing_branch not in possible_boolean_results( - statement.test, facts, policy, True - ): - guards.add(policy) - return guards - - -def guarded_policy(node: ast.AST, parents: Mapping[ast.AST, ast.AST], facts: Facts) -> str: - guards: set[str] = set() - child = node - current = parents.get(node) - while current is not None: - branch = branch_for_child(current, child) - if branch: - condition, branch_value = branch - for policy in ("model", "tool"): - if any( - f"policy:{policy}" in facts.infer(candidate) - for candidate in ast.walk(condition) - ) and branch_value not in possible_boolean_results(condition, facts, policy, True): - guards.add(policy) - guards.update(continuation_guards(current, child, facts)) - if isinstance(current, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda): - break - child = current - current = parents.get(current) - if guards == {"model", "tool"}: - return "model+tool-guard" - if "model" in guards: - return "model-guard" - if "tool" in guards: - return "tool-guard" - return "none" - - -def call_site_context(node: ast.AST, parents: Mapping[ast.AST, ast.AST], source: str) -> str: - parts: list[str] = [] - child = node - current = parents.get(node) - while current is not None: - if isinstance(current, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): - parts.append(f"{type(current).__name__}:{current.name}") - elif isinstance(current, ast.If): - branch = ( - "body" if child in current.body else "else" if child in current.orelse else "test" - ) - parts.append(f"if:{normalize_node(current.test, source)}:{branch}") - elif isinstance(current, ast.IfExp): - branch = ( - "body" if child is current.body else "else" if child is current.orelse else "test" - ) - parts.append(f"ifexp:{normalize_node(current.test, source)}:{branch}") - elif isinstance(current, ast.Try): - if child in current.body: - branch = "try" - elif child in current.handlers: - handler_index = next( - index for index, handler in enumerate(current.handlers) if handler is child - ) - branch = f"handler:{handler_index}" - elif child in current.orelse: - branch = "else" - elif child in current.finalbody: - branch = "finally" - else: - branch = "nested" - parts.append(f"try:{branch}") - elif isinstance(current, ast.ExceptHandler): - parts.append( - f"except:{normalize_node(current.type, source) if current.type else 'bare'}" - ) - elif isinstance(current, ast.match_case): - pattern = normalize_node(current.pattern, source) - guard = normalize_node(current.guard, source) if current.guard else "" - parts.append(f"case:{pattern}:{guard}") - elif isinstance(current, ast.Match): - parts.append(f"match:{normalize_node(current.subject, source)}") - elif isinstance(current, ast.For | ast.AsyncFor | ast.While): - branch = ( - "body" if child in current.body else "else" if child in current.orelse else "header" - ) - parts.append(f"{type(current).__name__}:{branch}") - elif isinstance(current, ast.Call): - if child in current.args: - argument_index = next( - index for index, argument in enumerate(current.args) if argument is child - ) - parts.append(f"callback:{normalize_node(current.func, source)}:{argument_index}") - elif isinstance(child, ast.keyword) and child in current.keywords: - keyword_name = child.arg if child.arg is not None else "**" - parts.append( - f"callback:{normalize_node(current.func, source)}:keyword:{keyword_name}" - ) - child = current - current = parents.get(current) - return ">".join(reversed(parts)) or "" - - -def enclosing_catch_names(node: ast.AST, parents: Mapping[ast.AST, ast.AST]) -> list[str]: - names: list[str] = [] - current = parents.get(node) - while current is not None: - if isinstance(current, ast.ExceptHandler) and current.name: - names.append(current.name) - current = parents.get(current) - return names - - -def call_shape(call: ast.Call, method: str) -> str: - if any(keyword.arg is None for keyword in call.keywords): - return "payload" - keywords = {keyword.arg: keyword.value for keyword in call.keywords if keyword.arg is not None} - exc_info = keywords.get("exc_info") - has_exc_info = exc_info is not None and not ( - isinstance(exc_info, ast.Constant) and exc_info.value in (None, False) - ) - if method in {"exception", "traceback.print_exc"} or has_exc_info: - return "exception-payload" - if len(call.args) > 1 or "extra" in keywords: - return "payload" - if method in SENSITIVE_HELPER_METHODS and any( - name not in {"logger", "target_logger", "message", "msg"} for name in keywords - ): - return "payload" - message = call.args[0] if call.args else keywords.get("msg", keywords.get("message")) - if message is None: - if call.keywords: - return "dynamic-message" - return "static-message" - if isinstance(message, ast.Constant) and isinstance(message.value, str): - return "static-message" - return "dynamic-message" - - -def call_shape_with_partial(call: ast.Call, method: str, callee_facts: set[str]) -> str: - shape = call_shape(call, method) - bound_shapes = { - fact.split(":", 1)[1] for fact in callee_facts if fact.startswith("partial-shape:") - } - if not bound_shapes or bound_shapes == {"empty"}: - return shape - if "exception-payload" in bound_shapes or shape == "exception-payload": - return "exception-payload" - if "payload" in bound_shapes or shape == "payload": - return "payload" - - has_call_arguments = bool(call.args or call.keywords) - if "dynamic-message" in bound_shapes: - return "payload" if has_call_arguments else "dynamic-message" - if "static-message" in bound_shapes: - return "payload" if has_call_arguments else "static-message" - return shape - - -def is_standard_file_descriptor_write(call: ast.Call, method: str) -> bool: - if method != "os.write": - return True - if not call.args: - return False - descriptor = call.args[0] - return ( - isinstance(descriptor, ast.Constant) - and type(descriptor.value) is int - and descriptor.value in {1, 2} - ) - - -def is_unknown_logging_callback(argument: ast.AST, keyword: str | None) -> bool: - if not isinstance(argument, ast.Attribute) or argument.attr not in LOG_METHODS: - return False - receiver = expression_key(argument.value) - receiver_name = receiver.rsplit(".", 1)[-1].lower() if receiver else "" - logger_like_receiver = receiver_name in {"log", "logger"} or receiver_name.endswith( - ("_log", "_logger") - ) - callback_keyword = keyword is not None and ( - keyword.startswith("on_") - or keyword.endswith(("_callback", "_handler")) - or keyword in {"callback", "handler"} - ) - return logger_like_receiver or callback_keyword - - -def signals_for(text: str) -> list[str]: - normalized = text.lower() - groups = [ - ("model", r"\b(model|response|request|completion|llm|realtime event)\b"), - ( - "tool", - r"\b(tool|function call|arguments|computer action|shell action|apply_patch|mcp)\b", - ), - ("error", r"\b(error|err|exception|failure|failed|reason|traceback)\b"), - ("payload", r"\b(input|output|item|event|payload|data|trace|span|extra)\b"), - ] - return [name for name, pattern in groups if re.search(pattern, normalized)] - - -def inventory_source(source: str, file_path: str = "fixture.py") -> list[Finding]: - normalized_path = normalize_path(file_path) - tree = ast.parse(source, filename=normalized_path) - parents = make_parent_map(tree) - facts = Facts(tree, normalized_path) - findings: list[Finding] = [] - recorded_nodes: set[tuple[int, str, str]] = set() - - def record( - node: ast.AST, - call_text: str, - kind: str, - confidence: str, - method: str, - shape: str, - policy: str, - catch_value: str | None, - ) -> None: - key = (id(node), kind, method) - if key in recorded_nodes: - return - recorded_nodes.add(key) - context = call_site_context(node, parents, source) - group = hashlib.sha256(f"{normalized_path}\0{context}\0{call_text}".encode()).hexdigest()[ - :12 - ] - findings.append( - Finding( - group_fingerprint=group, - fingerprint=group, - file=normalized_path, - line=getattr(node, "lineno", 1), - column=getattr(node, "col_offset", 0) + 1, - kind=kind, - confidence=confidence, - method=method, - shape=shape, - policy=policy, - catch_value=catch_value, - context=context, - signals=signals_for(call_text), - call=call_text, - ) - ) - - for node in ast.walk(tree): - if not isinstance(node, ast.Call): - continue - call_text = normalize_node(node, source) - callee_facts = facts.infer(node.func) - sink_facts = sorted( - fact for fact in callee_facts if fact.startswith(("method:", "helper:")) - ) - catch_names = enclosing_catch_names(node, parents) - referenced = [ - name - for name in catch_names - if any( - node_references_name(value, name) - for value in [*node.args, *(kw.value for kw in node.keywords)] - ) - ] - catch_value = ", ".join(referenced) or None - policy = guarded_policy(node, parents, facts) - - if sink_facts: - for fact in sink_facts: - category, sink_kind, sink_method = _split_sink_fact(fact) - if category == "helper": - helper_policy = f"{sink_kind}-helper" - record( - node, - call_text, - "sensitive-helper", - "confirmed", - sink_method, - call_shape(node, sink_method), - helper_policy, - catch_value, - ) - else: - if sink_kind == "logger": - shape = call_shape_with_partial(node, sink_method, callee_facts) - if shape == "exception-payload" and catch_value is None and catch_names: - catch_value = "active exception" - record( - node, - call_text, - "logger", - "confirmed", - sink_method, - shape, - policy, - catch_value, - ) - else: - if not is_standard_file_descriptor_write(node, sink_method): - continue - record( - node, - call_text, - "raw-output", - "confirmed", - sink_method, - call_shape_with_partial(node, sink_method, callee_facts), - policy, - catch_value, - ) - elif isinstance(node.func, ast.Attribute) and node.func.attr in LOG_METHODS: - shape = call_shape(node, node.func.attr) - if shape == "exception-payload" and catch_value is None and catch_names: - catch_value = "active exception" - record( - node, - call_text, - "logging-candidate", - "unknown", - node.func.attr, - shape, - policy, - catch_value, - ) - elif isinstance(node.func, ast.Name) and node.func.id in { - "log_model_action_error", - "log_tool_action_error", - }: - record( - node, - call_text, - "helper-candidate", - "unknown", - node.func.id, - call_shape(node, node.func.id), - "none", - catch_value, - ) - - if not _is_partial_call(callee_facts): - callback_arguments = [ - *((argument, None) for argument in node.args), - *( - (keyword.value, keyword.arg) - for keyword in node.keywords - if keyword.arg is not None - ), - ] - for argument, keyword in callback_arguments: - method_facts = { - fact for fact in facts.infer(argument) if fact.startswith("method:") - } - for fact in method_facts: - _, sink_kind, sink_method = fact.split(":", 2) - record( - argument, - normalize_node(argument, source), - "logger-callback" if sink_kind == "logger" else "raw-output-callback", - "confirmed", - sink_method, - "dynamic-message", - guarded_policy(argument, parents, facts) - if sink_kind == "logger" - else "none", - "callback value", - ) - if ( - not method_facts - and is_unknown_logging_callback(argument, keyword) - and isinstance(argument, ast.Attribute) - ): - record( - argument, - normalize_node(argument, source), - "logging-candidate-callback", - "unknown", - argument.attr, - "dynamic-message", - guarded_policy(argument, parents, facts), - "callback value", - ) - - findings.sort(key=lambda finding: (finding.file, finding.line, finding.column, finding.kind)) - counts = Counter(finding.group_fingerprint for finding in findings) - indexes: dict[str, int] = defaultdict(int) - for finding in findings: - index = indexes[finding.group_fingerprint] - indexes[finding.group_fingerprint] += 1 - count = counts[finding.group_fingerprint] - finding.site_index = index - finding.group_count = count - finding.identity_quality = "duplicate" if count > 1 else "unique" - finding.fingerprint = ( - finding.group_fingerprint if count == 1 else f"{finding.group_fingerprint}:{index}" - ) - return findings - - -def _split_sink_fact(fact: str) -> tuple[str, str, str]: - if fact.startswith("helper:"): - _, policy, method = fact.split(":", 2) - return "helper", policy, method - _, sink_kind, sink_method = fact.split(":", 2) - return "method", sink_kind, sink_method - - -def _is_partial_call(callee_facts: set[str]) -> bool: - return "factory:partial" in callee_facts - - -def summarize(findings: Sequence[Finding]) -> dict[str, int]: - dynamic = [finding for finding in findings if finding.shape != "static-message"] - duplicate_groups = { - finding.group_fingerprint for finding in findings if finding.identity_quality == "duplicate" - } - return { - "total": len(findings), - "dynamic": len(dynamic), - "unclassifiedDynamic": sum(finding.policy == "none" for finding in dynamic), - "catchValueLogs": sum(finding.catch_value is not None for finding in findings), - "unclassifiedCatchValueLogs": sum( - finding.catch_value is not None and finding.policy == "none" for finding in findings - ), - "rawOutputCalls": sum(finding.kind.startswith("raw-output") for finding in findings), - "unknownReceiverCalls": sum(finding.confidence == "unknown" for finding in findings), - "duplicateGroups": len(duplicate_groups), - } - - -def compare_findings( - baseline: Sequence[Mapping[str, Any]], findings: Sequence[Finding] -) -> dict[str, Any]: - before = Counter(str(item["group_fingerprint"]) for item in baseline) - after = Counter(finding.group_fingerprint for finding in findings) - before_classifications = _comparison_classifications(baseline) - after_classifications = _comparison_classifications(findings) - return { - "new": sorted(after.keys() - before.keys()), - "removed": sorted(before.keys() - after.keys()), - "countChanged": [ - {"group_fingerprint": key, "before": before[key], "after": after[key]} - for key in sorted(before.keys() & after.keys()) - if before[key] != after[key] - ], - "classificationChanged": [ - { - "group_fingerprint": key, - "before": _render_classification_counts(before_classifications[key]), - "after": _render_classification_counts(after_classifications[key]), - } - for key in sorted(before.keys() & after.keys()) - if before_classifications[key] != after_classifications[key] - ], - } - - -def _comparison_classifications( - findings: Sequence[Mapping[str, Any] | Finding], -) -> dict[str, Counter[tuple[Any, ...]]]: - result: dict[str, Counter[tuple[Any, ...]]] = defaultdict(Counter) - for finding in findings: - if isinstance(finding, Finding): - group = finding.group_fingerprint - classification = tuple( - getattr(finding, field) for field in COMPARISON_CLASSIFICATION_FIELDS - ) - else: - group = str(finding["group_fingerprint"]) - classification = tuple(finding.get(field) for field in COMPARISON_CLASSIFICATION_FIELDS) - result[group][classification] += 1 - return result - - -def _render_classification_counts( - counts: Counter[tuple[Any, ...]], -) -> list[dict[str, Any]]: - return [ - { - **dict(zip(COMPARISON_CLASSIFICATION_FIELDS, classification, strict=True)), - "count": count, - } - for classification, count in sorted(counts.items(), key=lambda item: repr(item[0])) - ] - - -def validate_review_ledger( - ledger: Mapping[str, Any], findings: Sequence[Finding] -) -> dict[str, Any]: - reviews = ledger.get("reviews") - if not isinstance(reviews, list): - return {"valid": False, "errors": ["Review ledger must contain a reviews list."]} - dynamic_counts = Counter( - finding.group_fingerprint for finding in findings if finding.shape != "static-message" - ) - review_groups = [ - str(review.get("group_fingerprint")) - for review in reviews - if isinstance(review, dict) and review.get("group_fingerprint") - ] - duplicate_review_groups = sorted( - group for group, count in Counter(review_groups).items() if count > 1 - ) - errors = [f"Duplicate review entries for {group}" for group in duplicate_review_groups] - review_by_group: dict[str, Mapping[str, Any]] = {} - for review in reviews: - if not isinstance(review, dict) or not review.get("group_fingerprint"): - continue - review_by_group.setdefault(str(review["group_fingerprint"]), review) - for group, count in sorted(dynamic_counts.items()): - review = review_by_group.get(group) - if review is None: - errors.append(f"Missing review for dynamic group {group}.") - continue - if review.get("disposition") not in DISPOSITIONS: - errors.append(f"Invalid disposition for {group}.") - evidence = review.get("evidence") - if not isinstance(evidence, str) or not evidence.strip(): - errors.append(f"Missing evidence for {group}.") - action = review.get("action") - if not isinstance(action, str) or not action.strip(): - errors.append(f"Missing action for {group}.") - if review.get("group_count") != count: - actual = review.get("group_count") - errors.append(f"Group count mismatch for {group}: expected {count}, got {actual}.") - extras = sorted(review_by_group.keys() - dynamic_counts.keys()) - if extras: - errors.append(f"Review ledger contains stale groups: {', '.join(extras)}.") - return {"valid": not errors, "errors": errors} diff --git a/.agents/skills/sensitive-logging-audit/scripts/inventory_logging.py b/.agents/skills/sensitive-logging-audit/scripts/inventory_logging.py index 4b3bfc916e..da2bea1f20 100644 --- a/.agents/skills/sensitive-logging-audit/scripts/inventory_logging.py +++ b/.agents/skills/sensitive-logging-audit/scripts/inventory_logging.py @@ -2,141 +2,345 @@ from __future__ import annotations import argparse +import ast +import hashlib import json +import re import sys -from collections.abc import Sequence +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import asdict, dataclass from pathlib import Path from typing import Any -from _inventory import ( - Finding, - collect_source_files, - compare_findings, - inventory_source, - summarize, - validate_review_ledger, -) +LOG_METHODS = { + "critical", + "debug", + "error", + "exception", + "fatal", + "info", + "log", + "warn", + "warning", +} +POLICY_HELPERS = { + "log_model_action_debug", + "log_model_action_error", + "log_model_action_warning", + "log_model_and_tool_action_debug", + "log_model_and_tool_action_error", + "log_model_and_tool_action_warning", + "log_tool_action_debug", + "log_tool_action_error", + "log_tool_action_warning", +} +RAW_OUTPUT_METHODS = { + "pp", + "pprint", + "print", + "print_exc", + "print_exception", + "warn", + "warn_explicit", + "write", + "writelines", +} +CALLBACK_KEYWORDS = {"callback", "handler"} + + +@dataclass(frozen=True) +class Candidate: + fingerprint: str + file: str + line: int + column: int + kind: str + method: str + context: str + call: str + reason: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def normalize_path(path: str | Path) -> str: + return str(path).replace("\\", "/") + + +def collect_source_files(roots: Sequence[str | Path]) -> list[Path]: + files: set[Path] = set() + for root_value in roots: + root = Path(root_value).resolve() + if root.is_file(): + if root.suffix == ".py": + files.add(root) + continue + if not root.is_dir(): + raise FileNotFoundError(f"Inventory root does not exist: {root_value}") + for path in root.rglob("*.py"): + relative_parts = path.relative_to(root).parts + if any(part.startswith(".") or part == "__pycache__" for part in relative_parts): + continue + files.add(path.resolve()) + return sorted(files) + + +def normalize_node(node: ast.AST, source: str) -> str: + segment = ast.get_source_segment(source, node) + if segment is None: + segment = ast.dump(node, annotate_fields=True, include_attributes=False) + return re.sub(r"\s+", " ", segment).strip() + + +def dotted_name(node: ast.AST) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + receiver = dotted_name(node.value) + return f"{receiver}.{node.attr}" if receiver else node.attr + return None + + +def terminal_name(node: ast.AST) -> str | None: + name = dotted_name(node) + return name.rsplit(".", 1)[-1] if name else None + + +def make_parent_map(tree: ast.AST) -> dict[ast.AST, ast.AST]: + return {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)} + + +def scope_context(node: ast.AST, parents: Mapping[ast.AST, ast.AST]) -> str: + parts: list[str] = [] + current = parents.get(node) + while current is not None: + if isinstance(current, ast.ClassDef): + parts.append(f"class:{current.name}") + elif isinstance(current, ast.FunctionDef | ast.AsyncFunctionDef): + parts.append(f"function:{current.name}") + elif isinstance(current, ast.Lambda): + parts.append("lambda") + current = parents.get(current) + return ">".join(reversed(parts)) or "" + + +def callback_arguments(call: ast.Call) -> Iterable[tuple[ast.AST, str | None]]: + yield from ((argument, None) for argument in call.args) + yield from ( + (keyword.value, keyword.arg) for keyword in call.keywords if keyword.arg is not None + ) + + +def looks_like_callback(node: ast.AST, keyword: str | None) -> bool: + method = terminal_name(node) + if method not in LOG_METHODS: + return False + if keyword is not None and ( + keyword.startswith("on_") + or keyword.endswith(("_callback", "_handler")) + or keyword in CALLBACK_KEYWORDS + ): + return True + if not isinstance(node, ast.Attribute): + return False + receiver = dotted_name(node.value) + receiver_name = receiver.rsplit(".", 1)[-1].lower() if receiver else "" + return receiver_name in {"log", "logger"} or receiver_name.endswith(("_log", "_logger")) + + +def selected_getattr_method(call: ast.Call) -> str | None: + if terminal_name(call.func) != "getattr" or len(call.args) < 2: + return None + attribute = call.args[1] + if not isinstance(attribute, ast.Constant) or not isinstance(attribute.value, str): + return None + if attribute.value in LOG_METHODS | RAW_OUTPUT_METHODS: + return attribute.value + return None + + +def classify_call(call: ast.Call) -> tuple[str, str, str] | None: + qualified_method = dotted_name(call.func) + method = terminal_name(call.func) + if method in POLICY_HELPERS: + return ( + "policy-helper-call", + method, + "Known redaction helper; review the caller's data classification and fixed message.", + ) + if method in LOG_METHODS and not ( + method == "warn" and qualified_method in {"warn", "warnings.warn"} + ): + return ( + "logging-call-candidate", + method, + "Logging-like method name; inspect the receiver and every attached value.", + ) + if method in RAW_OUTPUT_METHODS: + return ( + "raw-output-call-candidate", + method, + "Direct-output method name; verify its destination and whether values " + "can be sensitive.", + ) + selected = selected_getattr_method(call) + if selected is not None: + return ( + "getattr-sink-candidate", + selected, + "Constant getattr selects an output-like method; trace the receiver and later uses.", + ) + return None + + +def inventory_source(source: str, file_path: str = "fixture.py") -> list[Candidate]: + normalized_path = normalize_path(file_path) + tree = ast.parse(source, filename=normalized_path) + parents = make_parent_map(tree) + candidates: list[Candidate] = [] + recorded: set[tuple[int, str, str]] = set() + + def record(node: ast.AST, kind: str, method: str, call: str, reason: str) -> None: + key = (id(node), kind, method) + if key in recorded: + return + recorded.add(key) + line = getattr(node, "lineno", 1) + column = getattr(node, "col_offset", 0) + 1 + context = scope_context(node, parents) + fingerprint = hashlib.sha256( + f"{normalized_path}\0{line}\0{column}\0{kind}\0{method}\0{call}".encode() + ).hexdigest()[:12] + candidates.append( + Candidate( + fingerprint=fingerprint, + file=normalized_path, + line=line, + column=column, + kind=kind, + method=method, + context=context, + call=call, + reason=reason, + ) + ) + + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + classification = classify_call(node) + if classification is not None: + kind, method, reason = classification + record(node, kind, method, normalize_node(node, source), reason) + for argument, keyword in callback_arguments(node): + if not looks_like_callback(argument, keyword): + continue + method = terminal_name(argument) + if method is None: + continue + record( + argument, + "logging-callback-candidate", + method, + normalize_node(argument, source), + "Logging-like callable passed to a callback-shaped argument; inspect " + "registration and payloads.", + ) + + candidates.sort(key=lambda item: (item.file, item.line, item.column, item.kind, item.method)) + return candidates + + +def summarize(candidates: Sequence[Candidate]) -> dict[str, int]: + kinds = Counter(candidate.kind for candidate in candidates) + return { + "totalCandidates": len(candidates), + "loggingCalls": kinds["logging-call-candidate"], + "rawOutputCalls": kinds["raw-output-call-candidate"], + "policyHelperCalls": kinds["policy-helper-call"], + "getattrSelections": kinds["getattr-sink-candidate"], + "callbackReferences": kinds["logging-callback-candidate"], + } def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description=( - "Inventory Python runtime logging and raw-output sinks for sensitive-data review." + "Collect syntactic Python logging and raw-output candidates for manual review." ) ) parser.add_argument("roots", nargs="*", default=["src/agents"]) parser.add_argument("--format", choices=("json", "markdown"), default="markdown") parser.add_argument("--summary-only", action="store_true") parser.add_argument("--output", type=Path) - parser.add_argument("--compare", type=Path, metavar="BASELINE_JSON") - parser.add_argument("--validate-review", type=Path, metavar="REVIEW_JSON") return parser.parse_args(argv) -def build_report(args: argparse.Namespace) -> tuple[dict[str, Any], bool]: +def build_report(args: argparse.Namespace) -> dict[str, Any]: cwd = Path.cwd().resolve() - paths = collect_source_files(args.roots) - findings: list[Finding] = [] - for path in paths: + candidates: list[Candidate] = [] + for path in collect_source_files(args.roots): try: display_path = path.relative_to(cwd) except ValueError: display_path = path source = path.read_text(encoding="utf-8") try: - findings.extend(inventory_source(source, str(display_path))) + candidates.extend(inventory_source(source, str(display_path))) except SyntaxError as error: raise SyntaxError( f"Failed to parse {display_path}:{error.lineno}: {error.msg}" ) from error report: dict[str, Any] = { - "summary": summarize(findings), + "contract": ( + "Syntactic candidates only. Manual review and runtime tests are required; " + "absence from this report is not proof of safety." + ), + "summary": summarize(candidates), } if not args.summary_only: - report["findings"] = [finding.to_dict() for finding in findings] - - valid = True - if args.compare: - baseline = json.loads(args.compare.read_text(encoding="utf-8")) - baseline_findings = baseline.get("findings") - if not isinstance(baseline_findings, list): - raise ValueError("Baseline JSON must contain a findings list.") - report["comparison"] = compare_findings(baseline_findings, findings) + report["candidates"] = [candidate.to_dict() for candidate in candidates] + return report - if args.validate_review: - ledger = json.loads(args.validate_review.read_text(encoding="utf-8")) - if not isinstance(ledger, dict): - raise ValueError("Review ledger must be a JSON object.") - validation = validate_review_ledger(ledger, findings) - report["reviewValidation"] = validation - valid = bool(validation["valid"]) - return report, valid - - -def render_markdown(report: dict[str, Any], summary_only: bool) -> str: +def render_markdown(report: Mapping[str, Any], summary_only: bool) -> str: summary = report["summary"] lines = [ - "# Sensitive logging inventory", + "# Sensitive logging candidates", + "", + f"> {report['contract']}", "", - f"- Total output calls: {summary['total']}", - f"- Dynamic calls: {summary['dynamic']}", - f"- Dynamic calls without an explicit model/tool policy: {summary['unclassifiedDynamic']}", - f"- Calls that log a caught or active exception: {summary['catchValueLogs']}", - f"- Unclassified caught-value calls: {summary['unclassifiedCatchValueLogs']}", - f"- Raw output calls: {summary['rawOutputCalls']}", - f"- Unknown receiver calls: {summary['unknownReceiverCalls']}", - f"- Duplicate fingerprint groups: {summary['duplicateGroups']}", + f"- Total candidates: {summary['totalCandidates']}", + f"- Logging calls: {summary['loggingCalls']}", + f"- Raw-output calls: {summary['rawOutputCalls']}", + f"- Policy-helper calls: {summary['policyHelperCalls']}", + f"- Constant getattr selections: {summary['getattrSelections']}", + f"- Callback references: {summary['callbackReferences']}", ] if not summary_only: lines.extend( [ "", - "| Location | Kind | Shape | Policy | Catch value | Fingerprint |", - "| --- | --- | --- | --- | --- | --- |", + "| Location | Kind | Method | Context | Fingerprint |", + "| --- | --- | --- | --- | --- |", ] ) - for finding in report.get("findings", []): - location = f"{finding['file']}:{finding['line']}" - sink = f"{finding['kind']}.{finding['method']}" + for candidate in report.get("candidates", []): + location = f"{candidate['file']}:{candidate['line']}" lines.append( - f"| {location} | {sink} | {finding['shape']} | {finding['policy']} | " - f"{finding['catch_value'] or ''} | {finding['fingerprint']} |" + f"| {location} | {candidate['kind']} | {candidate['method']} | " + f"{candidate['context']} | {candidate['fingerprint']} |" ) - - comparison = report.get("comparison") - if comparison is not None: - lines.extend( - [ - "", - "## Baseline comparison", - "", - f"- New groups: {len(comparison['new'])}", - f"- Removed groups: {len(comparison['removed'])}", - f"- Count-changed groups: {len(comparison['countChanged'])}", - f"- Classification-changed groups: {len(comparison['classificationChanged'])}", - ] - ) - - validation = report.get("reviewValidation") - if validation is not None: - lines.extend( - [ - "", - "## Review ledger validation", - "", - f"- Valid: {'yes' if validation['valid'] else 'no'}", - ] - ) - lines.extend(f"- {error}" for error in validation["errors"]) return "\n".join(lines) + "\n" def main(argv: Sequence[str] | None = None) -> int: args = parse_args(argv) try: - report, valid = build_report(args) + report = build_report(args) output = ( json.dumps(report, indent=2, sort_keys=True) + "\n" if args.format == "json" @@ -146,9 +350,9 @@ def main(argv: Sequence[str] | None = None) -> int: args.output.write_text(output, encoding="utf-8") else: sys.stdout.write(output) - return 0 if valid else 1 + return 0 except (OSError, SyntaxError, ValueError, json.JSONDecodeError) as error: - print(f"Sensitive logging inventory failed: {error}", file=sys.stderr) + print(f"Sensitive logging candidate collection failed: {error}", file=sys.stderr) return 1 diff --git a/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py index 38eadab817..44a6b42a1a 100644 --- a/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py +++ b/.agents/skills/sensitive-logging-audit/scripts/test_inventory.py @@ -4,1129 +4,191 @@ from pathlib import Path from tempfile import TemporaryDirectory -from _inventory import ( - collect_source_files, - compare_findings, - inventory_source, - summarize, - validate_review_ledger, -) +from inventory_logging import collect_source_files, inventory_source, summarize class InventoryTests(unittest.TestCase): - def test_inventories_static_dynamic_and_exception_logger_calls(self) -> None: - findings = inventory_source( + def test_collects_direct_logging_calls_without_certifying_receivers(self) -> None: + candidates = inventory_source( """ -from ..logger import logger +from logging import error logger.debug("ready") -logger.warning(f"Request {request_id} failed") -logger.error("Request failed: %s", error) -logger.exception("Unhandled failure") -""", - "src/agents/run_internal/fixture.py", - ) - - self.assertEqual( - [(item.method, item.shape, item.confidence) for item in findings], - [ - ("debug", "static-message", "confirmed"), - ("warning", "dynamic-message", "confirmed"), - ("error", "payload", "confirmed"), - ("exception", "exception-payload", "confirmed"), - ], - ) - - def test_inventories_keyword_messages_and_helper_payloads(self) -> None: - findings = inventory_source( - """ -from agents.logger import logger, log_tool_action_error - -logger.error(msg=secret) -logger.info(msg="ready") -log_tool_action_error(target_logger=logger, message=secret, exc=error) -""" - ) - - self.assertEqual( - [(item.kind, item.shape) for item in findings], - [ - ("logger", "dynamic-message"), - ("logger", "static-message"), - ("sensitive-helper", "payload"), - ], - ) - self.assertEqual(summarize(findings)["dynamic"], 2) - - def test_treats_expanded_logging_keywords_as_payloads(self) -> None: - findings = inventory_source( - """ -from agents.logger import logger - -logger.error("failed", **{"exc_info": error}) -""" - ) - - self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], - [("logger", "error", "payload")], - ) - - def test_classifies_no_argument_print_exc_as_exception_payload(self) -> None: - findings = inventory_source( - """ -import traceback - -try: - run() -except Exception: - traceback.print_exc() -""" - ) - - self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], - [("raw-output", "traceback.print_exc", "exception-payload")], - ) - - def test_recognizes_logger_factories_aliases_methods_and_partial(self) -> None: - findings = inventory_source( - """ -import functools -import logging - -base = logging.getLogger(__name__) -alias = base -emit = alias.error -emit("failed", secret) -handler = functools.partial(alias.warning, "failed: %s") -handler(secret) -schedule(alias.info) -""" - ) - - self.assertEqual( - [(item.kind, item.method) for item in findings], - [ - ("logger", "error"), - ("logger", "warning"), - ("logger-callback", "info"), - ], - ) - - def test_imported_partial_is_resolved(self) -> None: - findings = inventory_source( - """ -from functools import partial -from agents.logger import logger - -report = partial(logger.error, "failed: %s") -report(secret) -""" - ) - - self.assertEqual([(item.kind, item.method) for item in findings], [("logger", "error")]) - - def test_recognizes_directly_constructed_logger_aliases(self) -> None: - findings = inventory_source( - """ -import logging -from logging import Logger - -direct = Logger("direct") -direct_emit = direct.error -direct_emit(secret) -qualified = logging.Logger("qualified") -qualified_emit = qualified.warning -qualified_emit(secret) -""" - ) - - self.assertEqual( - [(item.kind, item.method) for item in findings], - [("logger", "error"), ("logger", "warning")], - ) - - def test_recognizes_logger_adapter_sources(self) -> None: - findings = inventory_source( - """ -import logging -from logging import LoggerAdapter - -base = logging.getLogger("base") -direct = LoggerAdapter(base, {}) -direct_emit = direct.error -direct_emit(secret) -qualified = logging.LoggerAdapter(base, {}) -qualified_emit = qualified.warning -qualified_emit(secret) -""" - ) - - self.assertEqual( - [(item.kind, item.method) for item in findings], - [("logger", "error"), ("logger", "warning")], - ) - - def test_inventories_print_through_builtins_imports(self) -> None: - findings = inventory_source( - """ -import builtins -from builtins import print as emit - -builtins.print(secret) -emit(secret) -""" - ) - - self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], - [ - ("raw-output", "builtins.print", "dynamic-message"), - ("raw-output", "builtins.print", "dynamic-message"), - ], - ) - - def test_resolves_helpers_through_qualified_sdk_logger_imports(self) -> None: - findings = inventory_source( - """ -import agents.logger -import agents.logger as audit_logger -import agents.tracing.logger -from agents import logger as logger_module -from agents.logger import logger - -agents.logger.log_model_action_error(logger, secret, error) -audit_logger.log_tool_action_error(logger, secret, error) -logger_module.log_model_and_tool_action_warning(logger, secret, error) -agents.tracing.logger.logger.error(secret) +logger.error("failed: %s", secret) +error(secret) +task.exception() """ ) self.assertEqual( - [(item.kind, item.method, item.policy) for item in findings], + [(item.kind, item.method) for item in candidates], [ - ("sensitive-helper", "log_model_action_error", "model-helper"), - ("sensitive-helper", "log_tool_action_error", "tool-helper"), - ( - "sensitive-helper", - "log_model_and_tool_action_warning", - "model+tool-helper", - ), - ("logger", "error", "none"), + ("logging-call-candidate", "debug"), + ("logging-call-candidate", "error"), + ("logging-call-candidate", "error"), + ("logging-call-candidate", "exception"), ], ) - def test_partial_preserves_bound_payload_shape(self) -> None: - findings = inventory_source( + def test_collects_policy_helpers_without_claiming_their_callers_are_safe(self) -> None: + candidates = inventory_source( """ -from functools import partial -from agents.logger import logger +from agents.logger import log_model_action_error -emit = partial(logger.error, secret) -emit() -formatted = partial(logger.warning, "failed: %s") -formatted(secret) +log_model_action_error(logger, "failed", error) +agents.logger.log_model_and_tool_action_warning(logger, "failed", error) """ ) self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], + [(item.kind, item.method) for item in candidates], [ - ("logger", "error", "dynamic-message"), - ("logger", "warning", "payload"), + ("policy-helper-call", "log_model_action_error"), + ("policy-helper-call", "log_model_and_tool_action_warning"), ], ) - def test_nested_partial_preserves_bound_payload_shape(self) -> None: - findings = inventory_source( - """ -from functools import partial -from agents.logger import logger - -first = partial(logger.error, secret) -second = partial(first) -second() -""" - ) - - self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], - [("logger", "error", "dynamic-message")], - ) - - def test_inventories_raw_output_and_unknown_receivers(self) -> None: - findings = inventory_source( + def test_collects_raw_output_method_names(self) -> None: + candidates = inventory_source( """ +import os +import pprint import sys import traceback import warnings print(secret) +pprint.pp(secret) warnings.warn(secret) -sys.stderr.write(secret) +sys.stderr.buffer.write(secret_bytes) +sys.stdout.writelines([secret]) traceback.print_exception(error) -task.exception() -""" - ) - - self.assertEqual( - [(item.kind, item.method, item.confidence) for item in findings], - [ - ("raw-output", "print", "confirmed"), - ("raw-output", "warnings.warn", "confirmed"), - ("raw-output", "stderr.write", "confirmed"), - ("raw-output", "traceback.print_exception", "confirmed"), - ("logging-candidate", "exception", "unknown"), - ], - ) - - def test_inventories_pprint_pp_aliases(self) -> None: - findings = inventory_source( - """ -import pprint -from pprint import pp as emit - -pprint.pp(secret) -emit(secret) -""" - ) - - self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], - [ - ("raw-output", "pprint.pp", "dynamic-message"), - ("raw-output", "pprint.pp", "dynamic-message"), - ], - ) - - def test_inventories_standard_file_descriptor_writes(self) -> None: - findings = inventory_source( - """ -import os -from os import write as emit - os.write(2, secret_bytes) -emit(1, secret_bytes) -os.write(3, secret_bytes) -""" - ) - - self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], - [ - ("raw-output", "os.write", "payload"), - ("raw-output", "os.write", "payload"), - ], - ) - - def test_inventories_directly_imported_output_streams(self) -> None: - findings = inventory_source( - """ -from sys import stderr, stdout as out - -stderr.write(secret) -out.write(secret) -""" - ) - - self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], - [ - ("raw-output", "stderr.write", "dynamic-message"), - ("raw-output", "stdout.write", "dynamic-message"), - ], - ) - - def test_inventories_buffered_output_streams(self) -> None: - findings = inventory_source( - """ -import sys -from sys import stdout as out - -sys.stderr.buffer.write(secret_bytes) -writer = out.buffer -writer.write(secret_bytes) -""" - ) - - self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], - [ - ("raw-output", "stderr.write", "dynamic-message"), - ("raw-output", "stdout.write", "dynamic-message"), - ], - ) - - def test_inventories_stream_writelines(self) -> None: - findings = inventory_source( - """ -import sys -from sys import stdout as out - -sys.stderr.writelines([secret]) -out.writelines([secret]) """ ) self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], + [(item.kind, item.method) for item in candidates], [ - ("raw-output", "stderr.writelines", "dynamic-message"), - ("raw-output", "stdout.writelines", "dynamic-message"), + ("raw-output-call-candidate", "print"), + ("raw-output-call-candidate", "pp"), + ("raw-output-call-candidate", "warn"), + ("raw-output-call-candidate", "write"), + ("raw-output-call-candidate", "writelines"), + ("raw-output-call-candidate", "print_exception"), + ("raw-output-call-candidate", "write"), ], ) - def test_inventories_warn_explicit_module_and_direct_imports(self) -> None: - findings = inventory_source( + def test_collects_constant_getattr_sink_selections(self) -> None: + candidates = inventory_source( """ -import warnings -from warnings import warn_explicit as emit_warning - -warnings.warn_explicit(secret, UserWarning, "x.py", 1) -emit_warning(secret, UserWarning, "x.py", 1) -""" - ) - - self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], - [ - ("raw-output", "warnings.warn_explicit", "payload"), - ("raw-output", "warnings.warn_explicit", "payload"), - ], - ) - - def test_inventories_fatal_logger_aliases(self) -> None: - findings = inventory_source( - """ -import logging -from agents.logger import logger -from logging import fatal as die - -logger.fatal(secret) -logging.fatal(secret) -die(secret) -""" - ) - - self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], - [ - ("logger", "fatal", "dynamic-message"), - ("logger", "fatal", "dynamic-message"), - ("logger", "fatal", "dynamic-message"), - ], - ) - - def test_requires_exact_policy_provenance_and_correct_polarity(self) -> None: - findings = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -if not _debug.DONT_LOG_TOOL_DATA: - logger.error("diagnostic: %s", secret) -if _debug.DONT_LOG_TOOL_DATA: - logger.error("leak: %s", secret) -if not request.DONT_LOG_TOOL_DATA: - logger.error("unrelated: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["tool-guard", "none", "none"]) - - def test_scopes_policy_facts_to_lexical_bindings(self) -> None: - findings = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -def configure(): - flag = _debug.DONT_LOG_TOOL_DATA - return flag - -def report(flag, secret): - if not flag: - logger.error("unguarded: %s", secret) -""" - ) - - self.assertEqual(findings[0].policy, "none") - - def test_preserves_direct_policy_aliases_but_not_composite_boolean_aliases(self) -> None: - findings = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -def direct(secret): - flag = _debug.DONT_LOG_TOOL_DATA - if not flag: - logger.error("guarded: %s", secret) - -def composite(enabled, secret): - guard = _debug.DONT_LOG_TOOL_DATA and enabled - if not guard: - logger.error("unguarded: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["tool-guard", "none"]) - - def test_policy_assignments_do_not_flow_backward_or_through_conditional_rebinding( - self, - ) -> None: - findings = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -def assigned_later(flag, secret): - if not flag: - logger.error("unguarded before assignment: %s", secret) - flag = _debug.DONT_LOG_TOOL_DATA - -def conditionally_reassigned(flag, condition, secret): - if condition: - flag = _debug.DONT_LOG_TOOL_DATA - if not flag: - logger.error("unguarded after conditional assignment: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["none", "none"]) - - def test_destructuring_matches_policy_facts_to_individual_values(self) -> None: - findings = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -def report(user_value, secret): - redact, enabled = _debug.DONT_LOG_TOOL_DATA, user_value - if not enabled: - logger.error("unguarded: %s", secret) - if not redact: - logger.error("guarded: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["none", "tool-guard"]) - - def test_augmented_assignment_kills_policy_alias_provenance(self) -> None: - findings = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -def report(enabled, secret): - flag = _debug.DONT_LOG_TOOL_DATA - if not flag: - logger.error("guarded: %s", secret) - flag &= enabled - if not flag: - logger.error("unguarded: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["tool-guard", "none"]) - - def test_definitions_kill_policy_alias_provenance(self) -> None: - findings = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -def function_definition(secret): - flag = _debug.DONT_LOG_TOOL_DATA - - @lambda function: False - def flag(): - pass - - if not flag: - logger.error("function definition: %s", secret) - -def class_definition(secret): - flag = _debug.DONT_LOG_TOOL_DATA - - @lambda class_: False - class flag: - pass - - if not flag: - logger.error("class definition: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["none", "none"]) - - def test_non_policy_imports_kill_policy_alias_provenance(self) -> None: - findings = inventory_source( - """ -from agents.logger import logger - -def direct_import(secret): - from agents._debug import DONT_LOG_TOOL_DATA as flag - from settings import enabled as flag - if not flag: - logger.error("direct import: %s", secret) - -def module_import(secret): - from agents._debug import DONT_LOG_TOOL_DATA as flag - import settings as flag - if not flag: - logger.error("module import: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["none", "none"]) - - def test_starred_targets_kill_policy_alias_provenance(self) -> None: - findings = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -def assignment(secret): - flag = _debug.DONT_LOG_TOOL_DATA - *flag, tail = [False] - if not flag: - logger.error("starred assignment: %s", secret) - -def loop(values, secret): - flag = _debug.DONT_LOG_TOOL_DATA - for *flag, tail in values: - if not flag: - logger.error("starred loop: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["none", "none"]) - - def test_runtime_binders_kill_imported_policy_alias_provenance(self) -> None: - findings = inventory_source( - """ -from agents.logger import logger - -def report(values, manager, secret): - from agents._debug import DONT_LOG_TOOL_DATA as flag - for flag in values: - if not flag: - logger.error("for binder: %s", secret) - with manager() as flag: - if not flag: - logger.error("with binder: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["none", "none"]) - - def test_policy_imports_are_ordered_definitions(self) -> None: - findings = inventory_source( - """ -from agents.logger import logger - -def report(flag, secret): - if not flag: - logger.error("before import: %s", secret) - from agents._debug import DONT_LOG_TOOL_DATA as flag - if not flag: - logger.error("after import: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["none", "tool-guard"]) - - def test_module_policy_import_still_reaches_nested_function(self) -> None: - findings = inventory_source( - """ -from agents._debug import DONT_LOG_TOOL_DATA as flag -from agents.logger import logger - -def report(secret): - if not flag: - logger.error("guarded: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["tool-guard"]) - - def test_match_pattern_captures_kill_policy_alias_provenance(self) -> None: - findings = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -def direct_capture(value, secret): - flag = _debug.DONT_LOG_TOOL_DATA - match value: - case flag: - if not flag: - logger.error("direct capture: %s", secret) - -def nested_capture(value, secret): - flag = _debug.DONT_LOG_TOOL_DATA - match value: - case {"enabled": flag}: - if not flag: - logger.error("nested capture: %s", secret) - -def guarded_capture(value, secret): - flag = _debug.DONT_LOG_TOOL_DATA - match value: - case flag if not flag: - logger.error("guard capture: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["none", "none", "none"]) - - def test_combines_exact_model_and_tool_guards(self) -> None: - findings = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -if not _debug.DONT_LOG_MODEL_DATA and not _debug.DONT_LOG_TOOL_DATA: - logger.error("diagnostic: %s", secret) -""" - ) - - self.assertEqual(findings[0].policy, "model+tool-guard") - - def test_recognizes_early_return_policy_guards(self) -> None: - findings = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -def report(secret): - if _debug.DONT_LOG_TOOL_DATA: - logger.debug("Tool failed") - return - logger.error("Tool failed: %s", secret) -""" - ) - - self.assertEqual([item.policy for item in findings], ["none", "tool-guard"]) - - def test_recognizes_early_continue_policy_guards(self) -> None: - findings = inventory_source( - """ -from agents import _debug - -for item in items: - if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: - print("Data is redacted") - continue - print(item) -""", - "src/agents/fixture.py", - ) - - self.assertEqual( - [(item.shape, item.policy) for item in findings], - [("static-message", "none"), ("dynamic-message", "model+tool-guard")], - ) - - def test_trusts_only_the_exact_known_helper_import(self) -> None: - trusted = inventory_source( - """ -from agents.run_internal.tool_execution import log_tool_action_error as report_failure - -report_failure("Tool failed", error) -""" - ) - untrusted = inventory_source( - """ -from analytics.logger import log_tool_action_error - -log_tool_action_error("Tool failed", error) -""" - ) - - self.assertEqual( - (trusted[0].kind, trusted[0].policy, trusted[0].confidence), - ("sensitive-helper", "tool-helper", "confirmed"), - ) - self.assertEqual( - (untrusted[0].kind, untrusted[0].policy, untrusted[0].confidence), - ("helper-candidate", "none", "unknown"), - ) - - def test_recognizes_the_known_helper_in_its_defining_module(self) -> None: - findings = inventory_source( - """ -def log_tool_action_error(message, error): - pass - -log_tool_action_error("Tool failed", error) -""", - "src/agents/run_internal/tool_execution.py", - ) - - self.assertEqual( - (findings[0].kind, findings[0].policy, findings[0].confidence), - ("sensitive-helper", "tool-helper", "confirmed"), - ) - - def test_recognizes_shared_model_and_mixed_helpers(self) -> None: - findings = inventory_source( - """ -from agents.logger import ( - log_model_action_debug, - log_model_action_error, - log_model_and_tool_action_error, - log_model_and_tool_action_warning, -) - -log_model_action_debug(logger, "Model cleanup failed", error) -log_model_action_error(logger, "Model failed", error) -log_model_and_tool_action_error(logger, "Trace failed", error) -log_model_and_tool_action_warning(logger, "Session failed", error) -""" - ) - - self.assertEqual( - [(item.method, item.policy) for item in findings], - [ - ("log_model_action_debug", "model-helper"), - ("log_model_action_error", "model-helper"), - ("log_model_and_tool_action_error", "model+tool-helper"), - ("log_model_and_tool_action_warning", "model+tool-helper"), - ], - ) - - def test_recognizes_shared_tool_helpers_at_multiple_levels(self) -> None: - findings = inventory_source( - """ -from agents.logger import log_tool_action_debug, log_tool_action_warning - -log_tool_action_debug(logger, "Cleanup cancelled", error) -log_tool_action_warning(logger, "Cleanup failed", error) -""" - ) - - self.assertEqual( - [(item.method, item.policy) for item in findings], - [ - ("log_tool_action_debug", "tool-helper"), - ("log_tool_action_warning", "tool-helper"), - ], - ) - - def test_recognizes_conditional_helper_callees(self) -> None: - findings = inventory_source( - """ -from agents.logger import logger, log_tool_action_warning - -(log_tool_action_warning if enabled else log_tool_action_warning)( - logger, - "Cleanup failed", - error, -) -""" - ) - - self.assertEqual( - [(item.kind, item.method, item.policy) for item in findings], - [ - ( - "sensitive-helper", - "log_tool_action_warning", - "tool-helper", - ) - ], - ) - - def test_resolves_constant_getattr_on_known_loggers(self) -> None: - findings = inventory_source( - """ -import builtins -from agents.logger import logger - emit = getattr(logger, "error") -emit(secret) -builtins.getattr(logger, "warning")(secret) +writer = builtins.getattr(stream, "write") +ignored = getattr(logger, method_name) """ ) self.assertEqual( - [(item.kind, item.method, item.shape) for item in findings], + [(item.kind, item.method) for item in candidates], [ - ("logger", "error", "dynamic-message"), - ("logger", "warning", "dynamic-message"), + ("getattr-sink-candidate", "error"), + ("getattr-sink-candidate", "write"), ], ) - def test_inventories_callbacks_on_unknown_logger_receivers(self) -> None: - findings = inventory_source( + def test_collects_obvious_logging_callbacks(self) -> None: + candidates = inventory_source( """ register(log.warning) -register(on_error=log.error) +register(on_error=service.error) +register(result=request.error) """ ) self.assertEqual( - [(item.kind, item.method, item.confidence) for item in findings], + [(item.kind, item.method, item.call) for item in candidates], [ - ("logging-candidate-callback", "warning", "unknown"), - ("logging-candidate-callback", "error", "unknown"), + ("logging-callback-candidate", "warning", "log.warning"), + ("logging-callback-candidate", "error", "service.error"), ], ) - def test_flags_caught_values_and_implicit_exception_payloads(self) -> None: - findings = inventory_source( - """ -from agents.logger import logger - -try: - run() -except Exception as exc: - logger.error("Run failed: %s", exc) - logger.error("Run failed", exc_info=True) - logger.exception("Run failed") -""" - ) + def test_keeps_the_output_schema_free_of_security_certification(self) -> None: + candidate = inventory_source('logger.error("failed", secret)')[0].to_dict() self.assertEqual( - [(item.shape, item.catch_value) for item in findings], - [ - ("payload", "exc"), - ("exception-payload", "active exception"), - ("exception-payload", "active exception"), - ], + set(candidate), + { + "fingerprint", + "file", + "line", + "column", + "kind", + "method", + "context", + "call", + "reason", + }, ) + self.assertNotIn("policy", candidate) + self.assertNotIn("safe", candidate) - def test_fingerprints_survive_line_shifts_and_distinguish_branches(self) -> None: - before = inventory_source( + def test_reports_enclosing_scope_as_review_context(self) -> None: + candidate = inventory_source( """ -from agents.logger import logger -def report(secret): - if enabled: - logger.error("failed: %s", secret) - else: - logger.error("failed: %s", secret) +class Worker: + def report(self): + logger.error(secret) """ - ) - after = inventory_source( - """ -from agents.logger import logger - + )[0] -def report(secret): - unused = None - if enabled: - logger.error("failed: %s", secret) - else: - logger.error("failed: %s", secret) -""" - ) + self.assertEqual(candidate.context, "class:Worker>function:report") - self.assertEqual( - [item.group_fingerprint for item in before], - [item.group_fingerprint for item in after], - ) - self.assertNotEqual(before[0].group_fingerprint, before[1].group_fingerprint) - - def test_identical_sites_form_a_duplicate_group(self) -> None: - findings = inventory_source( + def test_does_not_claim_to_follow_assignment_aliases(self) -> None: + candidates = inventory_source( """ -from agents.logger import logger -def report(secret): - logger.error("failed: %s", secret) - logger.error("failed: %s", secret) +emit = logger.error +emit(secret) """ ) - self.assertEqual(findings[0].group_fingerprint, findings[1].group_fingerprint) - self.assertEqual([item.group_count for item in findings], [2, 2]) - self.assertEqual([item.identity_quality for item in findings], ["duplicate", "duplicate"]) - self.assertEqual([item.fingerprint.rsplit(":", 1)[-1] for item in findings], ["0", "1"]) + self.assertEqual(candidates, []) - def test_discovers_keyword_callback_sinks(self) -> None: - findings = inventory_source( + def test_summary_counts_only_syntactic_candidate_categories(self) -> None: + candidates = inventory_source( """ -import sys -from agents.logger import logger - -register(on_error=logger.error, writer=sys.stderr.write) +logger.error(secret) +print(secret) +log_tool_action_error(logger, "failed", error) +register(on_error=service.error) +getattr(logger, "warning") """ ) self.assertEqual( - [(item.kind, item.method) for item in findings], - [("logger-callback", "error"), ("raw-output-callback", "stderr.write")], - ) - - def test_keyword_callback_fingerprints_include_registration_context(self) -> None: - findings = inventory_source( - """ -from agents.logger import logger - -register_model(on_error=logger.error) -register_operational(on_error=logger.error) -register_model(on_warning=logger.error) -""" - ) - - self.assertEqual(len(findings), 3) - self.assertEqual(len({item.group_fingerprint for item in findings}), 3) - self.assertIn("callback:register_model:keyword:on_error", findings[0].context) - self.assertIn("callback:register_operational:keyword:on_error", findings[1].context) - self.assertIn("callback:register_model:keyword:on_warning", findings[2].context) - - def test_source_collection_ignores_hidden_paths_only_below_the_scan_root(self) -> None: - with TemporaryDirectory() as temp_dir: - root = Path(temp_dir) / ".cache" / "worktree" - visible = root / "src" / "visible.py" - hidden = root / ".generated" / "hidden.py" - visible.parent.mkdir(parents=True) - hidden.parent.mkdir() - visible.touch() - hidden.touch() - - self.assertEqual(collect_source_files([root]), [visible.resolve()]) - - def test_normalizes_path_separators_before_hashing(self) -> None: - source = "from agents.logger import logger\nlogger.error('failed', secret)\n" - posix = inventory_source(source, "src/agents/fixture.py") - windows = inventory_source(source, "src\\agents\\fixture.py") - self.assertEqual(posix[0].group_fingerprint, windows[0].group_fingerprint) - - def test_compare_reports_new_removed_and_duplicate_count_changes(self) -> None: - baseline = inventory_source("from agents.logger import logger\nlogger.error('x', secret)\n") - current = inventory_source( - """ -from agents.logger import logger -logger.error('x', secret) -logger.error('x', secret) -logger.warning('y', secret) -""" - ) - comparison = compare_findings([item.to_dict() for item in baseline], current) - - self.assertEqual(len(comparison["new"]), 1) - self.assertEqual(comparison["removed"], []) - self.assertEqual(len(comparison["countChanged"]), 1) - self.assertEqual(comparison["countChanged"][0]["before"], 1) - self.assertEqual(comparison["countChanged"][0]["after"], 2) - - def test_compare_reports_security_classification_changes(self) -> None: - baseline = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -def report(secret): - if _debug.DONT_LOG_TOOL_DATA: - return - logger.error("failed: %s", secret) -""" - ) - current = inventory_source( - """ -from agents import _debug -from agents.logger import logger - -def report(secret): - if False: - return - logger.error("failed: %s", secret) -""" - ) - - comparison = compare_findings([item.to_dict() for item in baseline], current) - - self.assertEqual(comparison["new"], []) - self.assertEqual(comparison["removed"], []) - self.assertEqual(comparison["countChanged"], []) - self.assertEqual(len(comparison["classificationChanged"]), 1) - change = comparison["classificationChanged"][0] - self.assertEqual(change["group_fingerprint"], baseline[0].group_fingerprint) - self.assertEqual(change["before"][0]["policy"], "tool-guard") - self.assertEqual(change["after"][0]["policy"], "none") - - def test_review_ledger_requires_every_dynamic_group_and_exact_count(self) -> None: - findings = inventory_source( - """ -from agents.logger import logger -logger.error('x', secret) -logger.error('x', secret) -logger.info('ready') -""" - ) - dynamic_group = findings[0].group_fingerprint - valid = validate_review_ledger( + summarize(candidates), { - "reviews": [ - { - "group_fingerprint": dynamic_group, - "group_count": 2, - "disposition": "tool", - "evidence": "The argument is the tool input.", - "action": "Guard with DONT_LOG_TOOL_DATA.", - } - ] + "totalCandidates": 5, + "loggingCalls": 1, + "rawOutputCalls": 1, + "policyHelperCalls": 1, + "getattrSelections": 1, + "callbackReferences": 1, }, - findings, ) - invalid = validate_review_ledger({"reviews": []}, findings) - - self.assertTrue(valid["valid"]) - self.assertFalse(invalid["valid"]) - self.assertIn("Missing review", invalid["errors"][0]) - def test_review_ledger_rejects_duplicate_group_entries(self) -> None: - findings = inventory_source("from agents.logger import logger\nlogger.error('x', secret)\n") - group = findings[0].group_fingerprint - review = { - "group_fingerprint": group, - "group_count": 1, - "disposition": "tool", - "evidence": "The argument is the tool input.", - "action": "Guard with DONT_LOG_TOOL_DATA.", - } + def test_collect_source_files_filters_hidden_children_relative_to_root(self) -> None: + with TemporaryDirectory(prefix=".hidden-parent-") as directory: + root = Path(directory) / "scan" + root.mkdir() + visible = root / "visible.py" + visible.write_text("print('visible')\n") + hidden = root / ".cache" + hidden.mkdir() + (hidden / "hidden.py").write_text("print('hidden')\n") - validation = validate_review_ledger({"reviews": [review, dict(review)]}, findings) - - self.assertFalse(validation["valid"]) - self.assertIn(f"Duplicate review entries for {group}", validation["errors"]) - - def test_review_ledger_rejects_non_text_evidence_and_actions(self) -> None: - findings = inventory_source("from agents.logger import logger\nlogger.error('x', secret)\n") - group = findings[0].group_fingerprint - review = { - "group_fingerprint": group, - "group_count": 1, - "disposition": "tool", - "evidence": {}, - "action": [], - } - - validation = validate_review_ledger({"reviews": [review]}, findings) - - self.assertFalse(validation["valid"]) - self.assertIn(f"Missing evidence for {group}.", validation["errors"]) - self.assertIn(f"Missing action for {group}.", validation["errors"]) - - def test_summary_separates_unknown_and_raw_candidates(self) -> None: - findings = inventory_source( - """ -from agents.logger import logger -logger.info('ready') -logger.error('failed', secret) -print(secret) -task.exception() -""" - ) - summary = summarize(findings) - - self.assertEqual(summary["total"], 4) - self.assertEqual(summary["dynamic"], 3) - self.assertEqual(summary["rawOutputCalls"], 1) - self.assertEqual(summary["unknownReceiverCalls"], 1) + self.assertEqual(collect_source_files([root]), [visible.resolve()]) if __name__ == "__main__": diff --git a/src/agents/tracing/provider.py b/src/agents/tracing/provider.py index 8f4d884e86..d569eca701 100644 --- a/src/agents/tracing/provider.py +++ b/src/agents/tracing/provider.py @@ -8,6 +8,7 @@ from abc import ABC, abstractmethod from collections.abc import Callable from datetime import datetime, timezone +from functools import partial from inspect import Parameter, signature from typing import Any, cast @@ -45,6 +46,14 @@ def _has_closed_stream_handler(log: logging.Logger) -> bool: return +def _processor_diagnostic_extra(processor: TracingProcessor) -> dict[str, object]: + return {"trace_processor": processor} + + +def _processor_shutdown_message(processor: TracingProcessor) -> str: + return f"Shutting down trace processor {processor}" + + def _remaining_timeout(deadline: float | None) -> float | None: if deadline is None: return None @@ -113,7 +122,7 @@ def on_trace_start(self, trace: Trace) -> None: logger, "Error in trace processor during on_trace_start", e, - diagnostic_extra=lambda processor=processor: {"trace_processor": processor}, + diagnostic_extra=partial(_processor_diagnostic_extra, processor), ) def on_trace_end(self, trace: Trace) -> None: @@ -128,7 +137,7 @@ def on_trace_end(self, trace: Trace) -> None: logger, "Error in trace processor during on_trace_end", e, - diagnostic_extra=lambda processor=processor: {"trace_processor": processor}, + diagnostic_extra=partial(_processor_diagnostic_extra, processor), ) def on_span_start(self, span: Span[Any]) -> None: @@ -143,7 +152,7 @@ def on_span_start(self, span: Span[Any]) -> None: logger, "Error in trace processor during on_span_start", e, - diagnostic_extra=lambda processor=processor: {"trace_processor": processor}, + diagnostic_extra=partial(_processor_diagnostic_extra, processor), ) def on_span_end(self, span: Span[Any]) -> None: @@ -158,7 +167,7 @@ def on_span_end(self, span: Span[Any]) -> None: logger, "Error in trace processor during on_span_end", e, - diagnostic_extra=lambda processor=processor: {"trace_processor": processor}, + diagnostic_extra=partial(_processor_diagnostic_extra, processor), ) def shutdown(self, timeout: float | None = None) -> None: @@ -170,9 +179,7 @@ def shutdown(self, timeout: float | None = None) -> None: if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA: _safe_debug("Shutting down trace processor") else: - _safe_debug( - lambda processor=processor: f"Shutting down trace processor {processor}" - ) + _safe_debug(partial(_processor_shutdown_message, processor)) try: processor_timeout = _remaining_timeout(deadline) if processor_timeout is not None and processor_timeout <= 0: @@ -189,7 +196,7 @@ def shutdown(self, timeout: float | None = None) -> None: logger, "Error shutting down trace processor", e, - diagnostic_extra=lambda processor=processor: {"trace_processor": processor}, + diagnostic_extra=partial(_processor_diagnostic_extra, processor), ) def force_flush(self): @@ -204,7 +211,7 @@ def force_flush(self): logger, "Error flushing trace processor", e, - diagnostic_extra=lambda processor=processor: {"trace_processor": processor}, + diagnostic_extra=partial(_processor_diagnostic_extra, processor), ) From af3eb0d2931034a4b7d2af47ba54550f999d7ace Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Fri, 24 Jul 2026 23:19:42 +0900 Subject: [PATCH 07/12] fix --- src/agents/agent.py | 5 ++ .../memory/advanced_sqlite_session.py | 11 +++- .../memory/test_advanced_sqlite_session.py | 55 +++++++++++++++++++ tests/test_agent_as_tool.py | 31 +++++++++-- 4 files changed, 97 insertions(+), 5 deletions(-) diff --git a/src/agents/agent.py b/src/agents/agent.py index ecc1291d1d..f5977d9e54 100644 --- a/src/agents/agent.py +++ b/src/agents/agent.py @@ -855,10 +855,15 @@ async def _run_handler(payload: AgentToolStreamEvent) -> None: if inspect.isawaitable(maybe_result): await maybe_result except Exception as exc: + + def diagnostic_extra() -> dict[str, object]: + return {"agent_name": self.name} + log_model_and_tool_action_error( logger, "Error while handling an agent tool on_stream event", exc, + diagnostic_extra=diagnostic_extra, ) async def dispatch_stream_events() -> None: diff --git a/src/agents/extensions/memory/advanced_sqlite_session.py b/src/agents/extensions/memory/advanced_sqlite_session.py index 088ac8bcac..822c123570 100644 --- a/src/agents/extensions/memory/advanced_sqlite_session.py +++ b/src/agents/extensions/memory/advanced_sqlite_session.py @@ -470,7 +470,16 @@ async def store_run_usage(self, result: RunResult) -> None: turn_anchor=turn_anchor, ) except Exception as e: - log_model_action_error(self._logger, "Failed to store session usage", e) + + def diagnostic_extra() -> dict[str, object]: + return {"session_id": self.session_id} + + log_model_action_error( + self._logger, + "Failed to store session usage", + e, + diagnostic_extra=diagnostic_extra, + ) def _capture_current_turn(self) -> tuple[int, str, int | None]: """Return (current_turn, branch_id, turn_anchor) in one locked read. diff --git a/tests/extensions/memory/test_advanced_sqlite_session.py b/tests/extensions/memory/test_advanced_sqlite_session.py index 9a9b50480e..7b27ff2025 100644 --- a/tests/extensions/memory/test_advanced_sqlite_session.py +++ b/tests/extensions/memory/test_advanced_sqlite_session.py @@ -3,6 +3,7 @@ import asyncio import contextlib import json +import logging import tempfile import threading from pathlib import Path @@ -1458,6 +1459,60 @@ async def test_error_handling_in_usage_tracking(usage_data: Usage): run_result = create_mock_run_result(usage_data) await session.store_run_usage(run_result) + +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted"), + [(True, False), (False, True), (False, False)], +) +async def test_usage_tracking_failure_identity_follows_model_data_policy( + usage_data: Usage, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + model_redacted: bool, + tool_redacted: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + session_id = "SECRET_USAGE_SESSION_ID" + test_logger = logging.getLogger("advanced-sqlite-usage-failure") + session = AdvancedSQLiteSession( + session_id=session_id, + create_tables=True, + logger=test_logger, + ) + secret = "SECRET_USAGE_FAILURE" + run_result = create_mock_run_result(usage_data) + + with ( + patch.object( + session, + "_update_turn_usage_internal", + side_effect=RuntimeError(secret), + ), + caplog.at_level(logging.ERROR, logger=test_logger.name), + ): + await session.store_run_usage(run_result) + + record = next( + record + for record in caplog.records + if "Failed to store session usage" in record.getMessage() + ) + if model_redacted: + assert record.msg == "%s" + assert record.args == ("Failed to store session usage",) + assert record.exc_info is None + assert "session_id" not in record.__dict__ + assert secret not in caplog.text + assert session_id not in caplog.text + else: + assert record.__dict__["session_id"] == session_id + assert record.exc_info is not None + assert record.exc_info[1] is not None + assert secret in caplog.text + + session.close() + # Close the session to simulate database errors session.close() diff --git a/tests/test_agent_as_tool.py b/tests/test_agent_as_tool.py index ab5efcce57..ba199dbdd0 100644 --- a/tests/test_agent_as_tool.py +++ b/tests/test_agent_as_tool.py @@ -2405,13 +2405,20 @@ async def _invoke_tool() -> Any: @pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted"), + [(True, False), (False, True), (False, False)], +) async def test_agent_as_tool_streaming_handler_exception_does_not_fail_call( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, + model_redacted: bool, + tool_redacted: bool, ) -> None: - monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", True) - monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) - agent = Agent(name="handler_error_agent") + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + agent_name = "SECRET_HANDLER_ERROR_AGENT" + agent = Agent(name=agent_name) secret = "SECRET_AGENT_STREAM_PAYLOAD" class DummyStreamingResult: @@ -2459,7 +2466,23 @@ def bad_handler(event: AgentToolStreamEvent) -> None: output = await tool.on_invoke_tool(tool_context, '{"input": "go"}') assert output == "ok" - assert secret not in caplog.text + record = next( + record + for record in caplog.records + if "Error while handling an agent tool on_stream event" in record.getMessage() + ) + if model_redacted or tool_redacted: + assert record.msg == "%s" + assert record.args == ("Error while handling an agent tool on_stream event",) + assert record.exc_info is None + assert "agent_name" not in record.__dict__ + assert secret not in caplog.text + assert agent_name not in caplog.text + else: + assert record.__dict__["agent_name"] == agent_name + assert record.exc_info is not None + assert record.exc_info[1] is not None + assert secret in caplog.text @pytest.mark.asyncio From 6e5811b61d1bb12f88eac01a8ffe4a4271d4582b Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Fri, 24 Jul 2026 23:22:08 +0900 Subject: [PATCH 08/12] fix --- .../memory/test_advanced_sqlite_session.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/extensions/memory/test_advanced_sqlite_session.py b/tests/extensions/memory/test_advanced_sqlite_session.py index 7b27ff2025..e0d3237f7a 100644 --- a/tests/extensions/memory/test_advanced_sqlite_session.py +++ b/tests/extensions/memory/test_advanced_sqlite_session.py @@ -1459,6 +1459,12 @@ async def test_error_handling_in_usage_tracking(usage_data: Usage): run_result = create_mock_run_result(usage_data) await session.store_run_usage(run_result) + # Close the session to simulate database errors + session.close() + + # This should not raise an exception (error should be caught) + await session.store_run_usage(run_result) + @pytest.mark.parametrize( ("model_redacted", "tool_redacted"), @@ -1513,12 +1519,6 @@ async def test_usage_tracking_failure_identity_follows_model_data_policy( session.close() - # Close the session to simulate database errors - session.close() - - # This should not raise an exception (error should be caught) - await session.store_run_usage(run_result) - async def test_advanced_tool_name_extraction(): """Test advanced tool name extraction for different tool types.""" From 7b9b48d3ee55c5d6b342811de345b8e0b2251581 Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Fri, 24 Jul 2026 23:35:05 +0900 Subject: [PATCH 09/12] fix --- .../extensions/sandbox/blaxel/mounts.py | 7 ++- src/agents/run.py | 4 +- src/agents/sandbox/runtime.py | 4 +- tests/extensions/sandbox/test_blaxel.py | 40 +++++++++++++- tests/sandbox/test_memory.py | 55 ++++++++++++++++--- 5 files changed, 95 insertions(+), 15 deletions(-) diff --git a/src/agents/extensions/sandbox/blaxel/mounts.py b/src/agents/extensions/sandbox/blaxel/mounts.py index d2cefac3d2..d31a61edb0 100644 --- a/src/agents/extensions/sandbox/blaxel/mounts.py +++ b/src/agents/extensions/sandbox/blaxel/mounts.py @@ -669,7 +669,12 @@ async def _detach_drive(sandbox: Any, mount_path: str) -> None: try: await drives.unmount(mount_path) except Exception as e: - log_tool_action_warning(logger, "Drive detach failed (non-fatal)", e) + log_tool_action_warning( + logger, + "Drive detach failed (non-fatal)", + e, + diagnostic_extra=lambda: {"mount_path": mount_path}, + ) __all__ = [ diff --git a/src/agents/run.py b/src/agents/run.py index 1473559048..bb07bd2554 100644 --- a/src/agents/run.py +++ b/src/agents/run.py @@ -27,7 +27,7 @@ TResponseInputItem, ) from .lifecycle import RunHooks -from .logger import log_tool_action_warning, logger +from .logger import log_model_and_tool_action_warning, log_tool_action_warning, logger from .memory import Session from .result import RunResult, RunResultStreaming from .run_config import ( @@ -1606,7 +1606,7 @@ def _finalize_result(result: RunResult) -> RunResult: terminal_metadata=terminal_metadata_for_exception(run_exception), ) except Exception as error: - log_tool_action_warning( + log_model_and_tool_action_warning( logger, "Failed to enqueue sandbox memory after run", error ) sandbox_resume_state = await sandbox_runtime.cleanup() diff --git a/src/agents/sandbox/runtime.py b/src/agents/sandbox/runtime.py index cfd1372251..0378323a63 100644 --- a/src/agents/sandbox/runtime.py +++ b/src/agents/sandbox/runtime.py @@ -9,7 +9,7 @@ from ..agent import Agent from ..exceptions import UserError from ..items import TResponseInputItem -from ..logger import log_tool_action_warning +from ..logger import log_model_and_tool_action_warning from ..result import RunResult, RunResultStreaming from ..run_config import RunConfig from ..run_context import RunContextWrapper, TContext @@ -107,7 +107,7 @@ async def _cleanup_and_store() -> None: input_override=_stream_memory_input_override(result), ) except Exception as error: - log_tool_action_warning( + log_model_and_tool_action_warning( logger, "Failed to enqueue sandbox memory after streamed run", error, diff --git a/tests/extensions/sandbox/test_blaxel.py b/tests/extensions/sandbox/test_blaxel.py index 8f189bcd4d..945d6ecee9 100644 --- a/tests/extensions/sandbox/test_blaxel.py +++ b/tests/extensions/sandbox/test_blaxel.py @@ -3,6 +3,7 @@ import asyncio import io import json +import logging import tarfile import time import uuid @@ -14,6 +15,7 @@ import pytest from pydantic import ValidationError +import agents._debug as _debug from agents.run_config import SandboxRunConfig from agents.sandbox import Manifest, SandboxPathGrant from agents.sandbox.config import DEFAULT_PYTHON_SANDBOX_IMAGE @@ -3539,13 +3541,45 @@ async def test_detach_drive_success(self) -> None: assert sandbox.drives.unmount_calls == ["/mnt/data"] @pytest.mark.asyncio - async def test_detach_drive_error_logged_not_raised(self) -> None: + @pytest.mark.parametrize("redacted", [True, False], ids=["redacted", "diagnostic"]) + async def test_detach_drive_error_logged_not_raised( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + redacted: bool, + ) -> None: from agents.extensions.sandbox.blaxel.mounts import _detach_drive + mount_path = "/mnt/SECRET_DRIVE_PATH" + error = RuntimeError("SECRET_UNMOUNT_ERROR") sandbox = _FakeSandboxInstance() - sandbox.drives.unmount_error = RuntimeError("unmount failed") + sandbox.drives.unmount_error = error + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + caplog.set_level(logging.WARNING) + # Should not raise; error is logged. - await _detach_drive(sandbox, "/mnt/data") + await _detach_drive(sandbox, mount_path) + + record = next( + record + for record in caplog.records + if "Drive detach failed" in logging.Formatter().format(record) + ) + if redacted: + assert record.msg == "%s" + assert record.args == ("Drive detach failed (non-fatal)",) + assert record.exc_info is None + assert record.exc_text is None + assert "mount_path" not in record.__dict__ + assert error not in record.__dict__.values() + rendered = logging.Formatter().format(record) + assert mount_path not in rendered + assert "SECRET_UNMOUNT_ERROR" not in rendered + else: + assert record.__dict__["mount_path"] == mount_path + assert record.exc_info is not None + assert record.exc_info[1] is error + assert "SECRET_UNMOUNT_ERROR" in logging.Formatter().format(record) @pytest.mark.asyncio async def test_detach_drive_no_drives_api(self) -> None: diff --git a/tests/sandbox/test_memory.py b/tests/sandbox/test_memory.py index c917dacd6d..4026cd6673 100644 --- a/tests/sandbox/test_memory.py +++ b/tests/sandbox/test_memory.py @@ -2,6 +2,7 @@ import io import json +import logging from dataclasses import dataclass from datetime import datetime from pathlib import Path @@ -12,6 +13,7 @@ from openai.types.responses.response_output_message import ResponseOutputMessage from openai.types.responses.response_reasoning_item import ResponseReasoningItem +import agents._debug as _debug import agents.sandbox.capabilities.memory as memory_module import agents.sandbox.memory.manager as memory_manager_module import agents.sandbox.memory.phase_one as phase_one_module @@ -1495,15 +1497,31 @@ async def test_sandbox_memory_unregisters_manager_on_session_close() -> None: await client.delete(session) +@pytest.mark.parametrize("streamed", [False, True], ids=["non_streamed", "streamed"]) +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted"), + [(True, False), (False, True), (False, False)], + ids=["model_redacted", "tool_redacted", "diagnostic"], +) @pytest.mark.asyncio -async def test_sandbox_memory_enqueue_failure_still_cleans_up_owned_session( +async def test_sandbox_memory_enqueue_failure_follows_both_data_policies( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + streamed: bool, + model_redacted: bool, + tool_redacted: bool, ) -> None: + secret = "SECRET_SANDBOX_MEMORY_PAYLOAD" + error = RuntimeError(secret) + async def _raise_write_rollout(*args: Any, **kwargs: Any) -> Path: _ = args, kwargs - raise RuntimeError("write_rollout failed") + raise error monkeypatch.setattr(memory_manager_module, "write_rollout", _raise_write_rollout) + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + caplog.set_level(logging.WARNING) client = _DeleteTrackingUnixLocalSandboxClient() agent = SandboxAgent( @@ -1513,16 +1531,39 @@ async def _raise_write_rollout(*args: Any, **kwargs: Any) -> Path: capabilities=[_memory_config()], ) - result = await Runner.run( - agent, - "hello", - run_config=RunConfig(sandbox=SandboxRunConfig(client=client)), - ) + run_config = RunConfig(sandbox=SandboxRunConfig(client=client)) + if streamed: + result = Runner.run_streamed(agent, "hello", run_config=run_config) + async for _ in result.stream_events(): + pass + expected_message = "Failed to enqueue sandbox memory after streamed run" + else: + result = await Runner.run(agent, "hello", run_config=run_config) + expected_message = "Failed to enqueue sandbox memory after run" assert result.final_output == "done" assert len(client.deleted_roots) == 1 assert not client.deleted_roots[0].exists() + record = next( + record + for record in caplog.records + if expected_message in logging.Formatter().format(record) + ) + redacted = model_redacted or tool_redacted + if redacted: + assert record.msg == "%s" + assert record.args == (expected_message,) + assert record.exc_info is None + assert record.exc_text is None + assert error not in record.__dict__.values() + assert secret not in logging.Formatter().format(record) + else: + assert record.args == (expected_message, error) + assert record.exc_info is not None + assert record.exc_info[1] is error + assert secret in logging.Formatter().format(record) + @pytest.mark.asyncio async def test_sandbox_memory_marks_interrupted_runs_in_phase_one_prompt() -> None: From 0e2d5015f392c3eb868e6f47639c793e0fb55c93 Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Fri, 24 Jul 2026 23:51:11 +0900 Subject: [PATCH 10/12] fix --- src/agents/logger.py | 13 ++++- src/agents/sandbox/sandboxes/unix_local.py | 6 +++ src/agents/sandbox/session/manager.py | 7 ++- src/agents/tracing/provider.py | 6 ++- tests/sandbox/test_memory.py | 3 +- tests/sandbox/test_runtime.py | 31 ++++++++++- tests/sandbox/test_session_manager.py | 48 +++++++++++++++++ tests/test_error_logging_redaction.py | 61 +++++++++++++++++++++- 8 files changed, 166 insertions(+), 9 deletions(-) diff --git a/src/agents/logger.py b/src/agents/logger.py index 14cf61e04d..7c65bcb51f 100644 --- a/src/agents/logger.py +++ b/src/agents/logger.py @@ -1,5 +1,6 @@ import logging from collections.abc import Callable, Mapping +from types import TracebackType from . import _debug @@ -8,6 +9,14 @@ _DiagnosticExtra = Callable[[], Mapping[str, object]] +def _exception_info( + exc: BaseException, +) -> tuple[type[BaseException], BaseException, TracebackType | None]: + """Build logging exception info without evaluating exception truthiness.""" + traceback = BaseException.__getattribute__(exc, "__traceback__") + return type(exc), exc, traceback + + def _log_action_error( target_logger: logging.Logger, message: str, @@ -25,7 +34,7 @@ def _log_action_error( "%s: %s", message, exc, - exc_info=exc, + exc_info=_exception_info(exc), extra=diagnostic_extra() if diagnostic_extra is not None else None, stacklevel=stacklevel, ) @@ -48,7 +57,7 @@ def _log_action_at_level( "%s: %s", message, exc, - exc_info=exc, + exc_info=_exception_info(exc), extra=diagnostic_extra() if diagnostic_extra is not None else None, stacklevel=stacklevel, ) diff --git a/src/agents/sandbox/sandboxes/unix_local.py b/src/agents/sandbox/sandboxes/unix_local.py index 6115faa328..a2bfeac2ba 100644 --- a/src/agents/sandbox/sandboxes/unix_local.py +++ b/src/agents/sandbox/sandboxes/unix_local.py @@ -24,6 +24,7 @@ from collections.abc import Mapping, Sequence from contextlib import suppress from dataclasses import dataclass, field +from functools import partial from pathlib import Path from typing import Literal, cast @@ -76,6 +77,10 @@ logger = logging.getLogger(__name__) +def _mount_path_diagnostic_extra(mount_path: Path) -> dict[str, object]: + return {"mount_path": str(mount_path)} + + def _close_fd_quietly(fd: int) -> None: with suppress(OSError): os.close(fd) @@ -1136,6 +1141,7 @@ async def delete(self, session: SandboxSession) -> SandboxSession: logger, "Failed to unmount UnixLocal workspace mount before deleting root", exc, + diagnostic_extra=partial(_mount_path_diagnostic_extra, mount_path), ) if unmount_failed: return session diff --git a/src/agents/sandbox/session/manager.py b/src/agents/sandbox/session/manager.py index 0cd1fb4298..1248ce19b9 100644 --- a/src/agents/sandbox/session/manager.py +++ b/src/agents/sandbox/session/manager.py @@ -161,7 +161,12 @@ def _handle_sink_error( ) -> None: if force_no_raise or sink.on_error in ("log", "ignore"): if sink.on_error == "log": - log_tool_action_error(logger, "Sandbox event sink failed (ignored)", exc) + log_tool_action_error( + logger, + "Sandbox event sink failed (ignored)", + exc, + diagnostic_extra=lambda: {"sink_type": type(sink).__name__}, + ) return raise RuntimeError( "sandbox event sink failed: " diff --git a/src/agents/tracing/provider.py b/src/agents/tracing/provider.py index d569eca701..57817642f8 100644 --- a/src/agents/tracing/provider.py +++ b/src/agents/tracing/provider.py @@ -47,7 +47,11 @@ def _has_closed_stream_handler(log: logging.Logger) -> bool: def _processor_diagnostic_extra(processor: TracingProcessor) -> dict[str, object]: - return {"trace_processor": processor} + processor_type = type(processor) + processor_identity = ( + f"{processor_type.__module__}.{processor_type.__qualname__}@{id(processor):x}" + ) + return {"trace_processor": processor_identity} def _processor_shutdown_message(processor: TracingProcessor) -> str: diff --git a/tests/sandbox/test_memory.py b/tests/sandbox/test_memory.py index 4026cd6673..fa6c3d4bca 100644 --- a/tests/sandbox/test_memory.py +++ b/tests/sandbox/test_memory.py @@ -34,7 +34,7 @@ ToolApprovalItem, TResponseOutputItem, ) -from agents.result import RunResultStreaming +from agents.result import RunResult, RunResultStreaming from agents.run import _sandbox_memory_input from agents.run_context import RunContextWrapper from agents.sandbox import ( @@ -1532,6 +1532,7 @@ async def _raise_write_rollout(*args: Any, **kwargs: Any) -> Path: ) run_config = RunConfig(sandbox=SandboxRunConfig(client=client)) + result: RunResult | RunResultStreaming if streamed: result = Runner.run_streamed(agent, "hello", run_config=run_config) async for _ in result.stream_events(): diff --git a/tests/sandbox/test_runtime.py b/tests/sandbox/test_runtime.py index 9f46ef02cc..fa68f0c69d 100644 --- a/tests/sandbox/test_runtime.py +++ b/tests/sandbox/test_runtime.py @@ -3,6 +3,7 @@ import asyncio import io import json +import logging import os import re import shutil @@ -18,6 +19,7 @@ from openai.types.responses.response_output_item import LocalShellCall, LocalShellCallAction from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary +import agents._debug as _debug import agents.sandbox.runtime_agent_preparation as runtime_agent_preparation_module from agents import Agent, AgentHooks, LocalShellTool, RunHooks, Runner, function_tool from agents.exceptions import InputGuardrailTripwireTriggered, UserError @@ -2115,14 +2117,17 @@ async def _fake_unmount( assert order == [root / "outer" / "child", root / "outer"] +@pytest.mark.parametrize("redacted", [True, False], ids=["redacted", "diagnostic"]) @pytest.mark.asyncio async def test_unix_local_client_delete_skips_rmtree_when_unmount_fails( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + redacted: bool, ) -> None: client = UnixLocalSandboxClient() manifest = _unix_local_manifest( entries={ - "remote": S3Mount( + "SECRET_REMOTE_MOUNT": S3Mount( bucket="bucket", mount_strategy=InContainerMountStrategy(pattern=MountpointMountPattern()), ), @@ -2139,7 +2144,7 @@ async def _failing_unmount( base_dir: Path, ) -> None: _ = (self, session, dest, base_dir) - raise RuntimeError("busy") + raise RuntimeError("SECRET_UNMOUNT_ERROR") def _fake_rmtree(path: Path, ignore_errors: bool = False) -> None: _ = (path, ignore_errors) @@ -2148,12 +2153,34 @@ def _fake_rmtree(path: Path, ignore_errors: bool = False) -> None: monkeypatch.setattr(S3Mount, "unmount", _failing_unmount) monkeypatch.setattr(shutil, "rmtree", _fake_rmtree) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + caplog.set_level(logging.WARNING) await client.delete(session) assert rmtree_called is False assert workspace_root.exists() + record = next( + record + for record in caplog.records + if "Failed to unmount UnixLocal workspace mount" in logging.Formatter().format(record) + ) + mount_path = str(workspace_root / "SECRET_REMOTE_MOUNT") + if redacted: + assert record.msg == "%s" + assert record.args == ("Failed to unmount UnixLocal workspace mount before deleting root",) + assert record.exc_info is None + assert record.exc_text is None + assert "mount_path" not in record.__dict__ + assert mount_path not in logging.Formatter().format(record) + assert "SECRET_UNMOUNT_ERROR" not in logging.Formatter().format(record) + else: + assert record.__dict__["mount_path"] == mount_path + assert record.exc_info is not None + assert record.exc_info[1] is not None + assert "SECRET_UNMOUNT_ERROR" in logging.Formatter().format(record) + shutil.rmtree(workspace_root, ignore_errors=True) diff --git a/tests/sandbox/test_session_manager.py b/tests/sandbox/test_session_manager.py index 67891b74c8..d6ed34ada6 100644 --- a/tests/sandbox/test_session_manager.py +++ b/tests/sandbox/test_session_manager.py @@ -1,11 +1,13 @@ from __future__ import annotations import asyncio +import logging import uuid from pathlib import Path import pytest +import agents._debug as _debug from agents.sandbox.manifest import Manifest from agents.sandbox.runtime_session_manager import SandboxRuntimeSessionManager from agents.sandbox.sandboxes.unix_local import ( @@ -193,6 +195,52 @@ async def handle(self, event: SandboxSessionEvent) -> None: await instrumentation.emit(event) +@pytest.mark.parametrize("redacted", [True, False], ids=["redacted", "diagnostic"]) +@pytest.mark.asyncio +async def test_logged_sink_failure_conditionally_includes_sink_type( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + redacted: bool, +) -> None: + class _FailingLogSink(_EventSink): + async def handle(self, event: SandboxSessionEvent) -> None: + _ = event + raise RuntimeError("SECRET_SINK_ERROR") + + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted) + caplog.set_level(logging.ERROR) + instrumentation = Instrumentation(sinks=[_FailingLogSink(mode="sync", on_error="log")]) + event = SandboxSessionFinishEvent( + session_id=uuid.uuid4(), + seq=1, + op="running", + span_id="span_running", + ok=True, + duration_ms=0.0, + ) + + await instrumentation.emit(event) + + record = next( + record + for record in caplog.records + if "Sandbox event sink failed" in logging.Formatter().format(record) + ) + if redacted: + assert record.msg == "%s" + assert record.args == ("Sandbox event sink failed (ignored)",) + assert record.exc_info is None + assert record.exc_text is None + assert "sink_type" not in record.__dict__ + assert "_FailingLogSink" not in logging.Formatter().format(record) + assert "SECRET_SINK_ERROR" not in logging.Formatter().format(record) + else: + assert record.__dict__["sink_type"] == "_FailingLogSink" + assert record.exc_info is not None + assert record.exc_info[1] is not None + assert "SECRET_SINK_ERROR" in logging.Formatter().format(record) + + def test_session_manager_uses_custom_snapshot_spec_without_resolving_default( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/test_error_logging_redaction.py b/tests/test_error_logging_redaction.py index 16f008eceb..72ba0883a0 100644 --- a/tests/test_error_logging_redaction.py +++ b/tests/test_error_logging_redaction.py @@ -9,7 +9,11 @@ from __future__ import annotations import logging +import pickle +import threading +from logging.handlers import QueueHandler from pathlib import Path +from queue import SimpleQueue from typing import Any from unittest.mock import patch @@ -71,9 +75,23 @@ def __getattribute__(self, name: str): return super().__getattribute__(name) +class _TruthinessException(Exception): + def __init__(self, *, truthy: bool) -> None: + super().__init__("diagnostic failure") + self.truthy = truthy + self.bool_calls = 0 + + def __bool__(self) -> bool: + self.bool_calls += 1 + if self.truthy: + raise AssertionError("logging inspected exception truthiness") + return False + + class _FailingTracingProcessor(TracingProcessor): def __init__(self) -> None: self.str_calls = 0 + self.lock = threading.Lock() def __str__(self) -> str: self.str_calls += 1 @@ -282,6 +300,35 @@ def test_shared_error_helper_preserves_diagnostics_when_enabled(monkeypatch) -> assert _SECRET in logging.Formatter().format(record) +@pytest.mark.parametrize( + "helper", + [log_shared_tool_action_error, log_tool_action_warning], +) +@pytest.mark.parametrize("truthy", [False, True], ids=["falsey", "hostile_bool"]) +def test_shared_error_helpers_do_not_evaluate_exception_truthiness( + monkeypatch, + helper, + truthy: bool, +) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + test_logger = logging.Logger("sensitive-logging-exception-truthiness") + handler = _RecordingHandler() + test_logger.addHandler(handler) + error = _TruthinessException(truthy=truthy) + + try: + raise error + except _TruthinessException: + helper(test_logger, "Tool failed", error) + + record = handler.records[0] + assert error.bool_calls == 0 + assert record.exc_info is not None + assert record.exc_info[0] is type(error) + assert record.exc_info[1] is error + assert record.exc_info[2] is error.__traceback__ + + @pytest.mark.parametrize("redacted", [True, False]) def test_shared_error_helper_conditionally_attaches_diagnostic_extra( monkeypatch, redacted: bool @@ -357,7 +404,13 @@ def test_trace_processor_failure_identity_follows_both_data_policies( assert _SECRET not in logging.Formatter().format(record) assert failing.str_calls == 0 else: - assert record.__dict__["trace_processor"] is failing + processor_identity = record.__dict__["trace_processor"] + assert isinstance(processor_identity, str) + assert type(failing).__module__ in processor_identity + assert type(failing).__qualname__ in processor_identity + assert f"{id(failing):x}" in processor_identity + prepared = QueueHandler(SimpleQueue()).prepare(record) + pickle.dumps(prepared) assert record.exc_info is not None assert record.exc_info[1] is not None assert _SECRET in logging.Formatter().format(record) @@ -425,7 +478,11 @@ def test_log_tool_action_error_logs_full_when_tool_data_enabled(monkeypatch) -> mock_logger.error.assert_called_once() logged = str(mock_logger.error.call_args) assert "/secret/path" in logged - assert isinstance(mock_logger.error.call_args.kwargs.get("exc_info"), ValueError) + exc_info = mock_logger.error.call_args.kwargs.get("exc_info") + assert isinstance(exc_info, tuple) + assert exc_info[0] is ValueError + assert isinstance(exc_info[1], ValueError) + assert exc_info[2] is None @pytest.mark.asyncio From 7c4b7e479239f7069a585e39b910680e1b563870 Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Sat, 25 Jul 2026 00:11:02 +0900 Subject: [PATCH 11/12] fix --- .../extensions/sandbox/cloudflare/sandbox.py | 2 +- src/agents/logger.py | 11 ++- src/agents/realtime/session.py | 27 +++++--- src/agents/run_internal/run_loop.py | 12 +++- .../memory/test_advanced_sqlite_session.py | 34 +++++++--- tests/extensions/sandbox/test_blaxel.py | 4 +- tests/extensions/sandbox/test_cloudflare.py | 22 +++--- tests/extensions/sandbox/test_e2b.py | 27 ++++---- tests/realtime/test_session.py | 67 ++++++++++++++++++ tests/sandbox/test_runtime.py | 4 +- tests/sandbox/test_session_manager.py | 6 +- tests/test_agent_as_tool.py | 4 +- tests/test_agent_runner_streamed.py | 68 +++++++++++++++++++ tests/test_error_logging_redaction.py | 8 +-- 14 files changed, 233 insertions(+), 63 deletions(-) diff --git a/src/agents/extensions/sandbox/cloudflare/sandbox.py b/src/agents/extensions/sandbox/cloudflare/sandbox.py index bbaf4138e5..ab492ff823 100644 --- a/src/agents/extensions/sandbox/cloudflare/sandbox.py +++ b/src/agents/extensions/sandbox/cloudflare/sandbox.py @@ -695,10 +695,10 @@ async def _shutdown_backend(self) -> None: async with http.delete(url) as resp: if resp.status < 400 or resp.status == 404: return - detail = await _read_cloudflare_response_body(resp) if _debug.DONT_LOG_TOOL_DATA: logger.debug("Failed to delete Cloudflare sandbox on shutdown") else: + detail = await _read_cloudflare_response_body(resp) logger.debug( "Failed to delete Cloudflare sandbox on shutdown: %s", _cloudflare_http_error_message("DELETE /sandbox", resp.status, detail), diff --git a/src/agents/logger.py b/src/agents/logger.py index 7c65bcb51f..af9b64867c 100644 --- a/src/agents/logger.py +++ b/src/agents/logger.py @@ -7,6 +7,7 @@ logger = logging.getLogger("openai.agents") _DiagnosticExtra = Callable[[], Mapping[str, object]] +_DIAGNOSTIC_CONTEXT_FIELD = "openai_agents_diagnostic_context" def _exception_info( @@ -17,6 +18,12 @@ def _exception_info( return type(exc), exc, traceback +def _log_record_extra(diagnostic_extra: _DiagnosticExtra | None) -> dict[str, object] | None: + if diagnostic_extra is None: + return None + return {_DIAGNOSTIC_CONTEXT_FIELD: dict(diagnostic_extra())} + + def _log_action_error( target_logger: logging.Logger, message: str, @@ -35,7 +42,7 @@ def _log_action_error( message, exc, exc_info=_exception_info(exc), - extra=diagnostic_extra() if diagnostic_extra is not None else None, + extra=_log_record_extra(diagnostic_extra), stacklevel=stacklevel, ) @@ -58,7 +65,7 @@ def _log_action_at_level( message, exc, exc_info=_exception_info(exc), - extra=diagnostic_extra() if diagnostic_extra is not None else None, + extra=_log_record_extra(diagnostic_extra), stacklevel=stacklevel, ) diff --git a/src/agents/realtime/session.py b/src/agents/realtime/session.py index 9781bc1a0c..bbdbc1863a 100644 --- a/src/agents/realtime/session.py +++ b/src/agents/realtime/session.py @@ -5,6 +5,7 @@ import inspect import json from collections.abc import AsyncIterator, Sequence +from functools import partial from typing import Any, cast from pydantic import BaseModel @@ -20,7 +21,12 @@ from ..exceptions import ToolInputGuardrailTripwireTriggered, UserError from ..handoffs import Handoff from ..items import ToolApprovalItem -from ..logger import log_model_action_error, log_tool_action_error, logger +from ..logger import ( + log_model_action_error, + log_model_and_tool_action_warning, + log_tool_action_error, + logger, +) from ..run_config import ToolErrorFormatterArgs from ..run_context import RunContextWrapper, TContext from ..tool import DEFAULT_APPROVAL_REJECTION_MESSAGE, FunctionTool, Tool, invoke_function_tool @@ -87,6 +93,10 @@ class _RealtimeSessionClosedSentinel: _BACKGROUND_TASK_CANCEL_GRACE_SECONDS = 1.0 +def _guardrail_diagnostic_extra(guardrail: Any) -> dict[str, object]: + return {"guardrail_name": guardrail.get_name()} + + def _serialize_tool_output(output: Any) -> str: """Serialize structured tool outputs to JSON when possible.""" if isinstance(output, str): @@ -1316,15 +1326,12 @@ async def _run_output_guardrails(self, text: str, response_id: str) -> bool: if result.output.tripwire_triggered: triggered_results.append(result) except Exception as exc: - if _debug.DONT_LOG_MODEL_DATA: - logger.warning("Output guardrail raised an exception; skipping it") - else: - logger.warning( - "Output guardrail %r raised %s; skipping it.", - guardrail.get_name(), - exc, - ) - logger.debug("Output guardrail failure details.", exc_info=exc) + log_model_and_tool_action_warning( + logger, + "Output guardrail raised an exception; skipping it", + exc, + diagnostic_extra=partial(_guardrail_diagnostic_extra, guardrail), + ) continue if triggered_results: diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index 848bc23abd..4873ed542d 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -9,6 +9,7 @@ import dataclasses as _dc import json from collections.abc import Awaitable, Callable, Mapping +from functools import partial from typing import Any, TypeVar, cast from openai.types.responses import ( @@ -57,9 +58,9 @@ ) from ..lifecycle import RunHooks from ..logger import ( - log_model_action_debug, log_model_action_error, log_model_action_warning, + log_model_and_tool_action_debug, log_tool_action_warning, logger, ) @@ -279,6 +280,10 @@ async def cleanup_models_after_run(tool_use_tracker: AgentToolUseTracker) -> Non log_model_action_warning(logger, "Failed to clean up model resources after run", error) +def _agent_diagnostic_extra(agent: Agent[Any]) -> dict[str, object]: + return {"agent_name": agent.name} + + def _should_attach_generic_agent_error(exc: Exception) -> bool: return not isinstance( exc, @@ -1308,10 +1313,11 @@ async def _save_stream_items_without_count( if first_trigger is not None: raise InputGuardrailTripwireTriggered(first_trigger) except Exception as e: - log_model_action_debug( + log_model_and_tool_action_debug( logger, - f"Error finalizing streamed result for agent {current_agent.name}", + "Error finalizing streamed result", e, + diagnostic_extra=partial(_agent_diagnostic_extra, current_agent), ) try: await dispose_resolved_computers(run_context=context_wrapper) diff --git a/tests/extensions/memory/test_advanced_sqlite_session.py b/tests/extensions/memory/test_advanced_sqlite_session.py index e0d3237f7a..1e1973ba39 100644 --- a/tests/extensions/memory/test_advanced_sqlite_session.py +++ b/tests/extensions/memory/test_advanced_sqlite_session.py @@ -1489,30 +1489,42 @@ async def test_usage_tracking_failure_identity_follows_model_data_policy( secret = "SECRET_USAGE_FAILURE" run_result = create_mock_run_result(usage_data) - with ( - patch.object( - session, - "_update_turn_usage_internal", - side_effect=RuntimeError(secret), - ), - caplog.at_level(logging.ERROR, logger=test_logger.name), - ): - await session.store_run_usage(run_result) + original_record_factory = logging.getLogRecordFactory() + + def application_record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord: + record = original_record_factory(*args, **kwargs) + record.session_id = "APPLICATION_SESSION_ID" + return record + + logging.setLogRecordFactory(application_record_factory) + try: + with ( + patch.object( + session, + "_update_turn_usage_internal", + side_effect=RuntimeError(secret), + ), + caplog.at_level(logging.ERROR, logger=test_logger.name), + ): + await session.store_run_usage(run_result) + finally: + logging.setLogRecordFactory(original_record_factory) record = next( record for record in caplog.records if "Failed to store session usage" in record.getMessage() ) + assert record.__dict__["session_id"] == "APPLICATION_SESSION_ID" if model_redacted: assert record.msg == "%s" assert record.args == ("Failed to store session usage",) assert record.exc_info is None - assert "session_id" not in record.__dict__ + assert "openai_agents_diagnostic_context" not in record.__dict__ assert secret not in caplog.text assert session_id not in caplog.text else: - assert record.__dict__["session_id"] == session_id + assert record.__dict__["openai_agents_diagnostic_context"] == {"session_id": session_id} assert record.exc_info is not None assert record.exc_info[1] is not None assert secret in caplog.text diff --git a/tests/extensions/sandbox/test_blaxel.py b/tests/extensions/sandbox/test_blaxel.py index 945d6ecee9..97d0168ab3 100644 --- a/tests/extensions/sandbox/test_blaxel.py +++ b/tests/extensions/sandbox/test_blaxel.py @@ -3570,13 +3570,13 @@ async def test_detach_drive_error_logged_not_raised( assert record.args == ("Drive detach failed (non-fatal)",) assert record.exc_info is None assert record.exc_text is None - assert "mount_path" not in record.__dict__ + assert "openai_agents_diagnostic_context" not in record.__dict__ assert error not in record.__dict__.values() rendered = logging.Formatter().format(record) assert mount_path not in rendered assert "SECRET_UNMOUNT_ERROR" not in rendered else: - assert record.__dict__["mount_path"] == mount_path + assert record.__dict__["openai_agents_diagnostic_context"] == {"mount_path": mount_path} assert record.exc_info is not None assert record.exc_info[1] is error assert "SECRET_UNMOUNT_ERROR" in logging.Formatter().format(record) diff --git a/tests/extensions/sandbox/test_cloudflare.py b/tests/extensions/sandbox/test_cloudflare.py index 20d67ac6ab..c0a32f13de 100644 --- a/tests/extensions/sandbox/test_cloudflare.py +++ b/tests/extensions/sandbox/test_cloudflare.py @@ -50,6 +50,7 @@ def __init__(self, status: int = 200, json_body: Any = None, raw_body: bytes = b self.status = status self._json_body = json_body self._raw_body = raw_body + self.read_calls = 0 async def json(self, *, content_type: str | None = None) -> Any: _ = content_type @@ -58,6 +59,7 @@ async def json(self, *, content_type: str | None = None) -> Any: return json.loads(self._raw_body) async def read(self) -> bytes: + self.read_calls += 1 if self._json_body is not None: return json.dumps(self._json_body).encode() return self._raw_body @@ -1546,19 +1548,14 @@ async def test_cloudflare_shutdown_logs_respect_tool_data_policy( monkeypatch.setattr("agents._debug.DONT_LOG_TOOL_DATA", redacted) - sess = _make_session( - fake_http=_FakeHttp( - { - "DELETE /v1/sandbox/": _FakeResponse( - status=502, - json_body={ - "error": "pool error: Failed to start container", - "code": "pool_error", - }, - ) - } - ) + response = _FakeResponse( + status=502, + json_body={ + "error": "pool error: Failed to start container", + "code": "pool_error", + }, ) + sess = _make_session(fake_http=_FakeHttp({"DELETE /v1/sandbox/": response})) with caplog.at_level(logging.DEBUG, logger="agents.extensions.sandbox.cloudflare.sandbox"): await sess._shutdown_backend() @@ -1569,3 +1566,4 @@ async def test_cloudflare_shutdown_logs_respect_tool_data_policy( for r in caplog.records ) assert has_detail is not redacted + assert response.read_calls == (0 if redacted else 1) diff --git a/tests/extensions/sandbox/test_e2b.py b/tests/extensions/sandbox/test_e2b.py index 1f5ab3d2fb..f830546517 100644 --- a/tests/extensions/sandbox/test_e2b.py +++ b/tests/extensions/sandbox/test_e2b.py @@ -2217,11 +2217,12 @@ async def test_e2b_shutdown_logs_pause_failure_and_falls_back_to_kill( assert "Failed to pause E2B sandbox on shutdown; falling back to kill." in caplog.text assert ("SECRET_E2B_PAUSE_FAILURE" not in caplog.text) is redacted record = caplog.records[-1] - assert ("sandbox_id" in record.__dict__) is not redacted - assert ("pause_on_exit" in record.__dict__) is not redacted + assert ("openai_agents_diagnostic_context" in record.__dict__) is not redacted if not redacted: - assert record.__dict__["sandbox_id"] == sandbox.sandbox_id - assert record.__dict__["pause_on_exit"] is True + assert record.__dict__["openai_agents_diagnostic_context"] == { + "sandbox_id": sandbox.sandbox_id, + "pause_on_exit": True, + } @pytest.mark.asyncio @@ -2253,11 +2254,12 @@ async def test_e2b_shutdown_logs_kill_failure_after_pause_fallback( assert sandbox.kill_calls == 1 assert "Failed to kill E2B sandbox after pause fallback failure." in caplog.text record = caplog.records[-1] - assert ("sandbox_id" in record.__dict__) is not redacted - assert ("pause_on_exit" in record.__dict__) is not redacted + assert ("openai_agents_diagnostic_context" in record.__dict__) is not redacted if not redacted: - assert record.__dict__["sandbox_id"] == sandbox.sandbox_id - assert record.__dict__["pause_on_exit"] is True + assert record.__dict__["openai_agents_diagnostic_context"] == { + "sandbox_id": sandbox.sandbox_id, + "pause_on_exit": True, + } @pytest.mark.asyncio @@ -2288,11 +2290,12 @@ async def test_e2b_shutdown_logs_direct_kill_failure( assert sandbox.kill_calls == 1 assert "Failed to kill E2B sandbox on shutdown." in caplog.text record = caplog.records[-1] - assert ("sandbox_id" in record.__dict__) is not redacted - assert ("pause_on_exit" in record.__dict__) is not redacted + assert ("openai_agents_diagnostic_context" in record.__dict__) is not redacted if not redacted: - assert record.__dict__["sandbox_id"] == sandbox.sandbox_id - assert record.__dict__["pause_on_exit"] is False + assert record.__dict__["openai_agents_diagnostic_context"] == { + "sandbox_id": sandbox.sandbox_id, + "pause_on_exit": False, + } @pytest.mark.asyncio diff --git a/tests/realtime/test_session.py b/tests/realtime/test_session.py index b3b2da6da8..c585a6ed47 100644 --- a/tests/realtime/test_session.py +++ b/tests/realtime/test_session.py @@ -1,6 +1,7 @@ import asyncio import dataclasses import json +import logging import threading from typing import Any, cast from unittest.mock import AsyncMock, Mock, PropertyMock, patch @@ -3517,6 +3518,72 @@ def guardrail_func(context, agent, output): return OutputGuardrail(guardrail_function=guardrail_func, name="safe_guardrail") + @pytest.mark.parametrize( + ("model_redacted", "tool_redacted"), + [(True, False), (False, True), (False, False)], + ids=["model_redacted", "tool_redacted", "diagnostic"], + ) + @pytest.mark.asyncio + async def test_output_guardrail_failure_follows_both_data_policies( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + mock_model: RealtimeModel, + model_redacted: bool, + tool_redacted: bool, + ) -> None: + class _GuardrailError(RuntimeError): + def __init__(self) -> None: + super().__init__("SECRET_REALTIME_GUARDRAIL_ERROR") + self.bool_calls = 0 + + def __bool__(self) -> bool: + self.bool_calls += 1 + raise AssertionError("logging inspected guardrail exception truthiness") + + error = _GuardrailError() + + async def failing_guardrail(context, agent, output): + _ = context, agent, output + raise error + + guardrail = OutputGuardrail( + guardrail_function=failing_guardrail, + name="SECRET_REALTIME_GUARDRAIL_NAME", + ) + agent = RealtimeAgent(name="agent", output_guardrails=[guardrail]) + session = RealtimeSession(mock_model, agent, None) + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + + with caplog.at_level(logging.DEBUG, logger="openai.agents"): + triggered = await session._run_output_guardrails("model text", "response-id") + + assert triggered is False + record = next( + record + for record in caplog.records + if "Output guardrail raised an exception" in record.getMessage() + ) + redacted = model_redacted or tool_redacted + if redacted: + assert record.msg == "%s" + assert record.args == ("Output guardrail raised an exception; skipping it",) + assert record.exc_info is None + assert record.exc_text is None + assert "openai_agents_diagnostic_context" not in record.__dict__ + assert error not in record.__dict__.values() + rendered = logging.Formatter().format(record) + assert "SECRET_REALTIME_GUARDRAIL_ERROR" not in rendered + assert "SECRET_REALTIME_GUARDRAIL_NAME" not in rendered + else: + context = record.__dict__["openai_agents_diagnostic_context"] + assert context == {"guardrail_name": "SECRET_REALTIME_GUARDRAIL_NAME"} + assert record.exc_info is not None + assert record.exc_info[1] is error + assert "SECRET_REALTIME_GUARDRAIL_ERROR" in logging.Formatter().format(record) + assert error.bool_calls == 0 + @pytest.mark.asyncio async def test_transcript_delta_triggers_guardrail_at_threshold( self, mock_model, mock_agent, triggered_guardrail diff --git a/tests/sandbox/test_runtime.py b/tests/sandbox/test_runtime.py index fa68f0c69d..0a76aed396 100644 --- a/tests/sandbox/test_runtime.py +++ b/tests/sandbox/test_runtime.py @@ -2172,11 +2172,11 @@ def _fake_rmtree(path: Path, ignore_errors: bool = False) -> None: assert record.args == ("Failed to unmount UnixLocal workspace mount before deleting root",) assert record.exc_info is None assert record.exc_text is None - assert "mount_path" not in record.__dict__ + assert "openai_agents_diagnostic_context" not in record.__dict__ assert mount_path not in logging.Formatter().format(record) assert "SECRET_UNMOUNT_ERROR" not in logging.Formatter().format(record) else: - assert record.__dict__["mount_path"] == mount_path + assert record.__dict__["openai_agents_diagnostic_context"] == {"mount_path": mount_path} assert record.exc_info is not None assert record.exc_info[1] is not None assert "SECRET_UNMOUNT_ERROR" in logging.Formatter().format(record) diff --git a/tests/sandbox/test_session_manager.py b/tests/sandbox/test_session_manager.py index d6ed34ada6..a6f5c6d70f 100644 --- a/tests/sandbox/test_session_manager.py +++ b/tests/sandbox/test_session_manager.py @@ -231,11 +231,13 @@ async def handle(self, event: SandboxSessionEvent) -> None: assert record.args == ("Sandbox event sink failed (ignored)",) assert record.exc_info is None assert record.exc_text is None - assert "sink_type" not in record.__dict__ + assert "openai_agents_diagnostic_context" not in record.__dict__ assert "_FailingLogSink" not in logging.Formatter().format(record) assert "SECRET_SINK_ERROR" not in logging.Formatter().format(record) else: - assert record.__dict__["sink_type"] == "_FailingLogSink" + assert record.__dict__["openai_agents_diagnostic_context"] == { + "sink_type": "_FailingLogSink" + } assert record.exc_info is not None assert record.exc_info[1] is not None assert "SECRET_SINK_ERROR" in logging.Formatter().format(record) diff --git a/tests/test_agent_as_tool.py b/tests/test_agent_as_tool.py index ba199dbdd0..3872bbb8f6 100644 --- a/tests/test_agent_as_tool.py +++ b/tests/test_agent_as_tool.py @@ -2475,11 +2475,11 @@ def bad_handler(event: AgentToolStreamEvent) -> None: assert record.msg == "%s" assert record.args == ("Error while handling an agent tool on_stream event",) assert record.exc_info is None - assert "agent_name" not in record.__dict__ + assert "openai_agents_diagnostic_context" not in record.__dict__ assert secret not in caplog.text assert agent_name not in caplog.text else: - assert record.__dict__["agent_name"] == agent_name + assert record.__dict__["openai_agents_diagnostic_context"] == {"agent_name": agent_name} assert record.exc_info is not None assert record.exc_info[1] is not None assert secret in caplog.text diff --git a/tests/test_agent_runner_streamed.py b/tests/test_agent_runner_streamed.py index 85b0c56bc5..b2f9e6e027 100644 --- a/tests/test_agent_runner_streamed.py +++ b/tests/test_agent_runner_streamed.py @@ -2,6 +2,7 @@ import asyncio import json +import logging from typing import Any, cast import httpx @@ -17,6 +18,7 @@ from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary from typing_extensions import TypedDict +import agents._debug as _debug from agents import ( Agent, GuardrailFunctionOutput, @@ -1188,6 +1190,72 @@ def guardrail_function( pass +@pytest.mark.parametrize( + ("model_redacted", "tool_redacted"), + [(True, False), (False, True), (False, False)], + ids=["model_redacted", "tool_redacted", "diagnostic"], +) +@pytest.mark.asyncio +async def test_streamed_finalizer_failure_follows_both_data_policies( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + model_redacted: bool, + tool_redacted: bool, +) -> None: + async def safe_guardrail( + context: RunContextWrapper[Any], agent: Agent[Any], input: Any + ) -> GuardrailFunctionOutput: + _ = context, agent, input + return GuardrailFunctionOutput(output_info=None, tripwire_triggered=False) + + error = RuntimeError("SECRET_STREAM_FINALIZER_ERROR") + + async def fail_finalizer(_result: Any) -> bool: + raise error + + monkeypatch.setattr( + run_loop, + "input_guardrail_tripwire_triggered_for_stream", + fail_finalizer, + ) + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", model_redacted) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", tool_redacted) + agent_name = "SECRET_STREAM_AGENT_NAME" + agent = Agent( + name=agent_name, + input_guardrails=[InputGuardrail(guardrail_function=safe_guardrail)], + model=FakeModel(initial_output=[get_text_message("done")]), + ) + + with caplog.at_level(logging.DEBUG, logger="openai.agents"): + result = Runner.run_streamed(agent, input="user_message") + async for _ in result.stream_events(): + pass + + assert result.final_output == "done" + record = next( + record + for record in caplog.records + if "Error finalizing streamed result" in record.getMessage() + ) + redacted = model_redacted or tool_redacted + if redacted: + assert record.msg == "%s" + assert record.args == ("Error finalizing streamed result",) + assert record.exc_info is None + assert record.exc_text is None + assert "openai_agents_diagnostic_context" not in record.__dict__ + rendered = logging.Formatter().format(record) + assert agent_name not in rendered + assert "SECRET_STREAM_FINALIZER_ERROR" not in rendered + else: + context = record.__dict__["openai_agents_diagnostic_context"] + assert context == {"agent_name": agent_name} + assert record.exc_info is not None + assert record.exc_info[1] is error + assert "SECRET_STREAM_FINALIZER_ERROR" in logging.Formatter().format(record) + + @pytest.mark.asyncio async def test_input_guardrail_streamed_does_not_save_assistant_message_to_session(): async def guardrail_function( diff --git a/tests/test_error_logging_redaction.py b/tests/test_error_logging_redaction.py index 72ba0883a0..2bb5f7458e 100644 --- a/tests/test_error_logging_redaction.py +++ b/tests/test_error_logging_redaction.py @@ -353,9 +353,9 @@ def diagnostic_extra() -> dict[str, object]: record = handler.records[0] assert extra_calls == (0 if redacted else 1) - assert ("sandbox_id" in record.__dict__) is not redacted + assert ("openai_agents_diagnostic_context" in record.__dict__) is not redacted if not redacted: - assert record.__dict__["sandbox_id"] == _SECRET + assert record.__dict__["openai_agents_diagnostic_context"] == {"sandbox_id": _SECRET} @pytest.mark.parametrize( @@ -398,13 +398,13 @@ def test_trace_processor_failure_identity_follows_both_data_policies( record = next(record for record in handler.records if record.levelno == logging.ERROR) redacted = model_redacted or tool_redacted if redacted: - assert "trace_processor" not in record.__dict__ + assert "openai_agents_diagnostic_context" not in record.__dict__ assert failing not in record.__dict__.values() assert record.exc_info is None assert _SECRET not in logging.Formatter().format(record) assert failing.str_calls == 0 else: - processor_identity = record.__dict__["trace_processor"] + processor_identity = record.__dict__["openai_agents_diagnostic_context"]["trace_processor"] assert isinstance(processor_identity, str) assert type(failing).__module__ in processor_identity assert type(failing).__qualname__ in processor_identity From 57dfa79bd8bfe28651f8918d046848b8de66e8c9 Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Sat, 25 Jul 2026 00:29:46 +0900 Subject: [PATCH 12/12] fix --- src/agents/logger.py | 5 ++- src/agents/realtime/session.py | 10 ++++- src/agents/run_internal/tool_execution.py | 25 ++++++++++-- tests/realtime/test_session.py | 50 +++++++++++++++++------ tests/test_error_logging_redaction.py | 44 ++++++++++++++++++-- 5 files changed, 113 insertions(+), 21 deletions(-) diff --git a/src/agents/logger.py b/src/agents/logger.py index af9b64867c..18b8670680 100644 --- a/src/agents/logger.py +++ b/src/agents/logger.py @@ -21,7 +21,10 @@ def _exception_info( def _log_record_extra(diagnostic_extra: _DiagnosticExtra | None) -> dict[str, object] | None: if diagnostic_extra is None: return None - return {_DIAGNOSTIC_CONTEXT_FIELD: dict(diagnostic_extra())} + try: + return {_DIAGNOSTIC_CONTEXT_FIELD: dict(diagnostic_extra())} + except Exception: + return None def _log_action_error( diff --git a/src/agents/realtime/session.py b/src/agents/realtime/session.py index bbdbc1863a..0a985c0387 100644 --- a/src/agents/realtime/session.py +++ b/src/agents/realtime/session.py @@ -94,7 +94,15 @@ class _RealtimeSessionClosedSentinel: def _guardrail_diagnostic_extra(guardrail: Any) -> dict[str, object]: - return {"guardrail_name": guardrail.get_name()} + try: + return {"guardrail_name": guardrail.get_name()} + except Exception: + try: + guardrail_type = type(guardrail.guardrail_function) + type_name = f"{guardrail_type.__module__}.{guardrail_type.__qualname__}" + except Exception: + type_name = "unknown" + return {"guardrail_type": type_name} def _serialize_tool_output(output: Any) -> str: diff --git a/src/agents/run_internal/tool_execution.py b/src/agents/run_internal/tool_execution.py index efcecf6a61..c38bdfe4e5 100644 --- a/src/agents/run_internal/tool_execution.py +++ b/src/agents/run_internal/tool_execution.py @@ -1041,14 +1041,29 @@ def format_shell_error(error: Exception | BaseException | Any) -> str: return repr(error) -def log_tool_action_error(message: str, exc: Exception | BaseException) -> None: +def _tool_name_diagnostic_extra(tool_name: str) -> dict[str, object]: + return {"tool_name": tool_name} + + +def log_tool_action_error( + message: str, + exc: Exception | BaseException, + *, + diagnostic_extra: Callable[[], Mapping[str, object]] | None = None, +) -> None: """Log a tool-action failure without leaking tool data. Tool exceptions can embed tool call arguments or output, so the exception is redacted by default (matching ``_debug.DONT_LOG_TOOL_DATA``). The full exception and traceback are logged only when tool-data logging is explicitly enabled. """ - _log_tool_action_error(logger, message, exc, stacklevel=4) + _log_tool_action_error( + logger, + message, + exc, + stacklevel=4, + diagnostic_extra=diagnostic_extra, + ) async def with_tool_function_span( @@ -1198,7 +1213,11 @@ async def resolve_approval_rejection_message( ) message = await maybe_message if inspect.isawaitable(maybe_message) else maybe_message except Exception as exc: - log_tool_action_error(f"Tool error formatter failed for {tool_name}", exc) + log_tool_action_error( + "Tool error formatter failed", + exc, + diagnostic_extra=functools.partial(_tool_name_diagnostic_extra, tool_name), + ) return REJECTION_MESSAGE if message is None: diff --git a/tests/realtime/test_session.py b/tests/realtime/test_session.py index c585a6ed47..8d69916505 100644 --- a/tests/realtime/test_session.py +++ b/tests/realtime/test_session.py @@ -3532,16 +3532,7 @@ async def test_output_guardrail_failure_follows_both_data_policies( model_redacted: bool, tool_redacted: bool, ) -> None: - class _GuardrailError(RuntimeError): - def __init__(self) -> None: - super().__init__("SECRET_REALTIME_GUARDRAIL_ERROR") - self.bool_calls = 0 - - def __bool__(self) -> bool: - self.bool_calls += 1 - raise AssertionError("logging inspected guardrail exception truthiness") - - error = _GuardrailError() + error = RuntimeError("SECRET_REALTIME_GUARDRAIL_ERROR") async def failing_guardrail(context, agent, output): _ = context, agent, output @@ -3560,11 +3551,13 @@ async def failing_guardrail(context, agent, output): triggered = await session._run_output_guardrails("model text", "response-id") assert triggered is False - record = next( + records = [ record for record in caplog.records if "Output guardrail raised an exception" in record.getMessage() - ) + ] + assert len(records) == 1 + record = records[0] redacted = model_redacted or tool_redacted if redacted: assert record.msg == "%s" @@ -3582,7 +3575,38 @@ async def failing_guardrail(context, agent, output): assert record.exc_info is not None assert record.exc_info[1] is error assert "SECRET_REALTIME_GUARDRAIL_ERROR" in logging.Formatter().format(record) - assert error.bool_calls == 0 + + @pytest.mark.asyncio + async def test_output_guardrail_failure_tolerates_missing_callable_name( + self, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + mock_model: RealtimeModel, + ) -> None: + class _FailingGuardrailCallable: + async def __call__(self, context, agent, output): + _ = context, agent, output + raise RuntimeError("SECRET_UNNAMED_GUARDRAIL_ERROR") + + guardrail = OutputGuardrail(guardrail_function=_FailingGuardrailCallable()) + agent = RealtimeAgent(name="agent", output_guardrails=[guardrail]) + session = RealtimeSession(mock_model, agent, None) + monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", False) + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + + with caplog.at_level(logging.WARNING, logger="openai.agents"): + triggered = await session._run_output_guardrails("model text", "response-id") + + assert triggered is False + records = [ + record + for record in caplog.records + if "Output guardrail raised an exception" in record.getMessage() + ] + assert len(records) == 1 + context = records[0].__dict__["openai_agents_diagnostic_context"] + assert context["guardrail_type"].endswith("._FailingGuardrailCallable") + assert records[0].exc_info is not None @pytest.mark.asyncio async def test_transcript_delta_triggers_guardrail_at_threshold( diff --git a/tests/test_error_logging_redaction.py b/tests/test_error_logging_redaction.py index 2bb5f7458e..4f08056915 100644 --- a/tests/test_error_logging_redaction.py +++ b/tests/test_error_logging_redaction.py @@ -358,6 +358,30 @@ def diagnostic_extra() -> dict[str, object]: assert record.__dict__["openai_agents_diagnostic_context"] == {"sandbox_id": _SECRET} +def test_shared_error_helper_ignores_diagnostic_extra_failure(monkeypatch) -> None: + monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False) + test_logger = logging.Logger("sensitive-logging-diagnostic-extra-failure") + handler = _RecordingHandler() + test_logger.addHandler(handler) + error = RuntimeError("original failure") + + def diagnostic_extra() -> dict[str, object]: + raise AttributeError("metadata failure") + + log_tool_action_warning( + test_logger, + "Tool failed", + error, + diagnostic_extra=diagnostic_extra, + ) + + record = handler.records[0] + assert "openai_agents_diagnostic_context" not in record.__dict__ + assert record.exc_info is not None + assert record.exc_info[1] is error + assert "original failure" in logging.Formatter().format(record) + + @pytest.mark.parametrize( "operation", [ @@ -493,16 +517,24 @@ async def test_approval_rejection_formatter_error_redacts_exception(monkeypatch, def boom(_args): raise ValueError("formatter blew up SECRET_FMT_123") + tool_name = "SECRET_FORMATTER_TOOL_NAME" result = await resolve_approval_rejection_message( context_wrapper=RunContextWrapper(context=None), run_config=RunConfig(tool_error_formatter=boom), tool_type="function", - tool_name="my_tool", + tool_name=tool_name, call_id="call_1", ) assert isinstance(result, str) and result - assert "Tool error formatter failed for my_tool" in caplog.text + record = next( + record for record in caplog.records if "Tool error formatter failed" in record.getMessage() + ) + assert record.msg == "%s" + assert record.args == ("Tool error formatter failed",) + assert record.exc_info is None + assert "openai_agents_diagnostic_context" not in record.__dict__ + assert tool_name not in caplog.text assert "SECRET_FMT_123" not in caplog.text @@ -516,12 +548,18 @@ async def test_approval_rejection_formatter_error_logs_full_when_enabled( def boom(_args): raise ValueError("formatter blew up SECRET_FMT_123") + tool_name = "diagnostic_tool" await resolve_approval_rejection_message( context_wrapper=RunContextWrapper(context=None), run_config=RunConfig(tool_error_formatter=boom), tool_type="function", - tool_name="my_tool", + tool_name=tool_name, call_id="call_1", ) + record = next( + record for record in caplog.records if "Tool error formatter failed" in record.getMessage() + ) + assert record.__dict__["openai_agents_diagnostic_context"] == {"tool_name": tool_name} + assert record.exc_info is not None assert "SECRET_FMT_123" in caplog.text