-
Notifications
You must be signed in to change notification settings - Fork 52
feat(packagesettings): add ExternalCommands model for env filtering
#1266
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,14 +5,15 @@ | |
| import logging | ||
| import os | ||
| import pathlib | ||
| import re | ||
| import typing | ||
| from collections.abc import Mapping | ||
|
|
||
| import pydantic | ||
| import yaml | ||
| from packaging.requirements import Requirement | ||
| from packaging.utils import canonicalize_name | ||
| from pydantic import AnyUrl, Field | ||
| from pydantic import AnyUrl, Field, PrivateAttr, StringConstraints | ||
| from pydantic_core import core_schema | ||
|
|
||
| # from ._resolver import SourceResolver | ||
|
|
@@ -72,6 +73,139 @@ class SbomSettings(pydantic.BaseModel): | |
| """ | ||
|
|
||
|
|
||
| # Environment variable filter patterns for ExternalCommands. | ||
| # Pattern: starts with letter or underscore, rest is letters/digits/underscores, | ||
| # optionally ending with ``*`` (trailing wildcard). | ||
| # DeleteEnvPattern additionally allows bare ``*`` (catch-all). | ||
| KeepEnvPattern = typing.Annotated[ | ||
| str, | ||
| StringConstraints(pattern=r"^[a-zA-Z_][a-zA-Z0-9_]*\*?$"), | ||
| ] | ||
|
|
||
| DeleteEnvPattern = typing.Annotated[ | ||
| str, | ||
| StringConstraints(pattern=r"^(\*|[a-zA-Z_][a-zA-Z0-9_]*\*?)$"), | ||
| ] | ||
|
|
||
|
|
||
| def _compile_env_patterns(patterns: tuple[str, ...]) -> re.Pattern[str]: | ||
| """Compile env filter patterns into a single regex for ``fullmatch``. | ||
|
|
||
| Exact patterns (e.g. ``HOME``) become ``HOME`` and prefix patterns | ||
| (e.g. ``LC_*``) become ``LC_.*``. The result is | ||
| ``HOME|LC_.*|...`` (used with ``fullmatch``). | ||
|
|
||
| *patterns* must be non-empty. | ||
| """ | ||
| parts: list[str] = [] | ||
| for p in patterns: | ||
| if p.endswith("*"): | ||
| parts.append(re.escape(p[:-1]) + ".*") | ||
| else: | ||
| parts.append(re.escape(p)) | ||
| return re.compile("|".join(parts)) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. https://fromager.readthedocs.io/en/latest/proposals/filter-env.html#evaluation-order |
||
|
|
||
|
|
||
| class ExternalCommands(pydantic.BaseModel): | ||
|
tiran marked this conversation as resolved.
|
||
| """Environment variable filtering for subprocesses | ||
|
|
||
| Patterns control which environment variables are passed to | ||
| child processes. ``keep_env`` is evaluated before ``delete_env``. | ||
|
|
||
| :: | ||
|
|
||
| external_commands: | ||
| keep_env: | ||
| - "CARGO_*" | ||
| delete_env: | ||
| - "CI_TOKEN" | ||
| - "AWS_*" | ||
|
|
||
| .. versionadded:: 0.92.0 | ||
| """ | ||
|
|
||
| model_config = MODEL_CONFIG | ||
|
|
||
| DEFAULT_KEEP_ENV: typing.ClassVar[tuple[str, ...]] = ( | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should |
||
| "HOME", | ||
| "HOSTNAME", | ||
| "LANG", | ||
| "LANGUAGE", | ||
| "LC_*", | ||
| "LOGNAME", | ||
| "NO_COLOR", | ||
| "PATH", | ||
| "SHELL", | ||
| "USER", | ||
| "http_proxy", | ||
| "https_proxy", | ||
| "no_proxy", | ||
| ) | ||
| """Patterns always kept regardless of user configuration.""" | ||
|
|
||
| keep_env: list[KeepEnvPattern] = Field(default_factory=list) | ||
| """Allowlist patterns (evaluated before ``delete_env``)""" | ||
|
|
||
| delete_env: list[DeleteEnvPattern] = Field(default_factory=list) | ||
| """Blocklist patterns (evaluated after ``keep_env``)""" | ||
|
|
||
| _keep_re: re.Pattern[str] | None = PrivateAttr(default=None) | ||
| _delete_re: re.Pattern[str] | None = PrivateAttr(default=None) | ||
|
|
||
| @pydantic.model_validator(mode="after") | ||
| def validate_delete_env(self) -> typing.Self: | ||
| """Validate ``delete_env`` for conflicts and redundancy.""" | ||
| if not self.delete_env: | ||
| return self | ||
| if "*" in self.delete_env and len(self.delete_env) > 1: | ||
| raise ValueError( | ||
| "delete_env: bare '*' must be the only entry, " | ||
| "additional patterns are redundant" | ||
| ) | ||
| keep = set(self.DEFAULT_KEEP_ENV) | set(self.keep_env) | ||
| overlap = keep & set(self.delete_env) | ||
| if overlap: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. https://fromager.readthedocs.io/en/latest/proposals/filter-env.html#evaluation-order The priority model on proposal says "If a variable matches the hard-coded always-keep set or an entry in keep_env, then the variable is kept". We should update the proposal. I think this strict approach is better for a security feature (fail loud) |
||
| raise ValueError( | ||
| f"delete_env overlaps with keep_env / DEFAULT_KEEP_ENV: " | ||
| f"{sorted(overlap)}" | ||
| ) | ||
| return self | ||
|
Comment on lines
+155
to
+172
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔒 Security & Privacy | 🟠 Major | ⚡ Quick win Overlap check misses wildcard-vs-literal conflicts, letting a
🔐 Proposed fix: detect pattern-level overlap, not just exact matches+def _patterns_conflict(a: str, b: str) -> bool:
+ """True if pattern *a* would also match anything pattern *b* matches (or vice versa)."""
+ a_base, a_wild = a.rstrip("*"), a.endswith("*")
+ b_base, b_wild = b.rstrip("*"), b.endswith("*")
+ if a_wild and b_wild:
+ return a_base.startswith(b_base) or b_base.startswith(a_base)
+ if a_wild:
+ return b_base.startswith(a_base)
+ if b_wild:
+ return a_base.startswith(b_base)
+ return a_base == b_base
+
+
`@pydantic.model_validator`(mode="after")
def validate_delete_env(self) -> typing.Self:
"""Validate ``delete_env`` for conflicts and redundancy."""
if not self.delete_env:
return self
if "*" in self.delete_env and len(self.delete_env) > 1:
raise ValueError(
"delete_env: bare '*' must be the only entry, "
"additional patterns are redundant"
)
keep = set(self.DEFAULT_KEEP_ENV) | set(self.keep_env)
- overlap = keep & set(self.delete_env)
+ overlap = {
+ d for d in self.delete_env if any(_patterns_conflict(k, d) for k in keep)
+ }
if overlap:
raise ValueError(
f"delete_env overlaps with keep_env / DEFAULT_KEEP_ENV: "
f"{sorted(overlap)}"
)
return self🤖 Prompt for AI Agents |
||
|
|
||
| def __bool__(self) -> bool: | ||
| """True when filtering is configured (``delete_env`` is non-empty).""" | ||
| return bool(self.delete_env) | ||
|
|
||
| def model_post_init(self, __context: typing.Any) -> None: | ||
|
tiran marked this conversation as resolved.
|
||
| """Pydantic post init hook to initialize internal data structures""" | ||
| if self.delete_env: | ||
| self._keep_re = _compile_env_patterns( | ||
| self.DEFAULT_KEEP_ENV + tuple(self.keep_env) | ||
| ) | ||
| if "*" not in self.delete_env: | ||
| self._delete_re = _compile_env_patterns(tuple(self.delete_env)) | ||
|
tiran marked this conversation as resolved.
|
||
|
|
||
| def filter_env(self, env: Mapping[str, str]) -> Mapping[str, str]: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this is the 1st time I seen business logic tied in the data model here.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I was initially thinking filter_env() and DEFAULT_KEEP_ENV would live in separate file or external_commands.py since the models here have always been data + validation only. But having logic along with data would simply things and maybe we can apply this to few of existing models too. |
||
| """Filter environment variables by keep/delete patterns. | ||
|
|
||
| Variables matching ``DEFAULT_KEEP_ENV`` or ``keep_env`` are | ||
| always kept. Of the remaining variables, those matching | ||
| ``delete_env`` are removed. Variables matching neither list | ||
| are kept. | ||
| """ | ||
| # _keep_re is only set in model_post_init when delete_env is | ||
| # non-empty, so None means no filtering is configured. | ||
| if self._keep_re is None: | ||
| return env | ||
| if self._delete_re is None: | ||
| # delete_env is ["*"]: keep only what matches | ||
| return {k: v for k, v in env.items() if self._keep_re.fullmatch(k)} | ||
| return { | ||
| k: v | ||
| for k, v in env.items() | ||
| if self._keep_re.fullmatch(k) or not self._delete_re.fullmatch(k) | ||
| } | ||
|
|
||
|
|
||
| class PurlConfig(pydantic.BaseModel): | ||
| """Per-package purl configuration for SBOM generation. | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This enforces the pattern to look like a valid POSIX environment variable name but the proposal doesn't say that. I'm inclined with having this but worried if any package has an unusual env var needed or need filtering that doesn't follow POSIX standard. eg. BOT-PAT.