Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/fromager/packagesettings/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from ._models import (
BuildOptions,
DownloadSource,
ExternalCommands,
GitOptions,
PackageSettings,
ProjectOverride,
Expand Down Expand Up @@ -57,6 +58,7 @@
"DownloadSource",
"EnvKey",
"EnvVars",
"ExternalCommands",
"GitHubTagCloneResolver",
"GitHubTagDownloadResolver",
"GitLabTagCloneResolver",
Expand Down
136 changes: 135 additions & 1 deletion src/fromager/packagesettings/_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,15 @@
import logging
import os
import pathlib
import re
import typing
from collections.abc import Mapping

import pydantic
import yaml
from packaging.requirements import Requirement
from packaging.utils import canonicalize_name
from pydantic import AnyUrl, Field
from pydantic import AnyUrl, Field, PrivateAttr, StringConstraints
from pydantic_core import core_schema

# from ._resolver import SourceResolver
Expand Down Expand Up @@ -72,6 +73,139 @@ class SbomSettings(pydantic.BaseModel):
"""


# Environment variable filter patterns for ExternalCommands.
# Pattern: starts with letter or underscore, rest is letters/digits/underscores,
# optionally ending with ``*`` (trailing wildcard).
# DeleteEnvPattern additionally allows bare ``*`` (catch-all).
KeepEnvPattern = typing.Annotated[
str,
StringConstraints(pattern=r"^[a-zA-Z_][a-zA-Z0-9_]*\*?$"),
]

DeleteEnvPattern = typing.Annotated[
str,
StringConstraints(pattern=r"^(\*|[a-zA-Z_][a-zA-Z0-9_]*\*?)$"),

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This enforces the pattern to look like a valid POSIX environment variable name but the proposal doesn't say that. I'm inclined with having this but worried if any package has an unusual env var needed or need filtering that doesn't follow POSIX standard. eg. BOT-PAT.

]


def _compile_env_patterns(patterns: tuple[str, ...]) -> re.Pattern[str]:
"""Compile env filter patterns into a single regex for ``fullmatch``.

Exact patterns (e.g. ``HOME``) become ``HOME`` and prefix patterns
(e.g. ``LC_*``) become ``LC_.*``. The result is
``HOME|LC_.*|...`` (used with ``fullmatch``).

*patterns* must be non-empty.
"""
parts: list[str] = []
for p in patterns:
if p.endswith("*"):
parts.append(re.escape(p[:-1]) + ".*")
else:
parts.append(re.escape(p))
return re.compile("|".join(parts))

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://fromager.readthedocs.io/en/latest/proposals/filter-env.html#evaluation-order
The propsal says "All checks are case-insensitive and short-circuit". We should that proposal here as well. I think it should be case-sensitive as Env vars in Linux and macOS are case-sensitive but not in WIN and we don't support WIN.



class ExternalCommands(pydantic.BaseModel):
Comment thread
tiran marked this conversation as resolved.
"""Environment variable filtering for subprocesses

Patterns control which environment variables are passed to
child processes. ``keep_env`` is evaluated before ``delete_env``.

::

external_commands:
keep_env:
- "CARGO_*"
delete_env:
- "CI_TOKEN"
- "AWS_*"

.. versionadded:: 0.92.0
"""

model_config = MODEL_CONFIG

DEFAULT_KEEP_ENV: typing.ClassVar[tuple[str, ...]] = (

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should TERM be in DEFAULT_KEEP_ENV?

"HOME",
"HOSTNAME",
"LANG",
"LANGUAGE",
"LC_*",
"LOGNAME",
"NO_COLOR",
"PATH",
"SHELL",
"USER",
"http_proxy",
"https_proxy",
"no_proxy",
)
"""Patterns always kept regardless of user configuration."""

keep_env: list[KeepEnvPattern] = Field(default_factory=list)
"""Allowlist patterns (evaluated before ``delete_env``)"""

delete_env: list[DeleteEnvPattern] = Field(default_factory=list)
"""Blocklist patterns (evaluated after ``keep_env``)"""

_keep_re: re.Pattern[str] | None = PrivateAttr(default=None)
_delete_re: re.Pattern[str] | None = PrivateAttr(default=None)

@pydantic.model_validator(mode="after")
def validate_delete_env(self) -> typing.Self:
"""Validate ``delete_env`` for conflicts and redundancy."""
if not self.delete_env:
return self
if "*" in self.delete_env and len(self.delete_env) > 1:
raise ValueError(
"delete_env: bare '*' must be the only entry, "
"additional patterns are redundant"
)
keep = set(self.DEFAULT_KEEP_ENV) | set(self.keep_env)
overlap = keep & set(self.delete_env)
if overlap:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://fromager.readthedocs.io/en/latest/proposals/filter-env.html#evaluation-order

The priority model on proposal says "If a variable matches the hard-coded always-keep set or an entry in keep_env, then the variable is kept". We should update the proposal. I think this strict approach is better for a security feature (fail loud)

raise ValueError(
f"delete_env overlaps with keep_env / DEFAULT_KEEP_ENV: "
f"{sorted(overlap)}"
)
return self
Comment on lines +155 to +172

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔒 Security & Privacy | 🟠 Major | ⚡ Quick win

Overlap check misses wildcard-vs-literal conflicts, letting a delete_env entry silently no-op.

validate_delete_env only rejects exact string collisions between delete_env and DEFAULT_KEEP_ENV | keep_env. It doesn't detect pattern-level overlaps, e.g. keep_env=["AWS_*"] with delete_env=["AWS_SECRET_KEY"] passes validation cleanly, but at runtime _keep_re (compiled from AWS_*) will fullmatch AWS_SECRET_KEY, so filter_env keeps it anyway despite the explicit delete entry — silently defeating the user's intent to strip that variable and potentially leaking a secret into a subprocess.

🔐 Proposed fix: detect pattern-level overlap, not just exact matches
+def _patterns_conflict(a: str, b: str) -> bool:
+    """True if pattern *a* would also match anything pattern *b* matches (or vice versa)."""
+    a_base, a_wild = a.rstrip("*"), a.endswith("*")
+    b_base, b_wild = b.rstrip("*"), b.endswith("*")
+    if a_wild and b_wild:
+        return a_base.startswith(b_base) or b_base.startswith(a_base)
+    if a_wild:
+        return b_base.startswith(a_base)
+    if b_wild:
+        return a_base.startswith(b_base)
+    return a_base == b_base
+
+
 `@pydantic.model_validator`(mode="after")
 def validate_delete_env(self) -> typing.Self:
     """Validate ``delete_env`` for conflicts and redundancy."""
     if not self.delete_env:
         return self
     if "*" in self.delete_env and len(self.delete_env) > 1:
         raise ValueError(
             "delete_env: bare '*' must be the only entry, "
             "additional patterns are redundant"
         )
     keep = set(self.DEFAULT_KEEP_ENV) | set(self.keep_env)
-    overlap = keep & set(self.delete_env)
+    overlap = {
+        d for d in self.delete_env if any(_patterns_conflict(k, d) for k in keep)
+    }
     if overlap:
         raise ValueError(
             f"delete_env overlaps with keep_env / DEFAULT_KEEP_ENV: "
             f"{sorted(overlap)}"
         )
     return self
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/fromager/packagesettings/_models.py` around lines 155 - 172, Update
validate_delete_env to detect wildcard pattern overlaps between delete_env and
DEFAULT_KEEP_ENV or keep_env, not only exact string matches. Use the same
full-match semantics as _keep_re to identify any delete entry that would also be
retained by a keep pattern, and raise the existing validation error with the
conflicting entries; preserve the bare-* and non-overlapping behavior.


def __bool__(self) -> bool:
"""True when filtering is configured (``delete_env`` is non-empty)."""
return bool(self.delete_env)

def model_post_init(self, __context: typing.Any) -> None:
Comment thread
tiran marked this conversation as resolved.
"""Pydantic post init hook to initialize internal data structures"""
if self.delete_env:
self._keep_re = _compile_env_patterns(
self.DEFAULT_KEEP_ENV + tuple(self.keep_env)
)
if "*" not in self.delete_env:
self._delete_re = _compile_env_patterns(tuple(self.delete_env))
Comment thread
tiran marked this conversation as resolved.

def filter_env(self, env: Mapping[str, str]) -> Mapping[str, str]:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is the 1st time I seen business logic tied in the data model here.
Just want to confirm: existing models here are pure data holders with validators and business logic on separate classes.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was initially thinking filter_env() and DEFAULT_KEEP_ENV would live in separate file or external_commands.py since the models here have always been data + validation only. But having logic along with data would simply things and maybe we can apply this to few of existing models too.

"""Filter environment variables by keep/delete patterns.

Variables matching ``DEFAULT_KEEP_ENV`` or ``keep_env`` are
always kept. Of the remaining variables, those matching
``delete_env`` are removed. Variables matching neither list
are kept.
"""
# _keep_re is only set in model_post_init when delete_env is
# non-empty, so None means no filtering is configured.
if self._keep_re is None:
return env
if self._delete_re is None:
# delete_env is ["*"]: keep only what matches
return {k: v for k, v in env.items() if self._keep_re.fullmatch(k)}
return {
k: v
for k, v in env.items()
if self._keep_re.fullmatch(k) or not self._delete_re.fullmatch(k)
}


class PurlConfig(pydantic.BaseModel):
"""Per-package purl configuration for SBOM generation.

Expand Down
20 changes: 19 additions & 1 deletion src/fromager/packagesettings/_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from pydantic import Field

from .. import overrides
from ._models import PackageSettings, SbomSettings
from ._models import ExternalCommands, PackageSettings, SbomSettings
from ._pbi import PackageBuildInfo
from ._typedefs import MODEL_CONFIG, GlobalChangelog, Package, Variant

Expand Down Expand Up @@ -45,6 +45,16 @@ class SettingsFile(pydantic.BaseModel):
are generated.
"""

external_commands: ExternalCommands = Field(default_factory=ExternalCommands)
"""Environment variable filtering for subprocesses

Controls which environment variables are passed to child processes
using ``keep_env`` / ``delete_env`` patterns. Defaults to no
filtering.

.. versionadded:: 0.92.0
"""

@classmethod
def from_string(
cls,
Expand Down Expand Up @@ -175,6 +185,14 @@ def sbom_settings(self) -> SbomSettings | None:
"""Get global SBOM settings, or None if SBOM generation is disabled."""
return self._settings.sbom

@property
def external_commands(self) -> ExternalCommands:
"""Get external commands settings.

.. versionadded:: 0.92.0
"""
return self._settings.external_commands

def variant_changelog(self) -> list[str]:
"""Get global changelog for current variant"""
return list(self._settings.changelog.get(self.variant, []))
Expand Down
Loading
Loading