From 71c7dd8d2c8233f124bb9951030da8e8e1f4700f Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 18 Sep 2026 13:19:48 -0700 Subject: [PATCH 01/12] refactor: simplify Node installation guidance --- AGENTS.md | 8 +- src/promptfoo/environment.py | 388 ++++------------------ src/promptfoo/instructions.py | 468 ++++---------------------- tests/test_environment.py | 601 ++++++---------------------------- tests/test_instructions.py | 499 ++++------------------------ 5 files changed, 288 insertions(+), 1676 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 569da0e..fb8915e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -508,12 +508,8 @@ git push --force ### Q: The release-please PR shows the wrong version. How do I fix it? -**A**: The version bump is determined by commit messages: -- Check all commits since the last release -- Ensure they follow conventional commits -- `fix:` commits bump minor version (pre-1.0.0) -- `feat:` commits bump minor version (pre-1.0.0) -- If the version is still wrong, you may need to manually adjust `.release-please-manifest.json` in a new PR +**A**: Check the commits since the last release against [Version Bumping Strategy](#version-bumping-strategy). +That section is the source of truth for the pre-1.0 rules and links to the release-please configuration. ### Q: How do I manually publish to PyPI? diff --git a/src/promptfoo/environment.py b/src/promptfoo/environment.py index 1d9ae43..ac8c44f 100644 --- a/src/promptfoo/environment.py +++ b/src/promptfoo/environment.py @@ -1,357 +1,95 @@ -""" -Environment detection for providing contextual Node.js installation instructions. - -This module detects the operating system, Linux distribution, cloud provider, -container environment, CI/CD platform, and Python environment to provide -tailored installation instructions for Node.js. -""" +"""Detect only the environment details that change the missing-Node help.""" import os +import platform import sys from dataclasses import dataclass from pathlib import Path -@dataclass +@dataclass(frozen=True) class Environment: - """Information about the current execution environment.""" + """Platform information used by the Node installation instructions.""" - os_type: str # "linux", "darwin", "windows" - linux_distro: str | None = None # "ubuntu", "debian", "rhel", "fedora", "alpine", "arch", etc. - linux_distro_version: str | None = None # e.g., "22.04", "11", "9" - cloud_provider: str | None = None # "aws", "gcp", "azure" - is_lambda: bool = False # AWS Lambda - is_cloud_function: bool = False # GCP Cloud Functions or Azure Functions + os_type: str + linux_distro: str | None = None + linux_distro_version: str | None = None is_docker: bool = False - is_kubernetes: bool = False - is_wsl: bool = False # Windows Subsystem for Linux - is_ci: bool = False - ci_platform: str | None = None # "github", "gitlab", "circleci", "jenkins", etc. - is_venv: bool = False - is_conda: bool = False - has_sudo: bool = False # Best guess if user has sudo access + is_wsl: bool = False + ci_platform: str | None = None + serverless: str | None = None -def _read_probe_file(path: Path) -> str | None: - """ - Read an optional environment probe file. +def _read_probe(path: str) -> str: + try: + return Path(path).read_text(encoding="utf-8") + except (OSError, UnicodeError): + return "" - Returns: - File contents, or None when the probe file does not exist or cannot be read. - """ - if not path.exists(): - return None +def _linux_release() -> tuple[str | None, str | None]: try: - with open(path) as f: - return f.read() + release = platform.freedesktop_os_release() except OSError: - # Environment detection is best-effort. Proc/sys metadata files can be - # unreadable or disappear between exists() and open(), so treat that as - # "signal unavailable" and continue with fallback probes. - return None - - -def _detect_linux_distro() -> tuple[str | None, str | None]: - """ - Detect Linux distribution and version. - - Returns: - Tuple of (distro_id, version) where distro_id is normalized - (e.g., "ubuntu", "debian", "rhel", "alpine", "arch") - """ - # Define known distros for normalization - known_base_distros = {"ubuntu", "debian", "alpine", "arch", "fedora"} - rhel_family = {"rhel", "centos", "rocky", "almalinux", "ol", "amzn"} - suse_family = {"opensuse", "opensuse-leap", "opensuse-tumbleweed", "sles"} - - # Try /etc/os-release first, then /usr/lib/os-release (per freedesktop spec) - for os_release_path in [Path("/etc/os-release"), Path("/usr/lib/os-release")]: - os_release_content = _read_probe_file(os_release_path) - if os_release_content is None: - continue - - os_release = {} - for line in os_release_content.splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - if "=" in line: - key, _, value = line.partition("=") - # Remove quotes - value = value.strip('"').strip("'") - os_release[key] = value - - distro_id = os_release.get("ID", "").lower() - version = os_release.get("VERSION_ID", "") - id_like = os_release.get("ID_LIKE", "").lower().split() + return None, None - # Normalize distro IDs - if distro_id in known_base_distros: - return distro_id, version - elif distro_id in rhel_family: - # Oracle Linux (ol), Amazon Linux (amzn) - return "rhel", version - elif distro_id in suse_family: - return "suse", version + distro = release.get("ID", "").lower() + version = release.get("VERSION_ID") or None + family = [distro, *release.get("ID_LIKE", "").lower().split()] + if "alpine" in family: + return "alpine", version + if distro in ("amzn", "amazon"): + return "amzn", version + return distro or None, version - # Check ID_LIKE for derivative distributions (e.g., Pop!_OS, Raspbian, Mint) - if id_like: - for parent in id_like: - if parent in known_base_distros: - return parent, version - elif parent in rhel_family: - return "rhel", version - elif parent in suse_family: - return "suse", version - # Return the raw distro_id if we couldn't normalize it - return distro_id, version - - # Fallback: check for specific files - if Path("/etc/debian_version").exists(): - return "debian", None - elif Path("/etc/redhat-release").exists(): - return "rhel", None - elif Path("/etc/alpine-release").exists(): - return "alpine", None - elif Path("/etc/arch-release").exists(): - return "arch", None - - return None, None - - -def _detect_cloud_provider() -> str | None: - """ - Detect if running on a cloud provider. +def _in_container() -> bool: + if os.environ.get("KUBERNETES_SERVICE_HOST") or Path("/.dockerenv").is_file(): + return True + cgroup = _read_probe("/proc/1/cgroup").lower() + return any(runtime in cgroup for runtime in ("docker", "containerd", "kubepods", "crio")) + + +def _ci_platform() -> str | None: + for variable, name in ( + ("GITHUB_ACTIONS", "GitHub Actions"), + ("GITLAB_CI", "GitLab CI"), + ("CIRCLECI", "CircleCI"), + ("JENKINS_URL", "Jenkins"), + ("BUILDKITE", "Buildkite"), + ("TF_BUILD", "Azure Pipelines"), + ("CI", "CI"), + ): + if os.environ.get(variable): + return name + return None - Returns: - One of "aws", "gcp", "azure", or None - """ - # AWS detection - # Check for EC2 metadata - uuid = _read_probe_file(Path("/sys/hypervisor/uuid")) - if uuid and uuid.strip().lower().startswith("ec2"): - return "aws" - # Check AWS environment variables - if os.getenv("AWS_EXECUTION_ENV") or os.getenv("AWS_REGION"): +def _serverless() -> str | None: + if os.environ.get("AWS_LAMBDA_FUNCTION_NAME"): return "aws" - - # GCP detection - # Check for GCP metadata - product = _read_probe_file(Path("/sys/class/dmi/id/product_name")) - if product: - product = product.strip() - if "Google" in product or "GCE" in product: - return "gcp" - - # Check GCP environment variables - if os.getenv("GOOGLE_CLOUD_PROJECT") or os.getenv("GCP_PROJECT"): - return "gcp" - - # Azure detection - vendor = _read_probe_file(Path("/sys/class/dmi/id/sys_vendor")) - # Could be Azure or Hyper-V, check for Azure-specific - if vendor and "Microsoft Corporation" in vendor.strip() and Path("/var/lib/waagent").exists(): - return "azure" - - # Check Azure environment variables - if os.getenv("AZURE_SUBSCRIPTION_ID") or os.getenv("WEBSITE_INSTANCE_ID"): + if os.environ.get("FUNCTIONS_WORKER_RUNTIME"): return "azure" - + if os.environ.get("FUNCTION_TARGET") or os.environ.get("FUNCTION_NAME"): + return "google" return None -def _detect_container() -> tuple[bool, bool]: - """ - Detect if running in a container. - - Returns: - Tuple of (is_docker, is_kubernetes) - """ - is_docker = False - is_kubernetes = False - - # Docker detection - if Path("/.dockerenv").exists(): - is_docker = True - - # Also check cgroup - cgroup_content = _read_probe_file(Path("/proc/1/cgroup")) - if cgroup_content and ("docker" in cgroup_content or "containerd" in cgroup_content): - is_docker = True - - # Kubernetes detection - if os.getenv("KUBERNETES_SERVICE_HOST"): - is_kubernetes = True - - return is_docker, is_kubernetes - - -def _detect_wsl() -> bool: - """ - Detect if running in Windows Subsystem for Linux (WSL). - - Returns: - True if running in WSL, False otherwise - """ - # Check for WSL environment variable - if os.getenv("WSL_DISTRO_NAME") or os.getenv("WSL_INTEROP"): - return True - - # Check /proc/version for Microsoft/WSL signatures - version_info = _read_probe_file(Path("/proc/version")) - if version_info: - version_info = version_info.lower() - if "microsoft" in version_info or "wsl" in version_info: - return True - - # Check for Windows filesystem mounts (WSL mounts Windows drives at /mnt/) - # This is less reliable but can catch WSL 1 - return Path("/mnt/c").exists() and Path("/proc/version").exists() - - -def _detect_ci() -> tuple[bool, str | None]: - """ - Detect if running in a CI/CD environment. - - Returns: - Tuple of (is_ci, ci_platform) - """ - ci_env_vars = { - "GITHUB_ACTIONS": "github", - "GITLAB_CI": "gitlab", - "CIRCLECI": "circleci", - "JENKINS_HOME": "jenkins", - "TRAVIS": "travis", - "BUILDKITE": "buildkite", - "DRONE": "drone", - "BITBUCKET_BUILD_NUMBER": "bitbucket", - "TEAMCITY_VERSION": "teamcity", - "TF_BUILD": "azure-devops", - } - - for env_var, platform in ci_env_vars.items(): - if os.getenv(env_var): - return True, platform - - # Generic CI detection - if os.getenv("CI"): - return True, None - - return False, None - - -def _detect_python_env() -> tuple[bool, bool]: - """ - Detect Python virtual environment. - - Returns: - Tuple of (is_venv, is_conda) - """ - # venv/virtualenv detection - is_venv = hasattr(sys, "real_prefix") or (hasattr(sys, "base_prefix") and sys.base_prefix != sys.prefix) - - # Conda detection - is_conda = "CONDA_DEFAULT_ENV" in os.environ or "CONDA_PREFIX" in os.environ - - return is_venv, is_conda - - -def _has_sudo_access() -> bool: - """ - Best-effort check if user likely has sudo access. - - Returns: - True if user is root or likely has sudo, False otherwise - """ - # Unix-like systems - if hasattr(os, "geteuid"): - # Root user - if os.geteuid() == 0: - return True - - # Check if sudo command exists - import shutil - - return shutil.which("sudo") is not None - - # Windows - check if admin (requires elevation detection) - if sys.platform == "win32": - try: - import ctypes - - return ctypes.windll.shell32.IsUserAnAdmin() != 0 - except Exception: - return False - - return False - - def detect_environment() -> Environment: - """ - Detect the current execution environment. - - Returns: - Environment object with detected platform information - """ - os_type = sys.platform - if os_type.startswith("linux"): - os_type = "linux" - elif os_type == "darwin": - os_type = "darwin" - elif os_type == "win32": - os_type = "windows" - - # Linux-specific detection - linux_distro = None - linux_distro_version = None - if os_type == "linux": - linux_distro, linux_distro_version = _detect_linux_distro() - - # Cloud provider detection - cloud_provider = _detect_cloud_provider() - - # Lambda and Cloud Functions detection - is_lambda = os.getenv("AWS_LAMBDA_FUNCTION_NAME") is not None - is_cloud_function = ( - os.getenv("FUNCTION_NAME") is not None # GCP Cloud Functions - or os.getenv("FUNCTIONS_WORKER_RUNTIME") is not None # Azure Functions + """Return the current platform and the deployment hints relevant to installation.""" + os_type = "linux" if sys.platform.startswith("linux") else {"win32": "windows"}.get(sys.platform, sys.platform) + distro, version = _linux_release() if os_type == "linux" else (None, None) + is_wsl = os_type == "linux" and bool( + os.environ.get("WSL_DISTRO_NAME") + or os.environ.get("WSL_INTEROP") + or "microsoft" in _read_probe("/proc/sys/kernel/osrelease").lower() ) - - # Container detection - is_docker, is_kubernetes = False, False - if os_type == "linux": - is_docker, is_kubernetes = _detect_container() - - # WSL detection - is_wsl = False - if os_type == "linux": - is_wsl = _detect_wsl() - - # CI detection - is_ci, ci_platform = _detect_ci() - - # Python environment detection - is_venv, is_conda = _detect_python_env() - - # Sudo detection - has_sudo = _has_sudo_access() - return Environment( os_type=os_type, - linux_distro=linux_distro, - linux_distro_version=linux_distro_version, - cloud_provider=cloud_provider, - is_lambda=is_lambda, - is_cloud_function=is_cloud_function, - is_docker=is_docker, - is_kubernetes=is_kubernetes, + linux_distro=distro, + linux_distro_version=version, + is_docker=os_type == "linux" and _in_container(), is_wsl=is_wsl, - is_ci=is_ci, - ci_platform=ci_platform, - is_venv=is_venv, - is_conda=is_conda, - has_sudo=has_sudo, + ci_platform=_ci_platform(), + serverless=_serverless(), ) diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index 2662cc5..e14dcf6 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -1,432 +1,84 @@ -""" -Platform-specific Node.js installation instructions. - -Generates tailored installation instructions based on the detected environment. -""" +"""Concise Node.js installation guidance for the current platform.""" from .environment import Environment from .node import MIN_NODE_VERSION_TEXT - -def get_installation_instructions(env: Environment) -> str: - """ - Generate Node.js installation instructions for the detected environment. - - Args: - env: Detected environment information - - Returns: - Formatted installation instructions as a multi-line string - """ - lines = [] - lines.append("=" * 70) - lines.append(f"ERROR: promptfoo requires Node.js {MIN_NODE_VERSION_TEXT} or newer but it's not installed") - lines.append("=" * 70) - lines.append("Install a supported version and verify it with: node --version") - lines.append("") - - # Special cases first (Lambda, Cloud Functions, etc.) - if env.is_lambda: - lines.extend(_get_lambda_instructions()) - return "\n".join(lines) - - if env.is_cloud_function: - lines.extend(_get_cloud_function_instructions(env)) - return "\n".join(lines) - - # CI/CD environment - if env.is_ci: - lines.extend(_get_ci_instructions(env)) - lines.append("") - - # Container environment - if env.is_docker: - lines.extend(_get_docker_instructions(env)) - lines.append("") - - # WSL environment - if env.is_wsl: - lines.extend(_get_wsl_instructions()) - lines.append("") - - # Platform-specific instructions - if env.os_type == "linux": - lines.extend(_get_linux_instructions(env)) - elif env.os_type == "darwin": - lines.extend(_get_macos_instructions()) - elif env.os_type == "windows": - lines.extend(_get_windows_instructions()) - - # Virtual environment alternative - if env.is_venv or env.is_conda: - lines.append("") - lines.extend(_get_venv_instructions()) - - # Direct npx usage - lines.append("") - lines.extend(_get_npx_instructions()) - - return "\n".join(lines) +_NODE_DOWNLOAD = "https://nodejs.org/en/download" +_NVM = "https://github.com/nvm-sh/nvm#installing-and-updating" +_SERVERLESS = { + "aws": ("AWS Lambda", "https://docs.aws.amazon.com/lambda/latest/dg/images-create.html"), + "google": ("Google Cloud Functions", "https://cloud.google.com/run/docs/runtimes/nodejs"), + "azure": ("Azure Functions", "https://learn.microsoft.com/en-us/azure/azure-functions/functions-reference-node"), +} -def _get_lambda_instructions() -> list[str]: - """Instructions for AWS Lambda environment.""" +def _container_instructions(env: Environment) -> list[str]: + if env.linux_distro == "alpine": + return [ + "CONTAINER: start from the official Node.js 24 Alpine image and add Python:", + " FROM node:24-alpine", + " RUN apk add --no-cache python3 py3-pip", + " RUN python3 -m venv /opt/venv", + ' ENV PATH="/opt/venv/bin:$PATH"', + ] return [ - "You are running in AWS Lambda with a Python runtime.", - "", - "AWS Lambda Python runtimes do not include Node.js. You have options:", - "", - "1. Use a Lambda Layer with Node.js:", - " https://docs.aws.amazon.com/lambda/latest/dg/chapter-layers.html", - "", - "2. Switch to Node.js runtime:", - " https://docs.aws.amazon.com/lambda/latest/dg/lambda-nodejs.html", - "", - "3. Use Lambda container images with both Python and Node.js:", - " https://docs.aws.amazon.com/lambda/latest/dg/images-create.html", - "", - "Note: promptfoo is primarily designed for local development and CI/CD,", - "not for Lambda runtime execution.", + "CONTAINER: include Node.js 24 in your image; keep the Node and Python base distributions compatible.", + " Official Node.js images: https://hub.docker.com/_/node", ] -def _get_cloud_function_instructions(env: Environment) -> list[str]: - """Instructions for Cloud Functions (GCP/Azure).""" - if env.cloud_provider == "gcp": +def _linux_instructions(env: Environment) -> list[str]: + if env.linux_distro == "alpine": return [ - "You are running in Google Cloud Functions with a Python runtime.", - "", - "GCP Cloud Functions Python runtimes do not include Node.js.", - "Consider using Node.js runtime instead:", - " https://cloud.google.com/functions/docs/concepts/nodejs-runtime", + "ALPINE: on a release whose repositories provide Node.js 24, run as root:", + " apk add --no-cache 'nodejs~24' npm", + " node --version", + "If apk cannot find Node.js 24, upgrade Alpine or use the official node:24-alpine container.", ] - else: # Azure or unknown + if env.linux_distro == "amzn" and env.linux_distro_version == "2023": return [ - "You are running in Azure Functions with a Python runtime.", - "", - "Azure Functions Python runtimes do not include Node.js.", - "Consider using Node.js runtime instead:", - " https://learn.microsoft.com/en-us/azure/azure-functions/functions-reference-node", + "AMAZON LINUX 2023: install and select Node.js 24 (omit sudo when running as root):", + " sudo dnf install -y nodejs24 nodejs24-npm", + " sudo alternatives --set node /usr/bin/node-24", + " node --version", + " https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html", ] - - -def _get_ci_instructions(env: Environment) -> list[str]: - """Instructions for CI/CD environments.""" - lines = ["RUNNING IN CI/CD: " + (env.ci_platform or "detected").upper(), ""] - - if env.ci_platform == "github": - lines.extend( - [ - "Add Node.js to your workflow:", - " - uses: actions/setup-node@v7", - " with:", - " node-version: '24'", - ] - ) - elif env.ci_platform == "gitlab": - lines.extend( - [ - "Use a Docker image with Node.js:", - " image: node:24", - ] - ) - elif env.ci_platform == "circleci": - lines.extend( - [ - "Use the CircleCI Node orb with your Python image:", - " orbs:", - " node: circleci/node@5", - " # Add under your job's steps:", - " - node/install:", - " node-version: '24'", - ] - ) - else: - lines.extend( - [ - "Install Node.js in your CI configuration.", - "Most CI platforms provide Node.js images or setup actions.", - ] - ) - - return lines - - -def _get_docker_instructions(env: Environment) -> list[str]: - """Instructions for Docker environments.""" - lines = ["RUNNING IN DOCKER CONTAINER:", ""] - - if env.linux_distro == "alpine": - lines.extend( - [ - "Start your Alpine Dockerfile from the official Node.js 24 image, then add Python:", - " FROM node:24-alpine", - " RUN apk add --no-cache python3 py3-pip", - " RUN python3 -m venv /opt/venv", - ' ENV PATH="/opt/venv/bin:$PATH"', - ] - ) - elif env.linux_distro in ("ubuntu", "debian"): - lines.extend( - [ - "Add to your Dockerfile (Debian/Ubuntu):", - " RUN apt-get update && apt-get install -y ca-certificates curl && \\", - " curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && \\", - " apt-get install -y nodejs && \\", - " rm -rf /var/lib/apt/lists/*", - ] - ) - else: - lines.extend( - [ - "Use matching Debian-based Node.js and Python stages in your Dockerfile:", - " FROM node:24-bookworm-slim AS node", - " FROM python:3.12-slim-bookworm", - " COPY --from=node /usr/local/ /usr/local/", - ] - ) - - return lines - - -def _get_wsl_instructions() -> list[str]: - """Instructions for Windows Subsystem for Linux (WSL).""" return [ - "WINDOWS SUBSYSTEM FOR LINUX (WSL) DETECTED:", - "", - "IMPORTANT: Install Node.js within WSL, not from Windows.", - "Using Windows Node.js from WSL can cause path and performance issues.", - "", - "Recommended approach:", - " 1. Use your Linux distribution's package manager (see below)", - " 2. Or use nvm for version management:", - " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash", - " source ~/.bashrc", - " nvm install 24", - "", - "Tips for WSL:", - " - Store project files in the WSL filesystem (~/), not /mnt/c/", - " - This improves file I/O performance significantly", - " - Use 'wsl --shutdown' to restart WSL if needed", + "LINUX: use a Node.js version manager or your distribution's instructions for Node.js 24.", + f" Install nvm: {_NVM}", + " Then run: nvm install 24", ] -def _get_linux_instructions(env: Environment) -> list[str]: - """Instructions for Linux systems.""" - lines = [] - distro = env.linux_distro - - if distro in ("ubuntu", "debian"): - lines.extend(_get_debian_instructions(env)) - elif distro == "rhel": - lines.extend(_get_rhel_instructions(env)) - elif distro == "alpine": - lines.extend(_get_alpine_instructions()) - elif distro == "arch": - lines.extend(_get_arch_instructions()) - elif distro == "suse": - lines.extend(_get_suse_instructions()) - else: - # Generic Linux instructions - lines.extend(_get_generic_linux_instructions()) - - return lines - - -def _get_debian_instructions(env: Environment) -> list[str]: - """Instructions for Debian/Ubuntu systems.""" - lines = ["UBUNTU/DEBIAN INSTALLATION:", ""] - - if env.has_sudo: - lines.extend( - [ - "Option 1 - Install from NodeSource (recommended for production):", - " curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash -", - " sudo apt install -y nodejs", - "", - "Option 2 - Use the default repository only if it supplies a supported version:", - " sudo apt update", - " sudo apt install -y nodejs npm", - "", - "Option 3 - Install using snap (not recommended for production):", - " sudo snap install node --classic", - " # Note: Snap auto-updates can cause unexpected behavior", - ] - ) - else: - lines.extend( - [ - "You don't have sudo access. Use nvm (Node Version Manager):", - " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash", - " source ~/.bashrc", - " nvm install 24", - ] - ) - - return lines - - -def _get_rhel_instructions(env: Environment) -> list[str]: - """Instructions for RHEL/CentOS/Fedora/Amazon Linux.""" - lines = [] - - # Detect Amazon Linux vs RHEL/CentOS/Fedora - is_amazon_linux = ( - env.linux_distro == "rhel" and env.linux_distro_version and env.linux_distro_version.startswith("202") - ) - - if is_amazon_linux: - lines.extend(["AMAZON LINUX INSTALLATION:", ""]) - if env.has_sudo: - lines.extend( - [ - "Amazon Linux 2023:", - " sudo dnf install -y nodejs", - "", - "Amazon Linux 2:", - " curl -fsSL https://rpm.nodesource.com/setup_24.x | sudo bash -", - " sudo yum install -y nodejs", - ] - ) - else: - lines.extend( - [ - "Use nvm (Node Version Manager) - no sudo needed:", - " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash", - " source ~/.bashrc", - " nvm install 24", - ] - ) - else: - lines.extend(["RHEL/CENTOS/FEDORA INSTALLATION:", ""]) - if env.has_sudo: - lines.extend( - [ - "Using dnf (RHEL 8+/Fedora):", - " sudo dnf install -y nodejs npm", - "", - "Using yum (RHEL 7):", - " sudo yum install -y nodejs npm", - "", - "Or use NodeSource for newer version:", - " curl -fsSL https://rpm.nodesource.com/setup_24.x | sudo bash -", - " sudo yum install -y nodejs", - ] - ) - else: - lines.extend( - [ - "Use nvm (Node Version Manager) - no sudo needed:", - " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash", - " source ~/.bashrc", - " nvm install 24", - ] - ) - - return lines - - -def _get_alpine_instructions() -> list[str]: - """Instructions for Alpine Linux.""" - return [ - "ALPINE LINUX INSTALLATION:", - "", - "On an Alpine release whose repositories offer Node.js 24, run as root:", - " apk add --no-cache 'nodejs~24' npm", - " node --version", - "", - "If apk cannot find Node.js 24, upgrade Alpine or use the official node:24-alpine container.", - ] - - -def _get_arch_instructions() -> list[str]: - """Instructions for Arch Linux.""" - return [ - "ARCH LINUX INSTALLATION:", - "", - " sudo pacman -S nodejs npm", - ] - - -def _get_suse_instructions() -> list[str]: - """Instructions for SUSE/openSUSE.""" - return [ - "SUSE/OPENSUSE INSTALLATION:", - "", - " sudo zypper install nodejs npm", - ] - - -def _get_generic_linux_instructions() -> list[str]: - """Fallback instructions for unknown Linux distributions.""" - return [ - "LINUX INSTALLATION:", - "", - "Use your package manager to install Node.js, or use nvm:", - "", - "Option 1 - nvm (Node Version Manager, works on any Linux):", - " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash", - " source ~/.bashrc", - " nvm install 24", - "", - "Option 2 - Download binary from https://nodejs.org/", - ] - - -def _get_macos_instructions() -> list[str]: - """Instructions for macOS.""" - return [ - "MACOS INSTALLATION:", - "", - "Option 1 - Homebrew (recommended):", - " brew install node", - "", - "Option 2 - nvm (Node Version Manager, for version management):", - " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash", - " source ~/.zshrc # or ~/.bashrc", - " nvm install 24", - "", - "Option 3 - Official installer:", - " Download from https://nodejs.org/", +def get_installation_instructions(env: Environment) -> str: + """Explain how to install a supported Node.js runtime without guessing package versions.""" + lines = [ + f"ERROR: promptfoo requires Node.js {MIN_NODE_VERSION_TEXT} or newer, but it was not found.", + f"Install Node.js 24 LTS with npm: {_NODE_DOWNLOAD}", + "Verify with: node --version && npx --version", ] + if env.serverless and env.serverless in _SERVERLESS: + name, documentation = _SERVERLESS[env.serverless] + lines += ["", f"{name}: use a deployment that includes both Node.js and Python.", f" {documentation}"] -def _get_windows_instructions() -> list[str]: - """Instructions for Windows.""" - return [ - "WINDOWS INSTALLATION:", - "", - "Option 1 - Official installer (recommended):", - " Download from https://nodejs.org/", - "", - "Option 2 - winget (Windows 10/11, built-in):", - " winget install OpenJS.NodeJS.LTS # For LTS version", - " # or: winget install OpenJS.NodeJS # For current version", - "", - "Option 3 - Chocolatey:", - " choco install nodejs-lts # For LTS version", - " # or: choco install nodejs # For current version", - "", - "Option 4 - Scoop:", - " scoop install nodejs-lts # For LTS version", - " # or: scoop install nodejs # For current version", - ] + if env.ci_platform == "GitHub Actions": + lines += ["", "GITHUB ACTIONS: add Node.js to your workflow:", " - uses: actions/setup-node@v7", " with:"] + lines.append(" node-version: '24'") + elif env.ci_platform: + lines += ["", f"{env.ci_platform}: use your CI provider's Node.js setup step or an image with Node.js 24."] + if env.is_docker: + lines += ["", *_container_instructions(env)] + if env.is_wsl: + lines += ["", "WSL: install Node.js inside your Linux distribution, not on the Windows host."] -def _get_venv_instructions() -> list[str]: - """Instructions for virtual environment users.""" - return [ - "ALTERNATIVE: Install Node.js in your Python virtualenv (no sudo):", - " pip install nodeenv", - " nodeenv -p # Installs Node.js in current virtualenv", - " # Then run promptfoo again", - ] - + if env.os_type == "linux": + lines += ["", *_linux_instructions(env)] + elif env.os_type == "darwin": + lines += ["", "MACOS: install Node.js with Homebrew (`brew install node`) or the installer linked above."] + elif env.os_type == "windows": + lines += ["", "WINDOWS: use the installer linked above or run: winget install OpenJS.NodeJS.LTS"] -def _get_npx_instructions() -> list[str]: - """Instructions for direct npx usage.""" - return [ - "DIRECT USAGE (bypasses Python wrapper):", - " npx promptfoo@latest eval", - " # This is often faster and always uses the latest version", - ] + lines += ["", "DIRECT USAGE after installing Node.js: npx promptfoo@latest eval"] + return "\n".join(lines) diff --git a/tests/test_environment.py b/tests/test_environment.py index e99ed7d..0d27944 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,530 +1,119 @@ -""" -Tests for environment detection. - -This module tests detection of operating systems, Linux distributions, -cloud providers, containers, CI/CD platforms, and Python environments. -""" - -import os from pathlib import Path -from unittest import mock +from unittest.mock import MagicMock import pytest -from promptfoo.environment import ( - _detect_ci, - _detect_cloud_provider, - _detect_container, - _detect_linux_distro, - _detect_python_env, - _detect_wsl, - _has_sudo_access, - _read_probe_file, - detect_environment, +from promptfoo import environment + + +@pytest.fixture(autouse=True) +def isolated_probes(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "GITHUB_ACTIONS", + "GITLAB_CI", + "CIRCLECI", + "JENKINS_URL", + "BUILDKITE", + "TF_BUILD", + "CI", + "AWS_LAMBDA_FUNCTION_NAME", + "FUNCTIONS_WORKER_RUNTIME", + "FUNCTION_TARGET", + "FUNCTION_NAME", + "WSL_DISTRO_NAME", + "WSL_INTEROP", + "KUBERNETES_SERVICE_HOST", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(environment, "_read_probe", lambda path: "") + monkeypatch.setattr(Path, "is_file", lambda path: False) + + +@pytest.mark.parametrize( + "release, expected", + [ + ({"ID": "alpine", "VERSION_ID": "3.24"}, ("alpine", "3.24")), + ({"ID": "derivative", "ID_LIKE": "alpine", "VERSION_ID": "1"}, ("alpine", "1")), + ({"ID": "amzn", "ID_LIKE": "fedora", "VERSION_ID": "2023"}, ("amzn", "2023")), + ({"ID": "amzn", "VERSION_ID": "2"}, ("amzn", "2")), + ({"ID": "ubuntu", "VERSION_ID": "24.04"}, ("ubuntu", "24.04")), + ({"ID": "unknown"}, ("unknown", None)), + ({}, (None, None)), + ], ) +def test_reads_the_standard_linux_release( + monkeypatch: pytest.MonkeyPatch, release: dict[str, str], expected: tuple[str | None, str | None] +) -> None: + monkeypatch.setattr(environment.platform, "freedesktop_os_release", lambda: release) + assert environment._linux_release() == expected -class TestProbeFileReads: - """Test best-effort probe file reads.""" - - def test_read_probe_file_returns_none_when_missing(self, tmp_path: Path) -> None: - """Missing probe files return None.""" - assert _read_probe_file(tmp_path / "missing") is None - - def test_read_probe_file_returns_content_when_readable(self, tmp_path: Path) -> None: - """Readable probe files return their text content.""" - probe_file = tmp_path / "probe" - probe_file.write_text("value") - - assert _read_probe_file(probe_file) == "value" - - def test_read_probe_file_returns_none_when_unreadable(self, tmp_path: Path) -> None: - """Unreadable probe files return None instead of raising.""" - probe_file = tmp_path / "probe" - probe_file.write_text("value") - - with mock.patch("builtins.open", side_effect=OSError("permission denied")): - assert _read_probe_file(probe_file) is None - - -class TestLinuxDistroDetection: - """Test Linux distribution detection.""" - - def test_detect_linux_distro_returns_tuple(self) -> None: - """Linux distro detection returns a tuple.""" - distro, version = _detect_linux_distro() - # Should return tuple even if both None - assert isinstance(distro, (str, type(None))) - assert isinstance(version, (str, type(None))) - - def test_detect_derivative_distro_pop_os(self, tmp_path: Path) -> None: - """Detect Pop!_OS as Ubuntu derivative via ID_LIKE.""" - os_release = tmp_path / "os-release" - os_release.write_text('ID=pop\nVERSION_ID="22.04"\nID_LIKE="ubuntu debian"') - - missing_path = mock.Mock() - missing_path.exists.return_value = False - - with mock.patch("promptfoo.environment.Path") as mock_path_class: - - def path_side_effect(path_str: str) -> object: - if path_str == "/etc/os-release": - return os_release - return missing_path - - mock_path_class.side_effect = path_side_effect - - distro, version = _detect_linux_distro() - assert distro == "ubuntu" # Should resolve to parent via ID_LIKE - assert version == "22.04" - - def test_detect_derivative_distro_raspbian(self, tmp_path: Path) -> None: - """Detect Raspbian as Debian derivative via ID_LIKE.""" - os_release_data = 'ID=raspbian\nVERSION_ID="11"\nID_LIKE=debian' - - with ( - mock.patch("builtins.open", mock.mock_open(read_data=os_release_data)), - mock.patch("promptfoo.environment.Path") as mock_path_class, - ): - mock_path_obj = mock.Mock() - mock_path_obj.exists.return_value = True - mock_path_class.return_value = mock_path_obj - - distro, version = _detect_linux_distro() - assert distro == "debian" # Should resolve to parent via ID_LIKE - assert version == "11" - - def test_detect_derivative_distro_linux_mint(self, tmp_path: Path) -> None: - """Detect Linux Mint as Ubuntu derivative via ID_LIKE.""" - os_release_data = 'ID=linuxmint\nVERSION_ID="21"\nID_LIKE="ubuntu debian"' - - with ( - mock.patch("builtins.open", mock.mock_open(read_data=os_release_data)), - mock.patch("promptfoo.environment.Path") as mock_path_class, - ): - mock_path_obj = mock.Mock() - mock_path_obj.exists.return_value = True - mock_path_class.return_value = mock_path_obj - - distro, version = _detect_linux_distro() - assert distro == "ubuntu" # Should resolve to first known parent in ID_LIKE - assert version == "21" - - def test_usr_lib_os_release_fallback(self, tmp_path: Path) -> None: - """Detect distro from /usr/lib/os-release if /etc/os-release missing.""" - with mock.patch("promptfoo.environment.Path") as mock_path_class: - # Create mock Path objects - etc_path = mock.Mock() - etc_path.exists.return_value = False - etc_path.__str__ = lambda self: "/etc/os-release" - - usr_path = mock.Mock() - usr_path.exists.return_value = True - usr_path.__str__ = lambda self: "/usr/lib/os-release" - - def path_constructor(path_str: str) -> mock.Mock: - if path_str == "/etc/os-release": - return etc_path - elif path_str == "/usr/lib/os-release": - return usr_path - return mock.Mock() - - mock_path_class.side_effect = path_constructor - - with mock.patch("builtins.open", mock.mock_open(read_data='ID=ubuntu\nVERSION_ID="22.04"')): - distro, version = _detect_linux_distro() - assert distro == "ubuntu" - assert version == "22.04" - - def test_detect_linux_distro_skips_unreadable_os_release(self) -> None: - """Unreadable /etc/os-release falls back to /usr/lib/os-release.""" - etc_path = mock.Mock() - etc_path.exists.return_value = True - - usr_path = mock.Mock() - usr_path.exists.return_value = True - - def path_constructor(path_str: str) -> mock.Mock: - if path_str == "/etc/os-release": - return etc_path - elif path_str == "/usr/lib/os-release": - return usr_path - fallback_path = mock.Mock() - fallback_path.exists.return_value = False - return fallback_path - - usr_open = mock.mock_open(read_data='ID=ubuntu\nVERSION_ID="22.04"') - - def open_side_effect(path: mock.Mock) -> mock.MagicMock: - if path is etc_path: - raise OSError("permission denied") - if path is usr_path: - return usr_open() - raise AssertionError(f"unexpected probe path: {path!r}") - - with ( - mock.patch("promptfoo.environment.Path", side_effect=path_constructor), - mock.patch("builtins.open", side_effect=open_side_effect), - ): - distro, version = _detect_linux_distro() - assert distro == "ubuntu" - assert version == "22.04" - - -class TestCloudProviderDetection: - """Test cloud provider detection.""" - - def test_detect_aws_from_hypervisor_uuid(self, tmp_path: Path) -> None: - """Detect AWS from hypervisor UUID.""" - uuid_file = tmp_path / "uuid" - uuid_file.write_text("ec2e1916-9099-7caf-fd21-012345abcdef\n") - - with mock.patch("promptfoo.environment.Path") as mock_path: - mock_path_instance = mock_path.return_value - mock_path_instance.exists.return_value = True - mock_path_instance.__truediv__.return_value = uuid_file - - with mock.patch("builtins.open", mock.mock_open(read_data="ec2e1916-9099-7caf-fd21-012345abcdef\n")): - provider = _detect_cloud_provider() - assert provider == "aws" - - def test_detect_aws_from_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect AWS from environment variables.""" - monkeypatch.setenv("AWS_EXECUTION_ENV", "AWS_Lambda_python3.11") - - with mock.patch("promptfoo.environment.Path") as mock_path: - mock_path.return_value.exists.return_value = False - - provider = _detect_cloud_provider() - assert provider == "aws" - - def test_detect_gcp_from_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect GCP from environment variables.""" - monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "my-project") - - with mock.patch("promptfoo.environment.Path") as mock_path: - mock_path.return_value.exists.return_value = False - - provider = _detect_cloud_provider() - assert provider == "gcp" - - def test_detect_azure_from_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect Azure from environment variables.""" - monkeypatch.setenv("AZURE_SUBSCRIPTION_ID", "12345") - - with mock.patch("promptfoo.environment.Path") as mock_path: - mock_path.return_value.exists.return_value = False +def test_missing_linux_release_does_not_prevent_generic_help(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(environment.platform, "freedesktop_os_release", MagicMock(side_effect=OSError)) + assert environment._linux_release() == (None, None) - provider = _detect_cloud_provider() - assert provider == "azure" - def test_no_cloud_provider_detected(self) -> None: - """Return None when no cloud provider is detected.""" - with mock.patch("promptfoo.environment.Path") as mock_path: - mock_path.return_value.exists.return_value = False - with mock.patch.dict(os.environ, {}, clear=True): - provider = _detect_cloud_provider() - assert provider is None +@pytest.mark.parametrize("platform, expected", [("win32", "windows"), ("darwin", "darwin"), ("freebsd14", "freebsd14")]) +def test_non_linux_platforms_do_not_probe_linux_files( + monkeypatch: pytest.MonkeyPatch, platform: str, expected: str +) -> None: + monkeypatch.setattr(environment.sys, "platform", platform) + probe = MagicMock(side_effect=AssertionError("Linux probe ran on another OS")) + monkeypatch.setattr(environment, "_linux_release", probe) + monkeypatch.setattr(environment, "_in_container", probe) + monkeypatch.setattr(environment, "_read_probe", probe) + assert environment.detect_environment() == environment.Environment(os_type=expected) - def test_detect_cloud_provider_ignores_unreadable_probe_files(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Unreadable cloud metadata files fall back to environment variables.""" - monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "my-project") - path_mock = mock.Mock() - path_mock.exists.return_value = True +def test_kubernetes_is_used_to_show_container_help(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(environment.sys, "platform", "linux") + monkeypatch.setattr(environment.platform, "freedesktop_os_release", lambda: {"ID": "alpine"}) + monkeypatch.setattr(environment, "_read_probe", lambda path: "0::/" if path == "/proc/1/cgroup" else "") + monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.0.0.1") - with ( - mock.patch("promptfoo.environment.Path", return_value=path_mock), - mock.patch("builtins.open", side_effect=OSError("permission denied")), - ): - provider = _detect_cloud_provider() - assert provider == "gcp" + result = environment.detect_environment() + assert result.is_docker + assert result.linux_distro == "alpine" -class TestContainerDetection: - """Test container detection.""" - def test_detect_kubernetes_from_env(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect Kubernetes from environment variable.""" - monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1") +@pytest.mark.parametrize("runtime", ["docker", "containerd", "kubepods", "crio"]) +def test_container_runtime_can_also_be_detected_from_cgroups(monkeypatch: pytest.MonkeyPatch, runtime: str) -> None: + monkeypatch.setattr(environment, "_read_probe", lambda path: f"0::/{runtime}/container") + assert environment._in_container() - with mock.patch("promptfoo.environment.Path") as mock_path: - mock_path.return_value.exists.return_value = False - is_docker, is_k8s = _detect_container() - assert is_docker is False - assert is_k8s is True +def test_docker_marker_works_without_cgroup_names(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "is_file", lambda path: str(path).replace("\\", "/").endswith("/.dockerenv")) + assert environment._in_container() - def test_detect_container_returns_tuple(self) -> None: - """Container detection returns a tuple of booleans.""" - is_docker, is_k8s = _detect_container() - assert isinstance(is_docker, bool) - assert isinstance(is_k8s, bool) - def test_detect_container_ignores_unreadable_cgroup(self) -> None: - """Unreadable cgroup metadata does not raise.""" - - def path_constructor(path_str: str) -> mock.Mock: - path_mock = mock.Mock() - path_mock.exists.return_value = path_str == "/proc/1/cgroup" - return path_mock - - with ( - mock.patch("promptfoo.environment.Path", side_effect=path_constructor), - mock.patch("builtins.open", side_effect=OSError("permission denied")), - mock.patch.dict(os.environ, {}, clear=True), - ): - is_docker, is_k8s = _detect_container() - assert is_docker is False - assert is_k8s is False - - -class TestWSLDetection: - """Test WSL detection.""" - - def test_detect_wsl_from_env_var(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect WSL from WSL_DISTRO_NAME environment variable.""" +@pytest.mark.parametrize("use_environment", [True, False]) +def test_detects_wsl_only_on_linux(monkeypatch: pytest.MonkeyPatch, use_environment: bool) -> None: + monkeypatch.setattr(environment.sys, "platform", "linux") + monkeypatch.setattr(environment.platform, "freedesktop_os_release", lambda: {"ID": "ubuntu"}) + if use_environment: monkeypatch.setenv("WSL_DISTRO_NAME", "Ubuntu") + else: + monkeypatch.setattr(environment, "_read_probe", lambda path: "6.6-Microsoft-standard-WSL2") + assert environment.detect_environment().is_wsl - assert _detect_wsl() is True - - def test_detect_wsl_from_interop_env(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect WSL from WSL_INTEROP environment variable.""" - monkeypatch.setenv("WSL_INTEROP", "/run/WSL/123_interop") - - assert _detect_wsl() is True - - def test_no_wsl_detected(self) -> None: - """Return False when not in WSL.""" - with mock.patch.dict(os.environ, {}, clear=True): - # This will return False unless we're actually in WSL - # Just verify it returns a boolean - result = _detect_wsl() - assert isinstance(result, bool) - - def test_detect_wsl_ignores_unreadable_proc_version(self) -> None: - """Unreadable /proc/version does not raise.""" - - def path_constructor(path_str: str) -> mock.Mock: - path_mock = mock.Mock() - path_mock.exists.return_value = path_str == "/proc/version" - return path_mock - - with ( - mock.patch("promptfoo.environment.Path", side_effect=path_constructor), - mock.patch("builtins.open", side_effect=OSError("permission denied")), - mock.patch.dict(os.environ, {}, clear=True), - ): - assert _detect_wsl() is False - - -class TestCIDetection: - """Test CI/CD platform detection.""" - - @pytest.mark.parametrize( - "env_var,expected_platform", - [ - ("GITHUB_ACTIONS", "github"), - ("GITLAB_CI", "gitlab"), - ("CIRCLECI", "circleci"), - ("JENKINS_HOME", "jenkins"), - ("TRAVIS", "travis"), - ("BUILDKITE", "buildkite"), - ], - ) - def test_detect_specific_ci_platforms( - self, env_var: str, expected_platform: str, monkeypatch: pytest.MonkeyPatch - ) -> None: - """Detect specific CI/CD platforms from environment variables.""" - with mock.patch.dict(os.environ, {}, clear=True): - monkeypatch.setenv(env_var, "true") - is_ci, platform = _detect_ci() - assert is_ci is True - assert platform == expected_platform - - def test_detect_generic_ci(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect generic CI from CI environment variable.""" - with mock.patch.dict(os.environ, {}, clear=True): - monkeypatch.setenv("CI", "true") - is_ci, platform = _detect_ci() - assert is_ci is True - assert platform is None - - def test_no_ci_detected(self) -> None: - """Return False when no CI is detected.""" - with mock.patch.dict(os.environ, {}, clear=True): - is_ci, platform = _detect_ci() - assert is_ci is False - assert platform is None - - -class TestPythonEnvDetection: - """Test Python environment detection.""" - - def test_detect_venv(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect virtualenv from sys.prefix.""" - import sys - - with mock.patch.object(sys, "prefix", "/home/user/venv"), mock.patch.object(sys, "base_prefix", "/usr"): - is_venv, is_conda = _detect_python_env() - assert is_venv is True - - def test_detect_conda(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect conda from environment variable.""" - monkeypatch.setenv("CONDA_DEFAULT_ENV", "base") - is_venv, is_conda = _detect_python_env() - assert is_conda is True - - def test_no_venv_detected(self) -> None: - """Return False when no venv is detected.""" - import sys - - with ( - mock.patch.object(sys, "prefix", "/usr"), - mock.patch.object(sys, "base_prefix", "/usr"), - mock.patch.dict(os.environ, {}, clear=True), - ): - is_venv, is_conda = _detect_python_env() - assert is_venv is False - assert is_conda is False +@pytest.mark.parametrize( + "variable, expected", + [("GITHUB_ACTIONS", "GitHub Actions"), ("GITLAB_CI", "GitLab CI"), ("CIRCLECI", "CircleCI"), ("CI", "CI")], +) +def test_detects_ci_guidance(monkeypatch: pytest.MonkeyPatch, variable: str, expected: str) -> None: + monkeypatch.setenv(variable, "1") + assert environment._ci_platform() == expected -class TestSudoAccess: - """Test sudo access detection.""" - - def test_has_sudo_when_root(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect sudo access when running as root.""" - if hasattr(os, "geteuid"): - with mock.patch("os.geteuid", return_value=0): - assert _has_sudo_access() is True - - def test_has_sudo_when_sudo_command_exists(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect sudo access when sudo command exists.""" - if hasattr(os, "geteuid"): - with ( - mock.patch("os.geteuid", return_value=1000), - mock.patch("shutil.which", return_value="/usr/bin/sudo"), - ): - assert _has_sudo_access() is True - - def test_no_sudo_when_command_missing(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Return False when sudo command doesn't exist.""" - if hasattr(os, "geteuid"): - with mock.patch("os.geteuid", return_value=1000), mock.patch("shutil.which", return_value=None): - assert _has_sudo_access() is False - - -class TestDetectEnvironment: - """Test complete environment detection.""" - - def test_detect_ubuntu_with_docker(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """Detect Ubuntu in Docker container.""" - os_release = tmp_path / "os-release" - os_release.write_text('ID=ubuntu\nVERSION_ID="22.04"') - - with ( - mock.patch("sys.platform", "linux"), - mock.patch("promptfoo.environment._detect_linux_distro", return_value=("ubuntu", "22.04")), - mock.patch("promptfoo.environment._detect_container", return_value=(True, False)), - mock.patch("promptfoo.environment._detect_wsl", return_value=False), - mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), - mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), - mock.patch("promptfoo.environment._detect_python_env", return_value=(True, False)), - mock.patch("promptfoo.environment._has_sudo_access", return_value=False), - ): - env = detect_environment() - - assert env.os_type == "linux" - assert env.linux_distro == "ubuntu" - assert env.linux_distro_version == "22.04" - assert env.is_docker is True - assert env.is_kubernetes is False - assert env.is_wsl is False - assert env.is_venv is True - - def test_detect_macos_environment(self) -> None: - """Detect macOS environment.""" - with ( - mock.patch("sys.platform", "darwin"), - mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), - mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), - mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), - mock.patch("promptfoo.environment._has_sudo_access", return_value=True), - ): - env = detect_environment() - - assert env.os_type == "darwin" - assert env.linux_distro is None - assert env.has_sudo is True - - def test_detect_windows_environment(self) -> None: - """Detect Windows environment.""" - with ( - mock.patch("sys.platform", "win32"), - mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), - mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), - mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), - mock.patch("promptfoo.environment._has_sudo_access", return_value=False), - ): - env = detect_environment() - - assert env.os_type == "windows" - - def test_detect_aws_lambda(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect AWS Lambda environment.""" - monkeypatch.setenv("AWS_LAMBDA_FUNCTION_NAME", "my-function") - - with ( - mock.patch("sys.platform", "linux"), - mock.patch("promptfoo.environment._detect_linux_distro", return_value=("amzn", "2")), - mock.patch("promptfoo.environment._detect_container", return_value=(False, False)), - mock.patch("promptfoo.environment._detect_wsl", return_value=False), - mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), - mock.patch("promptfoo.environment._detect_cloud_provider", return_value="aws"), - mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), - mock.patch("promptfoo.environment._has_sudo_access", return_value=False), - ): - env = detect_environment() - - assert env.is_lambda is True - assert env.cloud_provider == "aws" - - def test_detect_github_actions(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect GitHub Actions environment.""" - monkeypatch.setenv("GITHUB_ACTIONS", "true") - - with ( - mock.patch("sys.platform", "linux"), - mock.patch("promptfoo.environment._detect_linux_distro", return_value=("ubuntu", "22.04")), - mock.patch("promptfoo.environment._detect_container", return_value=(False, False)), - mock.patch("promptfoo.environment._detect_wsl", return_value=False), - mock.patch("promptfoo.environment._detect_ci", return_value=(True, "github")), - mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), - mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), - mock.patch("promptfoo.environment._has_sudo_access", return_value=True), - ): - env = detect_environment() - - assert env.is_ci is True - assert env.ci_platform == "github" - - def test_detect_wsl_ubuntu(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect WSL with Ubuntu.""" - monkeypatch.setenv("WSL_DISTRO_NAME", "Ubuntu") - - with ( - mock.patch("sys.platform", "linux"), - mock.patch("promptfoo.environment._detect_linux_distro", return_value=("ubuntu", "22.04")), - mock.patch("promptfoo.environment._detect_container", return_value=(False, False)), - mock.patch("promptfoo.environment._detect_wsl", return_value=True), - mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), - mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), - mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), - mock.patch("promptfoo.environment._has_sudo_access", return_value=True), - ): - env = detect_environment() - assert env.os_type == "linux" - assert env.linux_distro == "ubuntu" - assert env.is_wsl is True - assert env.is_docker is False +@pytest.mark.parametrize( + "variable, expected", + [("AWS_LAMBDA_FUNCTION_NAME", "aws"), ("FUNCTIONS_WORKER_RUNTIME", "azure"), ("FUNCTION_TARGET", "google")], +) +def test_detects_serverless_from_provider_supplied_variables( + monkeypatch: pytest.MonkeyPatch, variable: str, expected: str +) -> None: + monkeypatch.setenv(variable, "configured") + assert environment._serverless() == expected diff --git a/tests/test_instructions.py b/tests/test_instructions.py index c6ec6a0..366fe1c 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -1,456 +1,93 @@ -""" -Tests for platform-specific installation instructions. - -This module tests that appropriate instructions are generated for -different platforms and environments. -""" - import pytest from promptfoo.environment import Environment from promptfoo.instructions import get_installation_instructions +from promptfoo.node import MIN_NODE_VERSION_TEXT -@pytest.mark.parametrize( - ("env", "recommended"), - [ - (Environment(os_type="linux", is_lambda=True), "node --version"), - (Environment(os_type="linux", is_cloud_function=True, cloud_provider="gcp"), "node --version"), - (Environment(os_type="linux", is_ci=True, ci_platform="github"), "node-version: '24'"), - (Environment(os_type="linux", is_ci=True, ci_platform="gitlab"), "image: node:24"), - (Environment(os_type="linux", is_ci=True, ci_platform="circleci"), "node-version: '24'"), - (Environment(os_type="linux", linux_distro="alpine"), "apk add --no-cache 'nodejs~24' npm"), - (Environment(os_type="linux", linux_distro="alpine", is_docker=True), "FROM node:24-alpine"), - (Environment(os_type="linux", linux_distro="ubuntu", is_docker=True), "setup_24.x"), - (Environment(os_type="linux", is_docker=True), "FROM node:24-bookworm-slim"), - (Environment(os_type="linux", is_wsl=True), "nvm install 24"), - (Environment(os_type="linux", linux_distro="rhel", has_sudo=True), "setup_24.x"), - (Environment(os_type="darwin"), "nvm install 24"), - (Environment(os_type="windows"), "OpenJS.NodeJS.LTS"), - ], -) -def test_installation_help_requires_supported_node(env: Environment, recommended: str) -> None: - """Every platform states the exact minimum and versioned examples install a supported runtime.""" - instructions = get_installation_instructions(env) - - assert "requires Node.js 22.22.0 or newer" in instructions - assert recommended in instructions - assert "nvm install 20" not in instructions - assert "setup_20.x" not in instructions - assert "node-version: '20'" not in instructions - assert "image: node:20" not in instructions - - -class TestLambdaInstructions: - """Test instructions for AWS Lambda.""" - - def test_lambda_instructions(self) -> None: - """Generate Lambda-specific instructions.""" - env = Environment( - os_type="linux", - linux_distro="rhel", - cloud_provider="aws", - is_lambda=True, - ) - - instructions = get_installation_instructions(env) - - assert "AWS Lambda" in instructions - assert "Lambda Layer" in instructions - assert "Node.js runtime" in instructions - - -class TestCloudFunctionInstructions: - """Test instructions for Cloud Functions.""" - - def test_gcp_cloud_function_instructions(self) -> None: - """Generate GCP Cloud Functions instructions.""" - env = Environment( - os_type="linux", - cloud_provider="gcp", - is_cloud_function=True, - ) - - instructions = get_installation_instructions(env) - - assert "Google Cloud Functions" in instructions or "GCP" in instructions - - def test_azure_function_instructions(self) -> None: - """Generate Azure Functions instructions.""" - env = Environment( - os_type="linux", - cloud_provider="azure", - is_cloud_function=True, - ) - - instructions = get_installation_instructions(env) - - assert "Azure Functions" in instructions - - -class TestCIInstructions: - """Test instructions for CI/CD environments.""" - - def test_github_actions_instructions(self) -> None: - """Generate GitHub Actions-specific instructions.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_ci=True, - ci_platform="github", - ) - - instructions = get_installation_instructions(env) - - assert "actions/setup-node@v7" in instructions - assert "node-version: '24'" in instructions - assert "GITHUB" in instructions.upper() - - def test_gitlab_ci_instructions(self) -> None: - """Generate GitLab CI instructions.""" - env = Environment( - os_type="linux", - is_ci=True, - ci_platform="gitlab", - ) - - instructions = get_installation_instructions(env) - - assert "gitlab" in instructions.lower() or "GITLAB" in instructions - assert "image:" in instructions or "before_script" in instructions - - def test_circleci_instructions(self) -> None: - """Generate CircleCI instructions.""" - env = Environment( - os_type="linux", - is_ci=True, - ci_platform="circleci", - ) - - instructions = get_installation_instructions(env) - - assert "circleci" in instructions.lower() or "CIRCLECI" in instructions - - -class TestDockerInstructions: - """Test instructions for Docker containers.""" - - def test_docker_alpine_instructions(self) -> None: - """Generate Docker instructions for Alpine.""" - env = Environment( - os_type="linux", - linux_distro="alpine", - is_docker=True, - ) - - instructions = get_installation_instructions(env) - - assert "FROM node:24-alpine" in instructions - assert "RUN apk add --no-cache python3 py3-pip" in instructions - assert "python3 -m venv /opt/venv" in instructions - assert "apk add --no-cache nodejs npm" not in instructions - assert "Dockerfile" in instructions - - def test_docker_ubuntu_instructions(self) -> None: - """Generate Docker instructions for Ubuntu.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_docker=True, - ) - - instructions = get_installation_instructions(env) - - assert "apt-get" in instructions - assert "Dockerfile" in instructions - - -class TestWSLInstructions: - """Test instructions for WSL (Windows Subsystem for Linux).""" - - def test_wsl_instructions(self) -> None: - """Generate WSL-specific instructions.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_wsl=True, - ) - - instructions = get_installation_instructions(env) - - assert "WSL" in instructions or "Windows Subsystem for Linux" in instructions - assert "nvm" in instructions - assert "/mnt/c" in instructions # Should mention Windows filesystem - assert "performance" in instructions.lower() - - def test_wsl_with_ubuntu_shows_both(self) -> None: - """WSL instructions should show both WSL tips and Ubuntu instructions.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_wsl=True, - has_sudo=True, - ) - - instructions = get_installation_instructions(env) - - # Should have WSL-specific guidance - assert "WSL" in instructions - # Should also have Ubuntu/Debian instructions - assert "UBUNTU" in instructions or "DEBIAN" in instructions - - -class TestLinuxInstructions: - """Test instructions for various Linux distributions.""" - - def test_ubuntu_instructions_with_sudo(self) -> None: - """Generate Ubuntu instructions with sudo access.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - has_sudo=True, - ) - - instructions = get_installation_instructions(env) +@pytest.mark.parametrize("platform", ["linux", "darwin", "windows", "freebsd14"]) +def test_every_platform_gets_the_runtime_requirement_and_a_working_fallback(platform: str) -> None: + output = get_installation_instructions(Environment(os_type=platform)) - assert "UBUNTU/DEBIAN" in instructions - assert "sudo apt" in instructions - assert "NodeSource" in instructions + assert f"requires Node.js {MIN_NODE_VERSION_TEXT} or newer" in output + assert "Node.js 24 LTS with npm" in output + assert "https://nodejs.org/en/download" in output + assert "node --version && npx --version" in output + assert "DIRECT USAGE after installing Node.js: npx promptfoo@latest eval" in output - def test_ubuntu_instructions_without_sudo(self) -> None: - """Generate Ubuntu instructions without sudo access.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - has_sudo=False, - ) - instructions = get_installation_instructions(env) - - assert "nvm" in instructions - # Should NOT suggest sudo apt commands when user doesn't have sudo - assert "sudo apt" not in instructions - assert "sudo snap" not in instructions - - def test_debian_instructions(self) -> None: - """Generate Debian instructions.""" - env = Environment( - os_type="linux", - linux_distro="debian", - has_sudo=True, - ) - - instructions = get_installation_instructions(env) - - assert "UBUNTU/DEBIAN" in instructions - assert "apt" in instructions - - def test_rhel_instructions_with_sudo(self) -> None: - """Generate RHEL instructions with sudo.""" - env = Environment( - os_type="linux", - linux_distro="rhel", - has_sudo=True, - ) - - instructions = get_installation_instructions(env) - - assert "RHEL" in instructions or "CENTOS" in instructions or "FEDORA" in instructions - assert "dnf" in instructions or "yum" in instructions - - def test_amazon_linux_instructions(self) -> None: - """Generate Amazon Linux instructions.""" - env = Environment( - os_type="linux", - linux_distro="rhel", - linux_distro_version="2023", - has_sudo=True, - ) - - instructions = get_installation_instructions(env) - - assert "AMAZON LINUX" in instructions - assert "dnf" in instructions or "yum" in instructions - - def test_alpine_instructions(self) -> None: - """Generate Alpine Linux instructions.""" - env = Environment( - os_type="linux", - linux_distro="alpine", - linux_distro_version="3.20", - ) - - instructions = get_installation_instructions(env) - - assert "ALPINE" in instructions - assert "apk add --no-cache 'nodejs~24' npm" in instructions - assert "upgrade Alpine or use the official node:24-alpine container" in instructions - assert "node --version" in instructions - assert "apk add --update nodejs npm" not in instructions - assert "apk add --no-cache nodejs npm" not in instructions - - def test_arch_instructions(self) -> None: - """Generate Arch Linux instructions.""" - env = Environment( - os_type="linux", - linux_distro="arch", - ) - - instructions = get_installation_instructions(env) - - assert "ARCH" in instructions - assert "pacman" in instructions - - def test_suse_instructions(self) -> None: - """Generate SUSE instructions.""" - env = Environment( - os_type="linux", - linux_distro="suse", - ) - - instructions = get_installation_instructions(env) - - assert "SUSE" in instructions or "OPENSUSE" in instructions - assert "zypper" in instructions - - def test_generic_linux_instructions(self) -> None: - """Generate generic Linux instructions for unknown distro.""" - env = Environment( - os_type="linux", - linux_distro="unknown", - ) - - instructions = get_installation_instructions(env) - - assert "nvm" in instructions - - -class TestMacOSInstructions: - """Test instructions for macOS.""" - - def test_macos_instructions(self) -> None: - """Generate macOS instructions.""" - env = Environment(os_type="darwin") - - instructions = get_installation_instructions(env) - - assert "MACOS" in instructions - assert "brew install node" in instructions - assert "Official installer" in instructions - assert "nvm" in instructions - assert "nodejs.org" in instructions - - -class TestWindowsInstructions: - """Test instructions for Windows.""" - - def test_windows_instructions(self) -> None: - """Generate Windows instructions.""" - env = Environment(os_type="windows") - - instructions = get_installation_instructions(env) - - assert "WINDOWS" in instructions - assert "winget" in instructions - assert "Chocolatey" in instructions or "choco" in instructions - assert "Scoop" in instructions - - -class TestVenvInstructions: - """Test virtual environment instructions.""" - - def test_venv_instructions_included(self) -> None: - """Include venv instructions when in virtualenv.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_venv=True, - ) - - instructions = get_installation_instructions(env) - - assert "nodeenv" in instructions - assert "virtualenv" in instructions.lower() - - def test_conda_instructions_included(self) -> None: - """Include venv instructions when in conda.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_conda=True, - ) - - instructions = get_installation_instructions(env) - - assert "nodeenv" in instructions - - -class TestNpxInstructions: - """Test npx direct usage instructions.""" - - def test_npx_instructions_always_included(self) -> None: - """NPX instructions should always be included.""" - env = Environment(os_type="linux", linux_distro="ubuntu") - - instructions = get_installation_instructions(env) - - assert "npx promptfoo@latest" in instructions - assert "DIRECT USAGE" in instructions +@pytest.mark.parametrize( + "platform, hint", + [("linux", "nvm install 24"), ("darwin", "brew install node"), ("windows", "winget install OpenJS.NodeJS.LTS")], +) +def test_common_platforms_get_one_relevant_installation_hint(platform: str, hint: str) -> None: + assert hint in get_installation_instructions(Environment(os_type=platform)) -class TestErrorMessageFormat: - """Test error message formatting.""" +def test_alpine_dockerfile_contains_only_the_verified_setup_steps() -> None: + output = get_installation_instructions(Environment(os_type="linux", linux_distro="alpine", is_docker=True)) + commands = [line.strip() for line in output.splitlines() if line.strip().startswith(("FROM ", "RUN ", "ENV "))] - def test_error_message_has_clear_header(self) -> None: - """Error message should have a clear header.""" - env = Environment(os_type="linux", linux_distro="ubuntu") + assert commands == [ + "FROM node:24-alpine", + "RUN apk add --no-cache python3 py3-pip", + "RUN python3 -m venv /opt/venv", + 'ENV PATH="/opt/venv/bin:$PATH"', + ] + assert "apk add --no-cache 'nodejs~24' npm" in output + assert "upgrade Alpine or use the official node:24-alpine container" in output + assert "apk add --no-cache nodejs npm" not in output - instructions = get_installation_instructions(env) - assert "ERROR: promptfoo requires Node.js" in instructions - assert "=" * 70 in instructions +def test_amazon_linux_2023_installs_both_versioned_packages_and_selects_the_active_node() -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="amzn", linux_distro_version="2023") + ) - def test_multiline_output(self) -> None: - """Instructions should be multi-line.""" - env = Environment(os_type="linux", linux_distro="ubuntu") + assert "sudo dnf install -y nodejs24 nodejs24-npm" in output + assert "sudo alternatives --set node /usr/bin/node-24" in output + assert "omit sudo when running as root" in output + assert "https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html" in output + assert "dnf install -y nodejs\n" not in output - instructions = get_installation_instructions(env) - lines = instructions.split("\n") - assert len(lines) > 5 # Should have multiple lines +@pytest.mark.parametrize("version", [None, "2"]) +def test_other_amazon_releases_are_not_given_amazon_linux_2023_commands(version: str | None) -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="amzn", linux_distro_version=version) + ) + assert "nvm install 24" in output + assert "nodejs24-npm" not in output -class TestComplexEnvironments: - """Test instructions for complex, combined environments.""" +def test_ci_container_and_wsl_hints_can_coexist() -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="ubuntu", ci_platform="GitHub Actions", is_docker=True, is_wsl=True) + ) - def test_docker_github_actions_ubuntu(self) -> None: - """Generate instructions for Docker in GitHub Actions on Ubuntu.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_docker=True, - is_ci=True, - ci_platform="github", - ) + assert "- uses: actions/setup-node@v7\n with:\n node-version: '24'" in output + assert "keep the Node and Python base distributions compatible" in output + assert "https://hub.docker.com/_/node" in output + assert "install Node.js inside your Linux distribution" in output - instructions = get_installation_instructions(env) - # Should include both CI and Docker instructions - assert "GITHUB" in instructions.upper() - assert "DOCKER" in instructions.upper() +def test_other_ci_uses_the_provider_setup_instructions() -> None: + output = get_installation_instructions(Environment(os_type="linux", ci_platform="GitLab CI")) + assert "GitLab CI: use your CI provider's Node.js setup step or an image with Node.js 24." in output + assert "actions/setup-node" not in output - def test_aws_ec2_rhel_with_venv(self) -> None: - """Generate instructions for AWS EC2 RHEL with virtualenv.""" - env = Environment( - os_type="linux", - linux_distro="rhel", - cloud_provider="aws", - is_venv=True, - has_sudo=True, - ) - instructions = get_installation_instructions(env) +@pytest.mark.parametrize( + "provider, label, host", + [ + ("aws", "AWS Lambda", "docs.aws.amazon.com"), + ("google", "Google Cloud Functions", "cloud.google.com"), + ("azure", "Azure Functions", "learn.microsoft.com"), + ], +) +def test_serverless_links_explain_the_need_for_both_runtimes(provider: str, label: str, host: str) -> None: + output = get_installation_instructions(Environment(os_type="linux", serverless=provider)) - # Should include RHEL and venv instructions - assert "RHEL" in instructions or "CENTOS" in instructions or "FEDORA" in instructions - assert "nodeenv" in instructions + assert f"{label}: use a deployment that includes both Node.js and Python" in output + assert host in output + assert "https://nodejs.org/en/download" in output From d1f68087bc43da20a6769132dfb0b021ba987d9e Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 18 Sep 2026 13:30:41 -0700 Subject: [PATCH 02/12] fix: make runtime guidance usable across platforms --- src/promptfoo/environment.py | 6 ++++++ src/promptfoo/instructions.py | 22 ++++++++++++++++++---- tests/test_environment.py | 19 ++++++++++++++++++- tests/test_instructions.py | 33 ++++++++++++++++++++++++--------- 4 files changed, 66 insertions(+), 14 deletions(-) diff --git a/src/promptfoo/environment.py b/src/promptfoo/environment.py index ac8c44f..9bf88a5 100644 --- a/src/promptfoo/environment.py +++ b/src/promptfoo/environment.py @@ -56,8 +56,14 @@ def _ci_platform() -> str | None: ("GITLAB_CI", "GitLab CI"), ("CIRCLECI", "CircleCI"), ("JENKINS_URL", "Jenkins"), + ("JENKINS_HOME", "Jenkins"), ("BUILDKITE", "Buildkite"), ("TF_BUILD", "Azure Pipelines"), + ("TEAMCITY_VERSION", "TeamCity"), + ("TRAVIS", "Travis CI"), + ("DRONE", "Drone CI"), + ("BITBUCKET_BUILD_NUMBER", "Bitbucket Pipelines"), + ("CONTINUOUS_INTEGRATION", "CI"), ("CI", "CI"), ): if os.environ.get(variable): diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index e14dcf6..8feb395 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -7,8 +7,12 @@ _NVM = "https://github.com/nvm-sh/nvm#installing-and-updating" _SERVERLESS = { "aws": ("AWS Lambda", "https://docs.aws.amazon.com/lambda/latest/dg/images-create.html"), - "google": ("Google Cloud Functions", "https://cloud.google.com/run/docs/runtimes/nodejs"), - "azure": ("Azure Functions", "https://learn.microsoft.com/en-us/azure/azure-functions/functions-reference-node"), + "google": ("Google Cloud Functions / Cloud Run", "https://docs.cloud.google.com/run/docs/building/containers"), + "azure": ( + "Azure Functions", + "https://learn.microsoft.com/en-us/azure/azure-functions/functions-how-to-custom-container" + "?pivots=programming-language-python", + ), } @@ -42,6 +46,8 @@ def _linux_instructions(env: Environment) -> list[str]: " sudo alternatives --set node /usr/bin/node-24", " node --version", " https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html", + f" Without sudo, install nvm: {_NVM}", + " Then run: nvm install 24", ] return [ "LINUX: use a Node.js version manager or your distribution's instructions for Node.js 24.", @@ -55,12 +61,20 @@ def get_installation_instructions(env: Environment) -> str: lines = [ f"ERROR: promptfoo requires Node.js {MIN_NODE_VERSION_TEXT} or newer, but it was not found.", f"Install Node.js 24 LTS with npm: {_NODE_DOWNLOAD}", - "Verify with: node --version && npx --version", + "Verify with:", + " node --version", + " npx --version", ] if env.serverless and env.serverless in _SERVERLESS: name, documentation = _SERVERLESS[env.serverless] - lines += ["", f"{name}: use a deployment that includes both Node.js and Python.", f" {documentation}"] + lines += [ + "", + f"{name}: build and deploy a custom container that includes both Node.js and Python.", + "Install both runtimes when building the image; they cannot be installed in the running function.", + f" {documentation}", + ] + return "\n".join(lines) if env.ci_platform == "GitHub Actions": lines += ["", "GITHUB ACTIONS: add Node.js to your workflow:", " - uses: actions/setup-node@v7", " with:"] diff --git a/tests/test_environment.py b/tests/test_environment.py index 0d27944..92e05aa 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -13,8 +13,14 @@ def isolated_probes(monkeypatch: pytest.MonkeyPatch) -> None: "GITLAB_CI", "CIRCLECI", "JENKINS_URL", + "JENKINS_HOME", "BUILDKITE", "TF_BUILD", + "TEAMCITY_VERSION", + "TRAVIS", + "DRONE", + "BITBUCKET_BUILD_NUMBER", + "CONTINUOUS_INTEGRATION", "CI", "AWS_LAMBDA_FUNCTION_NAME", "FUNCTIONS_WORKER_RUNTIME", @@ -101,7 +107,18 @@ def test_detects_wsl_only_on_linux(monkeypatch: pytest.MonkeyPatch, use_environm @pytest.mark.parametrize( "variable, expected", - [("GITHUB_ACTIONS", "GitHub Actions"), ("GITLAB_CI", "GitLab CI"), ("CIRCLECI", "CircleCI"), ("CI", "CI")], + [ + ("GITHUB_ACTIONS", "GitHub Actions"), + ("GITLAB_CI", "GitLab CI"), + ("CIRCLECI", "CircleCI"), + ("JENKINS_HOME", "Jenkins"), + ("TEAMCITY_VERSION", "TeamCity"), + ("TRAVIS", "Travis CI"), + ("DRONE", "Drone CI"), + ("BITBUCKET_BUILD_NUMBER", "Bitbucket Pipelines"), + ("CONTINUOUS_INTEGRATION", "CI"), + ("CI", "CI"), + ], ) def test_detects_ci_guidance(monkeypatch: pytest.MonkeyPatch, variable: str, expected: str) -> None: monkeypatch.setenv(variable, "1") diff --git a/tests/test_instructions.py b/tests/test_instructions.py index 366fe1c..a9b18f5 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -12,7 +12,8 @@ def test_every_platform_gets_the_runtime_requirement_and_a_working_fallback(plat assert f"requires Node.js {MIN_NODE_VERSION_TEXT} or newer" in output assert "Node.js 24 LTS with npm" in output assert "https://nodejs.org/en/download" in output - assert "node --version && npx --version" in output + assert " node --version\n npx --version" in output + assert "&&" not in output assert "DIRECT USAGE after installing Node.js: npx promptfoo@latest eval" in output @@ -48,6 +49,8 @@ def test_amazon_linux_2023_installs_both_versioned_packages_and_selects_the_acti assert "sudo alternatives --set node /usr/bin/node-24" in output assert "omit sudo when running as root" in output assert "https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html" in output + assert "Without sudo, install nvm: https://github.com/nvm-sh/nvm#installing-and-updating" in output + assert "nvm install 24" in output assert "dnf install -y nodejs\n" not in output @@ -78,16 +81,28 @@ def test_other_ci_uses_the_provider_setup_instructions() -> None: @pytest.mark.parametrize( - "provider, label, host", + "provider, label, documentation", [ - ("aws", "AWS Lambda", "docs.aws.amazon.com"), - ("google", "Google Cloud Functions", "cloud.google.com"), - ("azure", "Azure Functions", "learn.microsoft.com"), + ("aws", "AWS Lambda", "docs.aws.amazon.com/lambda/latest/dg/images-create.html"), + ("google", "Google Cloud Functions / Cloud Run", "docs.cloud.google.com/run/docs/building/containers"), + ( + "azure", + "Azure Functions", + "learn.microsoft.com/en-us/azure/azure-functions/functions-how-to-custom-container", + ), ], ) -def test_serverless_links_explain_the_need_for_both_runtimes(provider: str, label: str, host: str) -> None: - output = get_installation_instructions(Environment(os_type="linux", serverless=provider)) +def test_serverless_links_explain_how_to_build_both_runtimes(provider: str, label: str, documentation: str) -> None: + output = get_installation_instructions( + Environment( + os_type="linux", linux_distro="amzn", linux_distro_version="2023", is_docker=True, serverless=provider + ) + ) - assert f"{label}: use a deployment that includes both Node.js and Python" in output - assert host in output + assert f"{label}: build and deploy a custom container that includes both Node.js and Python" in output + assert documentation in output assert "https://nodejs.org/en/download" in output + assert "when building the image" in output + assert "sudo" not in output + assert "nvm" not in output + assert "npx promptfoo" not in output From f573df121d1abcd0bb5b6cdbe0dc8c046f91752c Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 18 Sep 2026 13:36:31 -0700 Subject: [PATCH 03/12] fix: distinguish hosted functions and test runtime guidance --- .github/workflows/test.yml | 32 ++++++++++++-------- src/promptfoo/environment.py | 6 ++-- src/promptfoo/instructions.py | 29 ++++++++++++++---- tests/test_environment.py | 33 +++++++++++++++++--- tests/test_instructions.py | 57 ++++++++++++++++++++++++++++++----- 5 files changed, 125 insertions(+), 32 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46a309d..177bf05 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,28 +71,34 @@ jobs: - name: Check GitHub workflows run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 - - name: Verify Alpine installation instructions + - name: Verify container installation instructions run: | - alpine_dir="$RUNNER_TEMP/promptfoo-alpine" - mkdir -p "$alpine_dir" - uv run python - <<'PY' > "$alpine_dir/Dockerfile" + instruction_dir="$RUNNER_TEMP/promptfoo-docker-help" + uv run python - "$instruction_dir" <<'PY' + import sys + from pathlib import Path + from promptfoo.environment import Environment from promptfoo.instructions import get_installation_instructions - instructions = get_installation_instructions( - Environment(os_type="linux", linux_distro="alpine", is_docker=True) - ) - for line in instructions.splitlines(): - command = line.strip() - if command.startswith(("FROM ", "RUN ", "ENV ")): - print(command) + for distro in ("alpine", "debian"): + instructions = get_installation_instructions( + Environment(os_type="linux", linux_distro=distro, is_docker=True) + ) + commands = [line.strip() for line in instructions.splitlines()] + commands = [command for command in commands if command.startswith(("FROM ", "RUN ", "ENV "))] + directory = Path(sys.argv[1]) / distro + directory.mkdir(parents=True, exist_ok=True) + (directory / "Dockerfile").write_text("\n".join(commands) + "\n") PY - cat >> "$alpine_dir/Dockerfile" <<'EOF' + for distro in alpine debian; do + cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' RUN node --version | grep -E '^v24\.' \ && npm --version && npx --version && python --version && pip --version \ && python -c 'import sys; assert sys.prefix == "/opt/venv"' EOF - docker build --progress=plain "$alpine_dir" + docker build --progress=plain "$instruction_dir/$distro" + done docker run --rm alpine:3.20 sh -ec ' apk update apk add --no-cache nodejs npm diff --git a/src/promptfoo/environment.py b/src/promptfoo/environment.py index 9bf88a5..63f0229 100644 --- a/src/promptfoo/environment.py +++ b/src/promptfoo/environment.py @@ -74,9 +74,11 @@ def _ci_platform() -> str | None: def _serverless() -> str | None: if os.environ.get("AWS_LAMBDA_FUNCTION_NAME"): return "aws" - if os.environ.get("FUNCTIONS_WORKER_RUNTIME"): + if os.environ.get("FUNCTIONS_WORKER_RUNTIME") and any( + os.environ.get(name) for name in ("WEBSITE_INSTANCE_ID", "WEBSITE_SITE_NAME", "CONTAINER_APP_NAME") + ): return "azure" - if os.environ.get("FUNCTION_TARGET") or os.environ.get("FUNCTION_NAME"): + if os.environ.get("FUNCTION_NAME") or (os.environ.get("FUNCTION_TARGET") and os.environ.get("K_SERVICE")): return "google" return None diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index 8feb395..7f41f18 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -6,10 +6,21 @@ _NODE_DOWNLOAD = "https://nodejs.org/en/download" _NVM = "https://github.com/nvm-sh/nvm#installing-and-updating" _SERVERLESS = { - "aws": ("AWS Lambda", "https://docs.aws.amazon.com/lambda/latest/dg/images-create.html"), - "google": ("Google Cloud Functions / Cloud Run", "https://docs.cloud.google.com/run/docs/building/containers"), + "aws": ( + "AWS Lambda", + "For an existing zip-based function, create a new image-based function.", + "https://docs.aws.amazon.com/lambda/latest/dg/images-create.html", + ), + "google": ( + "Google Cloud Functions / Cloud Run", + "Functions deployed from source, including first-generation functions, must move to a Cloud Run service " + "to deploy a custom image.", + "https://docs.cloud.google.com/run/docs/building/containers", + ), "azure": ( "Azure Functions", + "Consumption and Flex Consumption do not accept custom images. Move to Azure Container Apps " + "or a Linux Premium/Dedicated plan.", "https://learn.microsoft.com/en-us/azure/azure-functions/functions-how-to-custom-container" "?pivots=programming-language-python", ), @@ -26,7 +37,12 @@ def _container_instructions(env: Environment) -> list[str]: ' ENV PATH="/opt/venv/bin:$PATH"', ] return [ - "CONTAINER: include Node.js 24 in your image; keep the Node and Python base distributions compatible.", + "CONTAINER: start from the official Node.js 24 Debian image and add Python:", + " FROM node:24-bookworm-slim", + " RUN apt-get update && apt-get install -y --no-install-recommends python3 python3-venv " + "&& rm -rf /var/lib/apt/lists/*", + " RUN python3 -m venv /opt/venv", + ' ENV PATH="/opt/venv/bin:$PATH"', " Official Node.js images: https://hub.docker.com/_/node", ] @@ -63,14 +79,15 @@ def get_installation_instructions(env: Environment) -> str: f"Install Node.js 24 LTS with npm: {_NODE_DOWNLOAD}", "Verify with:", " node --version", - " npx --version", + " npx.cmd --version" if env.os_type == "windows" else " npx --version", ] if env.serverless and env.serverless in _SERVERLESS: - name, documentation = _SERVERLESS[env.serverless] + name, hosting, documentation = _SERVERLESS[env.serverless] lines += [ "", f"{name}: build and deploy a custom container that includes both Node.js and Python.", + hosting, "Install both runtimes when building the image; they cannot be installed in the running function.", f" {documentation}", ] @@ -87,7 +104,7 @@ def get_installation_instructions(env: Environment) -> str: if env.is_wsl: lines += ["", "WSL: install Node.js inside your Linux distribution, not on the Windows host."] - if env.os_type == "linux": + if env.os_type == "linux" and (not env.is_docker or env.linux_distro == "alpine"): lines += ["", *_linux_instructions(env)] elif env.os_type == "darwin": lines += ["", "MACOS: install Node.js with Homebrew (`brew install node`) or the installer linked above."] diff --git a/tests/test_environment.py b/tests/test_environment.py index 92e05aa..73f77e3 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -24,8 +24,12 @@ def isolated_probes(monkeypatch: pytest.MonkeyPatch) -> None: "CI", "AWS_LAMBDA_FUNCTION_NAME", "FUNCTIONS_WORKER_RUNTIME", + "WEBSITE_INSTANCE_ID", + "WEBSITE_SITE_NAME", + "CONTAINER_APP_NAME", "FUNCTION_TARGET", "FUNCTION_NAME", + "K_SERVICE", "WSL_DISTRO_NAME", "WSL_INTEROP", "KUBERNETES_SERVICE_HOST", @@ -126,11 +130,32 @@ def test_detects_ci_guidance(monkeypatch: pytest.MonkeyPatch, variable: str, exp @pytest.mark.parametrize( - "variable, expected", - [("AWS_LAMBDA_FUNCTION_NAME", "aws"), ("FUNCTIONS_WORKER_RUNTIME", "azure"), ("FUNCTION_TARGET", "google")], + "variables, expected", + [ + (["AWS_LAMBDA_FUNCTION_NAME"], "aws"), + (["FUNCTIONS_WORKER_RUNTIME", "WEBSITE_INSTANCE_ID"], "azure"), + (["FUNCTIONS_WORKER_RUNTIME", "CONTAINER_APP_NAME"], "azure"), + (["FUNCTION_TARGET", "K_SERVICE"], "google"), + (["FUNCTION_NAME"], "google"), + ], ) def test_detects_serverless_from_provider_supplied_variables( - monkeypatch: pytest.MonkeyPatch, variable: str, expected: str + monkeypatch: pytest.MonkeyPatch, variables: list[str], expected: str ) -> None: - monkeypatch.setenv(variable, "configured") + for variable in variables: + monkeypatch.setenv(variable, "configured") assert environment._serverless() == expected + + +@pytest.mark.parametrize("variable", ["FUNCTION_TARGET", "FUNCTIONS_WORKER_RUNTIME", "K_SERVICE"]) +def test_local_function_framework_settings_do_not_suppress_host_guidance( + monkeypatch: pytest.MonkeyPatch, variable: str +) -> None: + monkeypatch.setattr(environment.sys, "platform", "darwin") + monkeypatch.setenv(variable, "configured") + + from promptfoo.instructions import get_installation_instructions + + output = get_installation_instructions(environment.detect_environment()) + assert "brew install node" in output + assert "custom container" not in output diff --git a/tests/test_instructions.py b/tests/test_instructions.py index a9b18f5..20dd0a7 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -1,3 +1,8 @@ +import re +import shutil +import subprocess +import sys + import pytest from promptfoo.environment import Environment @@ -12,8 +17,9 @@ def test_every_platform_gets_the_runtime_requirement_and_a_working_fallback(plat assert f"requires Node.js {MIN_NODE_VERSION_TEXT} or newer" in output assert "Node.js 24 LTS with npm" in output assert "https://nodejs.org/en/download" in output - assert " node --version\n npx --version" in output - assert "&&" not in output + npx = "npx.cmd" if platform == "windows" else "npx" + assert f" node --version\n {npx} --version" in output + assert "node --version &&" not in output assert "DIRECT USAGE after installing Node.js: npx promptfoo@latest eval" in output @@ -25,6 +31,26 @@ def test_common_platforms_get_one_relevant_installation_hint(platform: str, hint assert hint in get_installation_instructions(Environment(os_type=platform)) +@pytest.mark.skipif(sys.platform != "win32", reason="Tests native Windows PowerShell's restricted execution policy") +def test_windows_verification_commands_run_with_powershell_scripts_disabled() -> None: + powershell = shutil.which("powershell") + assert powershell + instructions = get_installation_instructions(Environment(os_type="windows")).splitlines() + start = instructions.index("Verify with:") + 1 + commands = [line.strip() for line in instructions[start : start + 2]] + script = "; ".join(["$ErrorActionPreference = 'Stop'", *commands, "if ($LASTEXITCODE) { exit $LASTEXITCODE }"]) + + result = subprocess.run( + [powershell, "-NoProfile", "-ExecutionPolicy", "Restricted", "-Command", script], + capture_output=True, + text=True, + timeout=20, + ) + + assert result.returncode == 0, result.stderr + assert len([line for line in result.stdout.splitlines() if re.fullmatch(r"v?\d+\.\d+\.\d+", line)]) == 2 + + def test_alpine_dockerfile_contains_only_the_verified_setup_steps() -> None: output = get_installation_instructions(Environment(os_type="linux", linux_distro="alpine", is_docker=True)) commands = [line.strip() for line in output.splitlines() if line.strip().startswith(("FROM ", "RUN ", "ENV "))] @@ -69,7 +95,10 @@ def test_ci_container_and_wsl_hints_can_coexist() -> None: ) assert "- uses: actions/setup-node@v7\n with:\n node-version: '24'" in output - assert "keep the Node and Python base distributions compatible" in output + assert "FROM node:24-bookworm-slim" in output + assert "apt-get install -y --no-install-recommends python3 python3-venv" in output + assert 'ENV PATH="/opt/venv/bin:$PATH"' in output + assert "nvm install" not in output assert "https://hub.docker.com/_/node" in output assert "install Node.js inside your Linux distribution" in output @@ -81,18 +110,31 @@ def test_other_ci_uses_the_provider_setup_instructions() -> None: @pytest.mark.parametrize( - "provider, label, documentation", + "provider, label, documentation, hosting", [ - ("aws", "AWS Lambda", "docs.aws.amazon.com/lambda/latest/dg/images-create.html"), - ("google", "Google Cloud Functions / Cloud Run", "docs.cloud.google.com/run/docs/building/containers"), + ( + "aws", + "AWS Lambda", + "docs.aws.amazon.com/lambda/latest/dg/images-create.html", + "create a new image-based function", + ), + ( + "google", + "Google Cloud Functions / Cloud Run", + "docs.cloud.google.com/run/docs/building/containers", + "first-generation functions, must move to a Cloud Run service", + ), ( "azure", "Azure Functions", "learn.microsoft.com/en-us/azure/azure-functions/functions-how-to-custom-container", + "Flex Consumption do not accept custom images. Move to Azure Container Apps", ), ], ) -def test_serverless_links_explain_how_to_build_both_runtimes(provider: str, label: str, documentation: str) -> None: +def test_serverless_links_explain_how_to_build_both_runtimes( + provider: str, label: str, documentation: str, hosting: str +) -> None: output = get_installation_instructions( Environment( os_type="linux", linux_distro="amzn", linux_distro_version="2023", is_docker=True, serverless=provider @@ -101,6 +143,7 @@ def test_serverless_links_explain_how_to_build_both_runtimes(provider: str, labe assert f"{label}: build and deploy a custom container that includes both Node.js and Python" in output assert documentation in output + assert hosting in output assert "https://nodejs.org/en/download" in output assert "when building the image" in output assert "sudo" not in output From 4feb9d565228d4c94d7f0bea60eba2f7aa69c04d Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 18 Sep 2026 13:44:53 -0700 Subject: [PATCH 04/12] fix: retain the Python container and isolate runtime smoke tests --- .github/workflows/test.yml | 5 ++++- src/promptfoo/environment.py | 7 +++++-- src/promptfoo/instructions.py | 16 ++++++++++------ tests/smoke/test_installation.py | 30 +++++++++++++++++++++++++++++ tests/test_environment.py | 7 +++++++ tests/test_instructions.py | 33 ++++++-------------------------- 6 files changed, 62 insertions(+), 36 deletions(-) create mode 100644 tests/smoke/test_installation.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 177bf05..9e9aeb1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -86,7 +86,7 @@ jobs: Environment(os_type="linux", linux_distro=distro, is_docker=True) ) commands = [line.strip() for line in instructions.splitlines()] - commands = [command for command in commands if command.startswith(("FROM ", "RUN ", "ENV "))] + commands = [command for command in commands if command.startswith(("FROM ", "COPY ", "RUN ", "ENV "))] directory = Path(sys.argv[1]) / distro directory.mkdir(parents=True, exist_ok=True) (directory / "Dockerfile").write_text("\n".join(commands) + "\n") @@ -97,6 +97,9 @@ jobs: && npm --version && npx --version && python --version && pip --version \ && python -c 'import sys; assert sys.prefix == "/opt/venv"' EOF + if [ "$distro" = debian ]; then + echo 'RUN python -c "import sys; assert sys.version_info[:2] == (3, 12)"' >> "$instruction_dir/$distro/Dockerfile" + fi docker build --progress=plain "$instruction_dir/$distro" done docker run --rm alpine:3.20 sh -ec ' diff --git a/src/promptfoo/environment.py b/src/promptfoo/environment.py index 63f0229..03fe094 100644 --- a/src/promptfoo/environment.py +++ b/src/promptfoo/environment.py @@ -31,10 +31,12 @@ def _linux_release() -> tuple[str | None, str | None]: try: release = platform.freedesktop_os_release() except OSError: - return None, None + release = {} distro = release.get("ID", "").lower() version = release.get("VERSION_ID") or None + if not distro and (alpine_version := _read_probe("/etc/alpine-release").strip()): + return "alpine", alpine_version family = [distro, *release.get("ID_LIKE", "").lower().split()] if "alpine" in family: return "alpine", version @@ -75,7 +77,8 @@ def _serverless() -> str | None: if os.environ.get("AWS_LAMBDA_FUNCTION_NAME"): return "aws" if os.environ.get("FUNCTIONS_WORKER_RUNTIME") and any( - os.environ.get(name) for name in ("WEBSITE_INSTANCE_ID", "WEBSITE_SITE_NAME", "CONTAINER_APP_NAME") + os.environ.get(name) + for name in ("WEBSITE_INSTANCE_ID", "WEBSITE_SITE_NAME", "CONTAINER_APP_NAME", "KUBERNETES_SERVICE_HOST") ): return "azure" if os.environ.get("FUNCTION_NAME") or (os.environ.get("FUNCTION_TARGET") and os.environ.get("K_SERVICE")): diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index 7f41f18..5d57c4f 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -20,7 +20,7 @@ "azure": ( "Azure Functions", "Consumption and Flex Consumption do not accept custom images. Move to Azure Container Apps " - "or a Linux Premium/Dedicated plan.", + "or a Linux Premium/Dedicated plan. On Kubernetes, keep the Azure Functions base image and add Node.", "https://learn.microsoft.com/en-us/azure/azure-functions/functions-how-to-custom-container" "?pivots=programming-language-python", ), @@ -37,11 +37,15 @@ def _container_instructions(env: Environment) -> list[str]: ' ENV PATH="/opt/venv/bin:$PATH"', ] return [ - "CONTAINER: start from the official Node.js 24 Debian image and add Python:", - " FROM node:24-bookworm-slim", - " RUN apt-get update && apt-get install -y --no-install-recommends python3 python3-venv " - "&& rm -rf /var/lib/apt/lists/*", - " RUN python3 -m venv /opt/venv", + "CONTAINER: add Node.js 24 to your existing Debian Bookworm Python image; this example uses Python 3.12.", + "Keep the second FROM set to your application's Python version and use matching Linux distributions:", + " FROM node:24-bookworm-slim AS node", + " FROM python:3.12-slim-bookworm", + " COPY --from=node /usr/local/bin/node /usr/local/bin/node", + " COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules", + " RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm " + "&& ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx", + " RUN python -m venv /opt/venv", ' ENV PATH="/opt/venv/bin:$PATH"', " Official Node.js images: https://hub.docker.com/_/node", ] diff --git a/tests/smoke/test_installation.py b/tests/smoke/test_installation.py new file mode 100644 index 0000000..04ea58b --- /dev/null +++ b/tests/smoke/test_installation.py @@ -0,0 +1,30 @@ +import re +import shutil +import subprocess +import sys + +import pytest + +from promptfoo.environment import Environment +from promptfoo.instructions import get_installation_instructions + + +@pytest.mark.smoke +@pytest.mark.skipif(sys.platform != "win32", reason="Tests native Windows PowerShell's restricted execution policy") +def test_windows_verification_commands_run_with_powershell_scripts_disabled() -> None: + powershell = shutil.which("powershell") + assert powershell + instructions = get_installation_instructions(Environment(os_type="windows")).splitlines() + start = instructions.index("Verify with:") + 1 + commands = [line.strip() for line in instructions[start : start + 2]] + script = "; ".join(["$ErrorActionPreference = 'Stop'", *commands, "if ($LASTEXITCODE) { exit $LASTEXITCODE }"]) + + result = subprocess.run( + [powershell, "-NoProfile", "-ExecutionPolicy", "Restricted", "-Command", script], + capture_output=True, + text=True, + timeout=20, + ) + + assert result.returncode == 0, result.stderr + assert len([line for line in result.stdout.splitlines() if re.fullmatch(r"v?\d+\.\d+\.\d+", line)]) == 2 diff --git a/tests/test_environment.py b/tests/test_environment.py index 73f77e3..9e65bc1 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -63,6 +63,12 @@ def test_missing_linux_release_does_not_prevent_generic_help(monkeypatch: pytest assert environment._linux_release() == (None, None) +def test_alpine_marker_survives_a_missing_standard_release(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(environment.platform, "freedesktop_os_release", MagicMock(side_effect=OSError)) + monkeypatch.setattr(environment, "_read_probe", lambda path: "3.24.1\n" if path == "/etc/alpine-release" else "") + assert environment._linux_release() == ("alpine", "3.24.1") + + @pytest.mark.parametrize("platform, expected", [("win32", "windows"), ("darwin", "darwin"), ("freebsd14", "freebsd14")]) def test_non_linux_platforms_do_not_probe_linux_files( monkeypatch: pytest.MonkeyPatch, platform: str, expected: str @@ -135,6 +141,7 @@ def test_detects_ci_guidance(monkeypatch: pytest.MonkeyPatch, variable: str, exp (["AWS_LAMBDA_FUNCTION_NAME"], "aws"), (["FUNCTIONS_WORKER_RUNTIME", "WEBSITE_INSTANCE_ID"], "azure"), (["FUNCTIONS_WORKER_RUNTIME", "CONTAINER_APP_NAME"], "azure"), + (["FUNCTIONS_WORKER_RUNTIME", "KUBERNETES_SERVICE_HOST"], "azure"), (["FUNCTION_TARGET", "K_SERVICE"], "google"), (["FUNCTION_NAME"], "google"), ], diff --git a/tests/test_instructions.py b/tests/test_instructions.py index 20dd0a7..84b2807 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -1,8 +1,3 @@ -import re -import shutil -import subprocess -import sys - import pytest from promptfoo.environment import Environment @@ -31,26 +26,6 @@ def test_common_platforms_get_one_relevant_installation_hint(platform: str, hint assert hint in get_installation_instructions(Environment(os_type=platform)) -@pytest.mark.skipif(sys.platform != "win32", reason="Tests native Windows PowerShell's restricted execution policy") -def test_windows_verification_commands_run_with_powershell_scripts_disabled() -> None: - powershell = shutil.which("powershell") - assert powershell - instructions = get_installation_instructions(Environment(os_type="windows")).splitlines() - start = instructions.index("Verify with:") + 1 - commands = [line.strip() for line in instructions[start : start + 2]] - script = "; ".join(["$ErrorActionPreference = 'Stop'", *commands, "if ($LASTEXITCODE) { exit $LASTEXITCODE }"]) - - result = subprocess.run( - [powershell, "-NoProfile", "-ExecutionPolicy", "Restricted", "-Command", script], - capture_output=True, - text=True, - timeout=20, - ) - - assert result.returncode == 0, result.stderr - assert len([line for line in result.stdout.splitlines() if re.fullmatch(r"v?\d+\.\d+\.\d+", line)]) == 2 - - def test_alpine_dockerfile_contains_only_the_verified_setup_steps() -> None: output = get_installation_instructions(Environment(os_type="linux", linux_distro="alpine", is_docker=True)) commands = [line.strip() for line in output.splitlines() if line.strip().startswith(("FROM ", "RUN ", "ENV "))] @@ -95,8 +70,12 @@ def test_ci_container_and_wsl_hints_can_coexist() -> None: ) assert "- uses: actions/setup-node@v7\n with:\n node-version: '24'" in output - assert "FROM node:24-bookworm-slim" in output - assert "apt-get install -y --no-install-recommends python3 python3-venv" in output + assert "FROM node:24-bookworm-slim AS node" in output + assert "FROM python:3.12-slim-bookworm" in output + assert "Keep the second FROM set to your application's Python version" in output + assert "COPY --from=node /usr/local/bin/node /usr/local/bin/node" in output + assert "npm/bin/npx-cli.js /usr/local/bin/npx" in output + assert "RUN python -m venv /opt/venv" in output assert 'ENV PATH="/opt/venv/bin:$PATH"' in output assert "nvm install" not in output assert "https://hub.docker.com/_/node" in output From 05dba1e7e748bb77e10476e795fea9f95f37ac46 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 18 Sep 2026 13:54:48 -0700 Subject: [PATCH 05/12] fix: preserve container runtimes and Lambda deployment options --- .github/workflows/test.yml | 16 +++++++--- src/promptfoo/instructions.py | 58 +++++++++++++++++++++++---------- tests/test_instructions.py | 60 +++++++++++++++++++++++++++++------ 3 files changed, 103 insertions(+), 31 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9e9aeb1..9bfb921 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,10 +83,15 @@ jobs: for distro in ("alpine", "debian"): instructions = get_installation_instructions( - Environment(os_type="linux", linux_distro=distro, is_docker=True) + Environment(os_type="linux", linux_distro=distro, + linux_distro_version="12" if distro == "debian" else None, is_docker=True) ) commands = [line.strip() for line in instructions.splitlines()] commands = [command for command in commands if command.startswith(("FROM ", "COPY ", "RUN ", "ENV "))] + if distro == "debian": + runtime = next(index for index, command in enumerate(commands) if command.startswith("FROM python:")) + commands.insert(runtime + 1, "RUN python -c 'import pathlib, sysconfig; " + "(pathlib.Path(sysconfig.get_path(\"purelib\")) / \"promptfoo_help_probe.py\").touch()'") directory = Path(sys.argv[1]) / distro directory.mkdir(parents=True, exist_ok=True) (directory / "Dockerfile").write_text("\n".join(commands) + "\n") @@ -94,11 +99,12 @@ jobs: for distro in alpine debian; do cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' RUN node --version | grep -E '^v24\.' \ - && npm --version && npx --version && python --version && pip --version \ - && python -c 'import sys; assert sys.prefix == "/opt/venv"' + && npm --version && npx --version && python --version && pip --version EOF - if [ "$distro" = debian ]; then - echo 'RUN python -c "import sys; assert sys.version_info[:2] == (3, 12)"' >> "$instruction_dir/$distro/Dockerfile" + if [ "$distro" = alpine ]; then + echo 'RUN python -c "import sys; assert sys.prefix == \"/opt/venv\""' >> "$instruction_dir/$distro/Dockerfile" + else + echo 'RUN python -c "import sys, promptfoo_help_probe; assert sys.version_info[:2] == (3, 12)"' >> "$instruction_dir/$distro/Dockerfile" fi docker build --progress=plain "$instruction_dir/$distro" done diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index 5d57c4f..97e8386 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -6,11 +6,6 @@ _NODE_DOWNLOAD = "https://nodejs.org/en/download" _NVM = "https://github.com/nvm-sh/nvm#installing-and-updating" _SERVERLESS = { - "aws": ( - "AWS Lambda", - "For an existing zip-based function, create a new image-based function.", - "https://docs.aws.amazon.com/lambda/latest/dg/images-create.html", - ), "google": ( "Google Cloud Functions / Cloud Run", "Functions deployed from source, including first-generation functions, must move to a Cloud Run service " @@ -36,18 +31,37 @@ def _container_instructions(env: Environment) -> list[str]: " RUN python3 -m venv /opt/venv", ' ENV PATH="/opt/venv/bin:$PATH"', ] + if env.linux_distro == "debian" and env.linux_distro_version in ("12", "bookworm"): + return [ + "CONTAINER: add Node.js 24 to your existing Debian Bookworm Python image; this example uses Python 3.12.", + "Keep the second FROM set to your existing image and Python environment:", + " FROM node:24-bookworm-slim AS node", + " FROM python:3.12-slim-bookworm", + " COPY --from=node /usr/local/bin/node /usr/local/bin/node", + " COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules", + " RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm " + "&& ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx", + " Official Node.js images: https://hub.docker.com/_/node", + ] + if env.linux_distro == "ubuntu": + return [ + "UBUNTU CONTAINER: keep your existing FROM and install Node.js 24 in place as root:", + " RUN apt-get update && apt-get install -y --no-install-recommends bash ca-certificates curl", + " RUN curl -fsSL https://deb.nodesource.com/setup_24.x -o /tmp/nodesource-setup.sh " + "&& bash /tmp/nodesource-setup.sh && rm /tmp/nodesource-setup.sh", + " RUN apt-get install -y --no-install-recommends nodejs && rm -rf /var/lib/apt/lists/*", + " NodeSource's apt repository: https://github.com/nodesource/distributions", + ] + if env.linux_distro == "amzn" and env.linux_distro_version == "2023": + return [ + "AMAZON LINUX 2023 CONTAINER: keep your existing FROM and install Node.js 24 as root:", + " RUN dnf install -y nodejs24 nodejs24-npm && alternatives --set node /usr/bin/node-24 && dnf clean all", + " https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html", + ] return [ - "CONTAINER: add Node.js 24 to your existing Debian Bookworm Python image; this example uses Python 3.12.", - "Keep the second FROM set to your application's Python version and use matching Linux distributions:", - " FROM node:24-bookworm-slim AS node", - " FROM python:3.12-slim-bookworm", - " COPY --from=node /usr/local/bin/node /usr/local/bin/node", - " COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules", - " RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm " - "&& ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx", - " RUN python -m venv /opt/venv", - ' ENV PATH="/opt/venv/bin:$PATH"', - " Official Node.js images: https://hub.docker.com/_/node", + "CONTAINER: keep your existing base image and install Node.js 24 with npm during the image build.", + "Use installation instructions compatible with your distribution and CPU architecture:", + f" {_NODE_DOWNLOAD}", ] @@ -86,6 +100,18 @@ def get_installation_instructions(env: Environment) -> str: " npx.cmd --version" if env.os_type == "windows" else " npx --version", ] + if env.serverless == "aws": + lines += [ + "", + "AWS Lambda: include Node.js and npm in a compatible Lambda layer or a custom Python container.", + "An existing ZIP function can keep its configuration: attach a layer with Node, npm and npx in /opt/bin.", + "Include npm's supporting files and target the function's Amazon Linux version and CPU architecture.", + " https://docs.aws.amazon.com/lambda/latest/dg/packaging-layers.html", + "If you choose a container image instead, create a new image-based function " + "and install Node at build time.", + " https://docs.aws.amazon.com/lambda/latest/dg/images-create.html", + ] + return "\n".join(lines) if env.serverless and env.serverless in _SERVERLESS: name, hosting, documentation = _SERVERLESS[env.serverless] lines += [ diff --git a/tests/test_instructions.py b/tests/test_instructions.py index 84b2807..aa58241 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -66,22 +66,55 @@ def test_other_amazon_releases_are_not_given_amazon_linux_2023_commands(version: def test_ci_container_and_wsl_hints_can_coexist() -> None: output = get_installation_instructions( - Environment(os_type="linux", linux_distro="ubuntu", ci_platform="GitHub Actions", is_docker=True, is_wsl=True) + Environment( + os_type="linux", + linux_distro="debian", + linux_distro_version="12", + ci_platform="GitHub Actions", + is_docker=True, + is_wsl=True, + ) ) assert "- uses: actions/setup-node@v7\n with:\n node-version: '24'" in output assert "FROM node:24-bookworm-slim AS node" in output assert "FROM python:3.12-slim-bookworm" in output - assert "Keep the second FROM set to your application's Python version" in output + assert "Keep the second FROM set to your existing image and Python environment" in output assert "COPY --from=node /usr/local/bin/node /usr/local/bin/node" in output assert "npm/bin/npx-cli.js /usr/local/bin/npx" in output - assert "RUN python -m venv /opt/venv" in output - assert 'ENV PATH="/opt/venv/bin:$PATH"' in output + assert "venv" not in output + assert "ENV PATH" not in output assert "nvm install" not in output assert "https://hub.docker.com/_/node" in output assert "install Node.js inside your Linux distribution" in output +@pytest.mark.parametrize( + ("distribution", "version", "command"), + [ + ("ubuntu", "24.04", "https://deb.nodesource.com/setup_24.x"), + ("amzn", "2023", "dnf install -y nodejs24 nodejs24-npm"), + ], +) +def test_non_bookworm_containers_keep_their_original_base(distribution: str, version: str, command: str) -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro=distribution, linux_distro_version=version, is_docker=True) + ) + assert "keep your existing FROM" in output + assert command in output + assert "FROM python:" not in output + assert "bookworm" not in output + + +def test_unknown_container_does_not_claim_to_be_bookworm() -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="debian", linux_distro_version="13", is_docker=True) + ) + assert "keep your existing base image" in output + assert "distribution and CPU architecture" in output + assert "FROM python:" not in output + + def test_other_ci_uses_the_provider_setup_instructions() -> None: output = get_installation_instructions(Environment(os_type="linux", ci_platform="GitLab CI")) assert "GitLab CI: use your CI provider's Node.js setup step or an image with Node.js 24." in output @@ -91,12 +124,6 @@ def test_other_ci_uses_the_provider_setup_instructions() -> None: @pytest.mark.parametrize( "provider, label, documentation, hosting", [ - ( - "aws", - "AWS Lambda", - "docs.aws.amazon.com/lambda/latest/dg/images-create.html", - "create a new image-based function", - ), ( "google", "Google Cloud Functions / Cloud Run", @@ -128,3 +155,16 @@ def test_serverless_links_explain_how_to_build_both_runtimes( assert "sudo" not in output assert "nvm" not in output assert "npx promptfoo" not in output + + +def test_lambda_zip_can_use_a_layer_or_choose_to_create_an_image_based_function() -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="amzn", linux_distro_version="2023", is_docker=True, serverless="aws") + ) + assert "existing ZIP function can keep its configuration" in output + assert "Node, npm and npx in /opt/bin" in output + assert "supporting files" in output + assert "https://docs.aws.amazon.com/lambda/latest/dg/packaging-layers.html" in output + assert "If you choose a container image instead, create a new image-based function" in output + assert "https://docs.aws.amazon.com/lambda/latest/dg/images-create.html" in output + assert "sudo" not in output From 9fc8f5c056b142035726f5f0d047413f557220d4 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 18 Sep 2026 14:07:02 -0700 Subject: [PATCH 06/12] fix: make platform help work across supported images --- .github/workflows/test.yml | 1 + src/promptfoo/instructions.py | 18 +++++++++++++----- tests/test_instructions.py | 14 +++++++++++--- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9bfb921..7d1ca3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -105,6 +105,7 @@ jobs: echo 'RUN python -c "import sys; assert sys.prefix == \"/opt/venv\""' >> "$instruction_dir/$distro/Dockerfile" else echo 'RUN python -c "import sys, promptfoo_help_probe; assert sys.version_info[:2] == (3, 12)"' >> "$instruction_dir/$distro/Dockerfile" + echo 'RUN env PATH=/opt/venv/bin:/usr/bin:/bin sh -c "node --version && npm --version && npx --version"' >> "$instruction_dir/$distro/Dockerfile" fi docker build --progress=plain "$instruction_dir/$distro" done diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index 97e8386..11574e7 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -39,8 +39,10 @@ def _container_instructions(env: Environment) -> list[str]: " FROM python:3.12-slim-bookworm", " COPY --from=node /usr/local/bin/node /usr/local/bin/node", " COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules", - " RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm " - "&& ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx", + " RUN ln -sf /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm " + "&& ln -sf /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx", + " RUN ln -sf /usr/local/bin/node /usr/bin/node && ln -sf /usr/local/bin/npm /usr/bin/npm " + "&& ln -sf /usr/local/bin/npx /usr/bin/npx", " Official Node.js images: https://hub.docker.com/_/node", ] if env.linux_distro == "ubuntu": @@ -55,8 +57,12 @@ def _container_instructions(env: Environment) -> list[str]: if env.linux_distro == "amzn" and env.linux_distro_version == "2023": return [ "AMAZON LINUX 2023 CONTAINER: keep your existing FROM and install Node.js 24 as root:", - " RUN dnf install -y nodejs24 nodejs24-npm && alternatives --set node /usr/bin/node-24 && dnf clean all", + " RUN dnf --releasever=latest install -y nodejs24 nodejs24-npm " + "&& /usr/sbin/alternatives --set node /usr/bin/node-24 && dnf clean all", + "Node 24 requires repository release 2023.9.20251110 or newer; replace latest with your approved recent " + "snapshot, or update an older base image before installing.", " https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html", + " https://docs.aws.amazon.com/linux/al2023/ug/managing-repos-os-updates.html", ] return [ "CONTAINER: keep your existing base image and install Node.js 24 with npm during the image build.", @@ -77,9 +83,11 @@ def _linux_instructions(env: Environment) -> list[str]: return [ "AMAZON LINUX 2023: install and select Node.js 24 (omit sudo when running as root):", " sudo dnf install -y nodejs24 nodejs24-npm", - " sudo alternatives --set node /usr/bin/node-24", + " sudo /usr/sbin/alternatives --set node /usr/bin/node-24", " node --version", " https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html", + "If your pinned repository is older than 2023.9.20251110, update the base/repository or install from a " + "newer approved snapshot: sudo dnf --releasever=latest install -y nodejs24 nodejs24-npm", f" Without sudo, install nvm: {_NVM}", " Then run: nvm install 24", ] @@ -97,7 +105,7 @@ def get_installation_instructions(env: Environment) -> str: f"Install Node.js 24 LTS with npm: {_NODE_DOWNLOAD}", "Verify with:", " node --version", - " npx.cmd --version" if env.os_type == "windows" else " npx --version", + " cmd /d /c npx --version" if env.os_type == "windows" else " npx --version", ] if env.serverless == "aws": diff --git a/tests/test_instructions.py b/tests/test_instructions.py index aa58241..65a8ec1 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -12,7 +12,7 @@ def test_every_platform_gets_the_runtime_requirement_and_a_working_fallback(plat assert f"requires Node.js {MIN_NODE_VERSION_TEXT} or newer" in output assert "Node.js 24 LTS with npm" in output assert "https://nodejs.org/en/download" in output - npx = "npx.cmd" if platform == "windows" else "npx" + npx = "cmd /d /c npx" if platform == "windows" else "npx" assert f" node --version\n {npx} --version" in output assert "node --version &&" not in output assert "DIRECT USAGE after installing Node.js: npx promptfoo@latest eval" in output @@ -47,9 +47,11 @@ def test_amazon_linux_2023_installs_both_versioned_packages_and_selects_the_acti ) assert "sudo dnf install -y nodejs24 nodejs24-npm" in output - assert "sudo alternatives --set node /usr/bin/node-24" in output + assert "sudo /usr/sbin/alternatives --set node /usr/bin/node-24" in output assert "omit sudo when running as root" in output assert "https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html" in output + assert "2023.9.20251110" in output + assert "dnf --releasever=latest install" in output assert "Without sudo, install nvm: https://github.com/nvm-sh/nvm#installing-and-updating" in output assert "nvm install 24" in output assert "dnf install -y nodejs\n" not in output @@ -82,6 +84,9 @@ def test_ci_container_and_wsl_hints_can_coexist() -> None: assert "Keep the second FROM set to your existing image and Python environment" in output assert "COPY --from=node /usr/local/bin/node /usr/local/bin/node" in output assert "npm/bin/npx-cli.js /usr/local/bin/npx" in output + assert "/usr/local/bin/node /usr/bin/node" in output + assert "/usr/local/bin/npm /usr/bin/npm" in output + assert "/usr/local/bin/npx /usr/bin/npx" in output assert "venv" not in output assert "ENV PATH" not in output assert "nvm install" not in output @@ -93,7 +98,7 @@ def test_ci_container_and_wsl_hints_can_coexist() -> None: ("distribution", "version", "command"), [ ("ubuntu", "24.04", "https://deb.nodesource.com/setup_24.x"), - ("amzn", "2023", "dnf install -y nodejs24 nodejs24-npm"), + ("amzn", "2023", "dnf --releasever=latest install -y nodejs24 nodejs24-npm"), ], ) def test_non_bookworm_containers_keep_their_original_base(distribution: str, version: str, command: str) -> None: @@ -104,6 +109,9 @@ def test_non_bookworm_containers_keep_their_original_base(distribution: str, ver assert command in output assert "FROM python:" not in output assert "bookworm" not in output + if distribution == "amzn": + assert "/usr/sbin/alternatives --set node" in output + assert "2023.9.20251110" in output def test_unknown_container_does_not_claim_to_be_bookworm() -> None: From e79838b746328f7934132efdab98d9ce50f8f159 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 18 Sep 2026 14:14:16 -0700 Subject: [PATCH 07/12] fix: cover Git Bash and legacy Linux installation paths --- src/promptfoo/instructions.py | 29 ++++++++++++++++++++----- tests/smoke/test_installation.py | 23 ++++++++++++++++++++ tests/test_instructions.py | 36 ++++++++++++++++++++++++++------ 3 files changed, 77 insertions(+), 11 deletions(-) diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index 11574e7..980e925 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -48,10 +48,10 @@ def _container_instructions(env: Environment) -> list[str]: if env.linux_distro == "ubuntu": return [ "UBUNTU CONTAINER: keep your existing FROM and install Node.js 24 in place as root:", - " RUN apt-get update && apt-get install -y --no-install-recommends bash ca-certificates curl", - " RUN curl -fsSL https://deb.nodesource.com/setup_24.x -o /tmp/nodesource-setup.sh " - "&& bash /tmp/nodesource-setup.sh && rm /tmp/nodesource-setup.sh", - " RUN apt-get install -y --no-install-recommends nodejs && rm -rf /var/lib/apt/lists/*", + " RUN apt-get update && apt-get install -y --no-install-recommends bash ca-certificates curl " + "&& curl -fsSL https://deb.nodesource.com/setup_24.x -o /tmp/nodesource-setup.sh " + "&& bash /tmp/nodesource-setup.sh && rm /tmp/nodesource-setup.sh " + "&& apt-get install -y --no-install-recommends nodejs && rm -rf /var/lib/apt/lists/*", " NodeSource's apt repository: https://github.com/nodesource/distributions", ] if env.linux_distro == "amzn" and env.linux_distro_version == "2023": @@ -64,6 +64,12 @@ def _container_instructions(env: Environment) -> list[str]: " https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html", " https://docs.aws.amazon.com/linux/al2023/ug/managing-repos-os-updates.html", ] + if env.linux_distro == "amzn" and env.linux_distro_version == "2": + return [ + "AMAZON LINUX 2 CONTAINER: move to an Amazon Linux 2023 or another Node.js 24 compatible base image.", + "Official Node.js 22/24 Linux binaries require newer glibc than Amazon Linux 2 provides.", + " https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html", + ] return [ "CONTAINER: keep your existing base image and install Node.js 24 with npm during the image build.", "Use installation instructions compatible with your distribution and CPU architecture:", @@ -91,6 +97,12 @@ def _linux_instructions(env: Environment) -> list[str]: f" Without sudo, install nvm: {_NVM}", " Then run: nvm install 24", ] + if env.linux_distro == "amzn" and env.linux_distro_version == "2": + return [ + "AMAZON LINUX 2: upgrade to Amazon Linux 2023 or run in a Node.js 24 compatible container.", + "Official Node.js 22/24 Linux binaries require newer glibc than Amazon Linux 2 provides.", + " https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html", + ] return [ "LINUX: use a Node.js version manager or your distribution's instructions for Node.js 24.", f" Install nvm: {_NVM}", @@ -105,8 +117,10 @@ def get_installation_instructions(env: Environment) -> str: f"Install Node.js 24 LTS with npm: {_NODE_DOWNLOAD}", "Verify with:", " node --version", - " cmd /d /c npx --version" if env.os_type == "windows" else " npx --version", + " npx.cmd --version" if env.os_type == "windows" else " npx --version", ] + if env.os_type == "windows": + lines.append("If your Node manager installs npx.exe instead (such as Volta), use: npx.exe --version") if env.serverless == "aws": lines += [ @@ -119,6 +133,11 @@ def get_installation_instructions(env: Environment) -> str: "and install Node at build time.", " https://docs.aws.amazon.com/lambda/latest/dg/images-create.html", ] + if env.linux_distro == "amzn" and env.linux_distro_version == "2": + lines.append( + "For an Amazon Linux 2 based function, first upgrade to an Amazon Linux 2023 based Python runtime " + "before packaging the official Node.js 22/24 binaries." + ) return "\n".join(lines) if env.serverless and env.serverless in _SERVERLESS: name, hosting, documentation = _SERVERLESS[env.serverless] diff --git a/tests/smoke/test_installation.py b/tests/smoke/test_installation.py index 04ea58b..d2128d0 100644 --- a/tests/smoke/test_installation.py +++ b/tests/smoke/test_installation.py @@ -1,7 +1,9 @@ +import os import re import shutil import subprocess import sys +from pathlib import Path import pytest @@ -28,3 +30,24 @@ def test_windows_verification_commands_run_with_powershell_scripts_disabled() -> assert result.returncode == 0, result.stderr assert len([line for line in result.stdout.splitlines() if re.fullmatch(r"v?\d+\.\d+\.\d+", line)]) == 2 + + +@pytest.mark.smoke +@pytest.mark.skipif(sys.platform != "win32", reason="Tests native Windows Git Bash") +def test_windows_verification_commands_run_in_git_bash() -> None: + bash = Path(os.environ["PROGRAMFILES"]) / "Git" / "bin" / "bash.exe" + if not bash.is_file(): + pytest.skip("Git for Windows is not installed") + instructions = get_installation_instructions(Environment(os_type="windows")).splitlines() + start = instructions.index("Verify with:") + 1 + commands = [line.strip() for line in instructions[start : start + 2]] + + result = subprocess.run( + [str(bash), "--noprofile", "--norc", "-c", "set -e; " + "; ".join(commands)], + capture_output=True, + text=True, + timeout=20, + ) + + assert result.returncode == 0, result.stderr + assert len([line for line in result.stdout.splitlines() if re.fullmatch(r"v?\d+\.\d+\.\d+", line)]) == 2 diff --git a/tests/test_instructions.py b/tests/test_instructions.py index 65a8ec1..d9e1b29 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -12,8 +12,10 @@ def test_every_platform_gets_the_runtime_requirement_and_a_working_fallback(plat assert f"requires Node.js {MIN_NODE_VERSION_TEXT} or newer" in output assert "Node.js 24 LTS with npm" in output assert "https://nodejs.org/en/download" in output - npx = "cmd /d /c npx" if platform == "windows" else "npx" + npx = "npx.cmd" if platform == "windows" else "npx" assert f" node --version\n {npx} --version" in output + if platform == "windows": + assert "use: npx.exe --version" in output assert "node --version &&" not in output assert "DIRECT USAGE after installing Node.js: npx promptfoo@latest eval" in output @@ -57,15 +59,25 @@ def test_amazon_linux_2023_installs_both_versioned_packages_and_selects_the_acti assert "dnf install -y nodejs\n" not in output -@pytest.mark.parametrize("version", [None, "2"]) -def test_other_amazon_releases_are_not_given_amazon_linux_2023_commands(version: str | None) -> None: - output = get_installation_instructions( - Environment(os_type="linux", linux_distro="amzn", linux_distro_version=version) - ) +def test_unknown_amazon_release_is_not_given_amazon_linux_2023_commands() -> None: + output = get_installation_instructions(Environment(os_type="linux", linux_distro="amzn")) assert "nvm install 24" in output assert "nodejs24-npm" not in output +@pytest.mark.parametrize("container", [False, True]) +def test_amazon_linux_2_recommends_a_compatible_os(container: bool) -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="amzn", linux_distro_version="2", is_docker=container) + ) + assert "Amazon Linux 2023" in output + assert "require newer glibc" in output + assert "nvm install 24" not in output + if container: + assert "compatible base image" in output + assert "keep your existing base image" not in output + + def test_ci_container_and_wsl_hints_can_coexist() -> None: output = get_installation_instructions( Environment( @@ -112,6 +124,11 @@ def test_non_bookworm_containers_keep_their_original_base(distribution: str, ver if distribution == "amzn": assert "/usr/sbin/alternatives --set node" in output assert "2023.9.20251110" in output + if distribution == "ubuntu": + commands = [line for line in output.splitlines() if line.strip().startswith("RUN ")] + assert len(commands) == 1 + assert "apt-get update" in commands[0] + assert "rm -rf /var/lib/apt/lists/*" in commands[0] def test_unknown_container_does_not_claim_to_be_bookworm() -> None: @@ -176,3 +193,10 @@ def test_lambda_zip_can_use_a_layer_or_choose_to_create_an_image_based_function( assert "If you choose a container image instead, create a new image-based function" in output assert "https://docs.aws.amazon.com/lambda/latest/dg/images-create.html" in output assert "sudo" not in output + + +def test_lambda_on_amazon_linux_2_requires_a_compatible_runtime_before_packaging() -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="amzn", linux_distro_version="2", is_docker=True, serverless="aws") + ) + assert "first upgrade to an Amazon Linux 2023 based Python runtime" in output From fb92405c966500b0ed020018cc6f42d8f5577f83 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 18 Sep 2026 14:22:30 -0700 Subject: [PATCH 08/12] fix: cover Debian Trixie and native Windows npm launchers --- .github/workflows/test.yml | 10 ++--- src/promptfoo/instructions.py | 12 ++++-- tests/smoke/test_installation.py | 71 ++++++++++++++++++++++++++++---- tests/test_instructions.py | 13 +++++- 4 files changed, 88 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d1ca3e..e1198b6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -81,14 +81,14 @@ jobs: from promptfoo.environment import Environment from promptfoo.instructions import get_installation_instructions - for distro in ("alpine", "debian"): + for distro, version in (("alpine", None), ("bookworm", "12"), ("trixie", "13")): instructions = get_installation_instructions( - Environment(os_type="linux", linux_distro=distro, - linux_distro_version="12" if distro == "debian" else None, is_docker=True) + Environment(os_type="linux", linux_distro="alpine" if distro == "alpine" else "debian", + linux_distro_version=version, is_docker=True) ) commands = [line.strip() for line in instructions.splitlines()] commands = [command for command in commands if command.startswith(("FROM ", "COPY ", "RUN ", "ENV "))] - if distro == "debian": + if distro != "alpine": runtime = next(index for index, command in enumerate(commands) if command.startswith("FROM python:")) commands.insert(runtime + 1, "RUN python -c 'import pathlib, sysconfig; " "(pathlib.Path(sysconfig.get_path(\"purelib\")) / \"promptfoo_help_probe.py\").touch()'") @@ -96,7 +96,7 @@ jobs: directory.mkdir(parents=True, exist_ok=True) (directory / "Dockerfile").write_text("\n".join(commands) + "\n") PY - for distro in alpine debian; do + for distro in alpine bookworm trixie; do cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' RUN node --version | grep -E '^v24\.' \ && npm --version && npx --version && python --version && pip --version diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index 980e925..f3ef148 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -31,12 +31,16 @@ def _container_instructions(env: Environment) -> list[str]: " RUN python3 -m venv /opt/venv", ' ENV PATH="/opt/venv/bin:$PATH"', ] - if env.linux_distro == "debian" and env.linux_distro_version in ("12", "bookworm"): + debian_release = {"12": "bookworm", "bookworm": "bookworm", "13": "trixie", "trixie": "trixie"}.get( + env.linux_distro_version or "" + ) + if env.linux_distro == "debian" and debian_release: return [ - "CONTAINER: add Node.js 24 to your existing Debian Bookworm Python image; this example uses Python 3.12.", + f"CONTAINER: add Node.js 24 to your existing Debian {debian_release.title()} Python image; " + "this example uses Python 3.12.", "Keep the second FROM set to your existing image and Python environment:", - " FROM node:24-bookworm-slim AS node", - " FROM python:3.12-slim-bookworm", + f" FROM node:24-{debian_release}-slim AS node", + f" FROM python:3.12-slim-{debian_release}", " COPY --from=node /usr/local/bin/node /usr/local/bin/node", " COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules", " RUN ln -sf /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm " diff --git a/tests/smoke/test_installation.py b/tests/smoke/test_installation.py index d2128d0..c8fce5b 100644 --- a/tests/smoke/test_installation.py +++ b/tests/smoke/test_installation.py @@ -11,14 +11,43 @@ from promptfoo.instructions import get_installation_instructions +def windows_verification_commands(launcher: str | None = None) -> list[str]: + instructions = get_installation_instructions(Environment(os_type="windows")).splitlines() + start = instructions.index("Verify with:") + 1 + if launcher is None: + launcher = next((name for name in ("npx.cmd", "npx.exe") if shutil.which(name)), None) + assert launcher, "Neither npx.cmd nor npx.exe is available" + npx = instructions[start + 1].strip() + if launcher == "npx.exe": + npx = next(line.partition("use: ")[2] for line in instructions if "use: npx.exe" in line) + return [instructions[start].strip(), npx] + + +def git_bash() -> Path | None: + roots = [] + if git := shutil.which("git"): + location = Path(git).resolve() + roots += [location.parent, location.parent.parent] + for variable, suffix in (("PROGRAMFILES", "Git"), ("LOCALAPPDATA", "Programs/Git")): + if directory := os.environ.get(variable): + roots.append(Path(directory) / suffix) + return next( + ( + candidate + for root in roots + for candidate in (root / "bin/bash.exe", root / "usr/bin/bash.exe") + if candidate.is_file() + ), + None, + ) + + @pytest.mark.smoke @pytest.mark.skipif(sys.platform != "win32", reason="Tests native Windows PowerShell's restricted execution policy") def test_windows_verification_commands_run_with_powershell_scripts_disabled() -> None: powershell = shutil.which("powershell") assert powershell - instructions = get_installation_instructions(Environment(os_type="windows")).splitlines() - start = instructions.index("Verify with:") + 1 - commands = [line.strip() for line in instructions[start : start + 2]] + commands = windows_verification_commands() script = "; ".join(["$ErrorActionPreference = 'Stop'", *commands, "if ($LASTEXITCODE) { exit $LASTEXITCODE }"]) result = subprocess.run( @@ -35,12 +64,9 @@ def test_windows_verification_commands_run_with_powershell_scripts_disabled() -> @pytest.mark.smoke @pytest.mark.skipif(sys.platform != "win32", reason="Tests native Windows Git Bash") def test_windows_verification_commands_run_in_git_bash() -> None: - bash = Path(os.environ["PROGRAMFILES"]) / "Git" / "bin" / "bash.exe" - if not bash.is_file(): + if not (bash := git_bash()): pytest.skip("Git for Windows is not installed") - instructions = get_installation_instructions(Environment(os_type="windows")).splitlines() - start = instructions.index("Verify with:") + 1 - commands = [line.strip() for line in instructions[start : start + 2]] + commands = windows_verification_commands() result = subprocess.run( [str(bash), "--noprofile", "--norc", "-c", "set -e; " + "; ".join(commands)], @@ -51,3 +77,32 @@ def test_windows_verification_commands_run_in_git_bash() -> None: assert result.returncode == 0, result.stderr assert len([line for line in result.stdout.splitlines() if re.fullmatch(r"v?\d+\.\d+\.\d+", line)]) == 2 + + +@pytest.mark.smoke +@pytest.mark.skipif(sys.platform != "win32", reason="Tests native Windows executable lookup") +@pytest.mark.parametrize("shell", ["powershell", "git-bash"]) +def test_windows_npx_executable_alternative_works_in_each_shell(tmp_path: Path, shell: str) -> None: + node = shutil.which("node") + assert node + shutil.copy2(node, tmp_path / "npx.exe") + command = windows_verification_commands("npx.exe")[1] + if shell == "powershell": + powershell = shutil.which("powershell") + assert powershell + prefix = [powershell, "-NoProfile", "-ExecutionPolicy", "Restricted", "-Command"] + elif bash := git_bash(): + prefix = [str(bash), "--noprofile", "--norc", "-c"] + else: + pytest.skip("Git for Windows is not installed") + + result = subprocess.run( + [*prefix, command], + env=os.environ | {"PATH": str(tmp_path) + os.pathsep + os.environ["PATH"]}, + capture_output=True, + text=True, + timeout=20, + ) + + assert result.returncode == 0, result.stderr + assert re.fullmatch(r"v\d+\.\d+\.\d+", result.stdout.strip()) diff --git a/tests/test_instructions.py b/tests/test_instructions.py index d9e1b29..f205ddf 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -131,10 +131,21 @@ def test_non_bookworm_containers_keep_their_original_base(distribution: str, ver assert "rm -rf /var/lib/apt/lists/*" in commands[0] -def test_unknown_container_does_not_claim_to_be_bookworm() -> None: +def test_trixie_container_uses_matching_supported_node_and_python_images() -> None: output = get_installation_instructions( Environment(os_type="linux", linux_distro="debian", linux_distro_version="13", is_docker=True) ) + assert "Debian Trixie" in output + assert "FROM node:24-trixie-slim AS node" in output + assert "FROM python:3.12-slim-trixie" in output + assert "/usr/local/bin/node /usr/bin/node" in output + assert "bookworm" not in output + + +def test_unknown_container_does_not_claim_to_be_bookworm() -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="debian", linux_distro_version="14", is_docker=True) + ) assert "keep your existing base image" in output assert "distribution and CPU architecture" in output assert "FROM python:" not in output From 13b588e2053fe02eec232371d9607f5d178dda83 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 18 Sep 2026 14:28:28 -0700 Subject: [PATCH 09/12] fix: keep global npm commands visible in Debian images --- .github/workflows/test.yml | 2 ++ src/promptfoo/instructions.py | 7 ++++--- tests/test_instructions.py | 6 +++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e1198b6..8c84639 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -92,6 +92,7 @@ jobs: runtime = next(index for index, command in enumerate(commands) if command.startswith("FROM python:")) commands.insert(runtime + 1, "RUN python -c 'import pathlib, sysconfig; " "(pathlib.Path(sysconfig.get_path(\"purelib\")) / \"promptfoo_help_probe.py\").touch()'") + commands.insert(runtime + 2, 'ENV PATH="/opt/venv/bin:/usr/bin:/bin"') directory = Path(sys.argv[1]) / distro directory.mkdir(parents=True, exist_ok=True) (directory / "Dockerfile").write_text("\n".join(commands) + "\n") @@ -106,6 +107,7 @@ jobs: else echo 'RUN python -c "import sys, promptfoo_help_probe; assert sys.version_info[:2] == (3, 12)"' >> "$instruction_dir/$distro/Dockerfile" echo 'RUN env PATH=/opt/venv/bin:/usr/bin:/bin sh -c "node --version && npm --version && npx --version"' >> "$instruction_dir/$distro/Dockerfile" + echo 'RUN prefix="$(npm prefix --global)" && case ":$PATH:" in *":$prefix/bin:"*) ;; *) exit 1 ;; esac' >> "$instruction_dir/$distro/Dockerfile" fi docker build --progress=plain "$instruction_dir/$distro" done diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index f3ef148..2a40fa0 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -15,9 +15,9 @@ "azure": ( "Azure Functions", "Consumption and Flex Consumption do not accept custom images. Move to Azure Container Apps " - "or a Linux Premium/Dedicated plan. On Kubernetes, keep the Azure Functions base image and add Node.", - "https://learn.microsoft.com/en-us/azure/azure-functions/functions-how-to-custom-container" - "?pivots=programming-language-python", + "or a Linux Premium/Dedicated plan. Select that hosting environment in the documentation. " + "On Kubernetes, keep the Azure Functions base image and add Node.", + "https://learn.microsoft.com/en-us/azure/azure-functions/functions-how-to-custom-container", ), } @@ -47,6 +47,7 @@ def _container_instructions(env: Environment) -> list[str]: "&& ln -sf /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx", " RUN ln -sf /usr/local/bin/node /usr/bin/node && ln -sf /usr/local/bin/npm /usr/bin/npm " "&& ln -sf /usr/local/bin/npx /usr/bin/npx", + ' ENV PATH="${PATH}:/usr/local/bin"', " Official Node.js images: https://hub.docker.com/_/node", ] if env.linux_distro == "ubuntu": diff --git a/tests/test_instructions.py b/tests/test_instructions.py index f205ddf..1cdf16c 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -100,7 +100,7 @@ def test_ci_container_and_wsl_hints_can_coexist() -> None: assert "/usr/local/bin/npm /usr/bin/npm" in output assert "/usr/local/bin/npx /usr/bin/npx" in output assert "venv" not in output - assert "ENV PATH" not in output + assert 'ENV PATH="${PATH}:/usr/local/bin"' in output assert "nvm install" not in output assert "https://hub.docker.com/_/node" in output assert "install Node.js inside your Linux distribution" in output @@ -139,6 +139,7 @@ def test_trixie_container_uses_matching_supported_node_and_python_images() -> No assert "FROM node:24-trixie-slim AS node" in output assert "FROM python:3.12-slim-trixie" in output assert "/usr/local/bin/node /usr/bin/node" in output + assert 'ENV PATH="${PATH}:/usr/local/bin"' in output assert "bookworm" not in output @@ -191,6 +192,9 @@ def test_serverless_links_explain_how_to_build_both_runtimes( assert "sudo" not in output assert "nvm" not in output assert "npx promptfoo" not in output + if provider == "azure": + assert "Select that hosting environment in the documentation" in output + assert "?pivots=" not in output def test_lambda_zip_can_use_a_layer_or_choose_to_create_an_image_based_function() -> None: From c1aebd549e4d33bc27c628095c3accad86e0e050 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 18 Sep 2026 14:30:14 -0700 Subject: [PATCH 10/12] fix: isolate npm globals from Python container scripts --- .github/workflows/test.yml | 10 +++++++--- src/promptfoo/instructions.py | 6 ++++-- tests/test_instructions.py | 6 ++++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8c84639..c1a0fb0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -105,9 +105,13 @@ jobs: if [ "$distro" = alpine ]; then echo 'RUN python -c "import sys; assert sys.prefix == \"/opt/venv\""' >> "$instruction_dir/$distro/Dockerfile" else - echo 'RUN python -c "import sys, promptfoo_help_probe; assert sys.version_info[:2] == (3, 12)"' >> "$instruction_dir/$distro/Dockerfile" - echo 'RUN env PATH=/opt/venv/bin:/usr/bin:/bin sh -c "node --version && npm --version && npx --version"' >> "$instruction_dir/$distro/Dockerfile" - echo 'RUN prefix="$(npm prefix --global)" && case ":$PATH:" in *":$prefix/bin:"*) ;; *) exit 1 ;; esac' >> "$instruction_dir/$distro/Dockerfile" + cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' + RUN python -c "import sys, promptfoo_help_probe; assert sys.version_info[:2] == (3, 12)" + RUN env PATH=/opt/venv/bin:/usr/bin:/bin sh -c "node --version && npm --version && npx --version" + RUN prefix="$(npm prefix --global)" && test "$prefix" = /opt/npm-global \ + && python -c "import sysconfig; assert sysconfig.get_path('scripts') != '/opt/npm-global/bin'" \ + && case ":$PATH:" in *":$prefix/bin:"*) ;; *) exit 1 ;; esac + EOF fi docker build --progress=plain "$instruction_dir/$distro" done diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index 2a40fa0..a02efc1 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -39,6 +39,7 @@ def _container_instructions(env: Environment) -> list[str]: f"CONTAINER: add Node.js 24 to your existing Debian {debian_release.title()} Python image; " "this example uses Python 3.12.", "Keep the second FROM set to your existing image and Python environment:", + "Run these build steps as root; restore your original USER afterward if needed.", f" FROM node:24-{debian_release}-slim AS node", f" FROM python:3.12-slim-{debian_release}", " COPY --from=node /usr/local/bin/node /usr/local/bin/node", @@ -46,8 +47,9 @@ def _container_instructions(env: Environment) -> list[str]: " RUN ln -sf /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm " "&& ln -sf /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx", " RUN ln -sf /usr/local/bin/node /usr/bin/node && ln -sf /usr/local/bin/npm /usr/bin/npm " - "&& ln -sf /usr/local/bin/npx /usr/bin/npx", - ' ENV PATH="${PATH}:/usr/local/bin"', + "&& ln -sf /usr/local/bin/npx /usr/bin/npx && mkdir -p /opt/npm-global", + ' ENV NPM_CONFIG_PREFIX="/opt/npm-global"', + ' ENV PATH="${PATH}:/usr/local/bin:/opt/npm-global/bin"', " Official Node.js images: https://hub.docker.com/_/node", ] if env.linux_distro == "ubuntu": diff --git a/tests/test_instructions.py b/tests/test_instructions.py index 1cdf16c..d4b5a27 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -100,7 +100,8 @@ def test_ci_container_and_wsl_hints_can_coexist() -> None: assert "/usr/local/bin/npm /usr/bin/npm" in output assert "/usr/local/bin/npx /usr/bin/npx" in output assert "venv" not in output - assert 'ENV PATH="${PATH}:/usr/local/bin"' in output + assert 'ENV NPM_CONFIG_PREFIX="/opt/npm-global"' in output + assert 'ENV PATH="${PATH}:/usr/local/bin:/opt/npm-global/bin"' in output assert "nvm install" not in output assert "https://hub.docker.com/_/node" in output assert "install Node.js inside your Linux distribution" in output @@ -139,7 +140,8 @@ def test_trixie_container_uses_matching_supported_node_and_python_images() -> No assert "FROM node:24-trixie-slim AS node" in output assert "FROM python:3.12-slim-trixie" in output assert "/usr/local/bin/node /usr/bin/node" in output - assert 'ENV PATH="${PATH}:/usr/local/bin"' in output + assert 'ENV NPM_CONFIG_PREFIX="/opt/npm-global"' in output + assert 'ENV PATH="${PATH}:/usr/local/bin:/opt/npm-global/bin"' in output assert "bookworm" not in output From d8592ca3ae88b0b83eaebf4c3cbb82b052826c87 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 18 Sep 2026 14:40:28 -0700 Subject: [PATCH 11/12] fix: preserve existing npm settings in Docker guidance --- .github/workflows/test.yml | 64 ++++++++++++++++++++++++++------ src/promptfoo/instructions.py | 18 +++++---- tests/smoke/test_installation.py | 7 ++-- tests/test_instructions.py | 21 ++++++----- 4 files changed, 77 insertions(+), 33 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c1a0fb0..68ab9ee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -81,23 +81,35 @@ jobs: from promptfoo.environment import Environment from promptfoo.instructions import get_installation_instructions - for distro, version in (("alpine", None), ("bookworm", "12"), ("trixie", "13")): + for label, distro, version in (("alpine", "alpine", None), ("bookworm", "debian", "12"), + ("trixie", "debian", "13"), ("bookworm-env", "debian", "12"), + ("bookworm-npmrc", "debian", "12")): instructions = get_installation_instructions( - Environment(os_type="linux", linux_distro="alpine" if distro == "alpine" else "debian", - linux_distro_version=version, is_docker=True) + Environment(os_type="linux", linux_distro=distro, linux_distro_version=version, is_docker=True) ) commands = [line.strip() for line in instructions.splitlines()] commands = [command for command in commands if command.startswith(("FROM ", "COPY ", "RUN ", "ENV "))] - if distro != "alpine": + if distro == "debian": runtime = next(index for index, command in enumerate(commands) if command.startswith("FROM python:")) - commands.insert(runtime + 1, "RUN python -c 'import pathlib, sysconfig; " - "(pathlib.Path(sysconfig.get_path(\"purelib\")) / \"promptfoo_help_probe.py\").touch()'") - commands.insert(runtime + 2, 'ENV PATH="/opt/venv/bin:/usr/bin:/bin"') - directory = Path(sys.argv[1]) / distro + inherited = ["RUN python -c 'import pathlib, sysconfig; " + "(pathlib.Path(sysconfig.get_path(\"purelib\")) / \"promptfoo_help_probe.py\").touch()'"] + if label in ("bookworm-env", "bookworm-npmrc"): + inherited += ["RUN useradd -m app && mkdir -p /home/app/.npm-global " + "&& chown app:app /home/app/.npm-global", + 'ENV PATH="/usr/local/bin:/home/app/.npm-global/bin:/usr/bin:/bin"'] + if label == "bookworm-env": + inherited.append('ENV NPM_CONFIG_PREFIX="/home/app/.npm-global"') + else: + inherited.append("RUN printf '%s\\n' 'prefix=/home/app/.npm-global' > /home/app/.npmrc " + "&& chown app:app /home/app/.npmrc") + else: + inherited.append('ENV PATH="/opt/venv/bin:/usr/bin:/bin"') + commands[runtime + 1:runtime + 1] = inherited + directory = Path(sys.argv[1]) / label directory.mkdir(parents=True, exist_ok=True) (directory / "Dockerfile").write_text("\n".join(commands) + "\n") PY - for distro in alpine bookworm trixie; do + for distro in alpine bookworm trixie bookworm-env bookworm-npmrc; do cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' RUN node --version | grep -E '^v24\.' \ && npm --version && npx --version && python --version && pip --version @@ -108,9 +120,37 @@ jobs: cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' RUN python -c "import sys, promptfoo_help_probe; assert sys.version_info[:2] == (3, 12)" RUN env PATH=/opt/venv/bin:/usr/bin:/bin sh -c "node --version && npm --version && npx --version" - RUN prefix="$(npm prefix --global)" && test "$prefix" = /opt/npm-global \ - && python -c "import sysconfig; assert sysconfig.get_path('scripts') != '/opt/npm-global/bin'" \ - && case ":$PATH:" in *":$prefix/bin:"*) ;; *) exit 1 ;; esac + RUN mkdir -p /tmp/promptfoo-fixture /opt/venv/bin \ + && printf '%s\n' '{"name":"promptfoo","version":"1.0.0","bin":"cli.js"}' > /tmp/promptfoo-fixture/package.json \ + && printf '%s\n' '#!/usr/bin/env node' 'console.log("global-promptfoo")' > /tmp/promptfoo-fixture/cli.js \ + && chmod +x /tmp/promptfoo-fixture/cli.js \ + && printf '%s\n' '#!/bin/sh' 'echo python-promptfoo' > /usr/local/bin/promptfoo \ + && chmod +x /usr/local/bin/promptfoo && cp /usr/local/bin/promptfoo /opt/venv/bin/promptfoo \ + && npm pack --offline --pack-destination /tmp /usr/lib/node_modules/npm + EOF + case "$distro" in + bookworm-env|bookworm-npmrc) + cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' + ENV HOME=/home/app + USER app + RUN test "$(npm prefix --global)" = /home/app/.npm-global && test "$(command -v promptfoo)" = /usr/local/bin/promptfoo + EOF + ;; + *) + cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' + RUN test "$(npm prefix --global)" = /usr && test "$(command -v promptfoo)" = /opt/venv/bin/promptfoo + EOF + ;; + esac + cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' + RUN prefix="$(npm prefix --global)" \ + && python -c "import sysconfig; assert sysconfig.get_path('scripts') != '$prefix/bin'" \ + && npm install --offline --no-audit --no-fund --global /tmp/promptfoo-fixture \ + && test "$(promptfoo)" = python-promptfoo && test "$("$prefix/bin/promptfoo")" = global-promptfoo \ + && npm install --offline --no-audit --no-fund --global /tmp/npm-*.tgz \ + && test "$(readlink -f "$(command -v npm)")" = "$prefix/lib/node_modules/npm/bin/npm-cli.js" \ + && test "$(readlink -f "$(command -v npx)")" = "$prefix/lib/node_modules/npm/bin/npx-cli.js" \ + && npm --version && npx --version EOF fi docker build --progress=plain "$instruction_dir/$distro" diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index a02efc1..03ad1e4 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -42,14 +42,16 @@ def _container_instructions(env: Environment) -> list[str]: "Run these build steps as root; restore your original USER afterward if needed.", f" FROM node:24-{debian_release}-slim AS node", f" FROM python:3.12-slim-{debian_release}", - " COPY --from=node /usr/local/bin/node /usr/local/bin/node", - " COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules", - " RUN ln -sf /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm " - "&& ln -sf /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx", - " RUN ln -sf /usr/local/bin/node /usr/bin/node && ln -sf /usr/local/bin/npm /usr/bin/npm " - "&& ln -sf /usr/local/bin/npx /usr/bin/npx && mkdir -p /opt/npm-global", - ' ENV NPM_CONFIG_PREFIX="/opt/npm-global"', - ' ENV PATH="${PATH}:/usr/local/bin:/opt/npm-global/bin"', + " COPY --from=node /usr/local/bin/node /usr/bin/node", + " COPY --from=node /usr/local/lib/node_modules /usr/lib/node_modules", + " RUN ln -sf /usr/lib/node_modules/npm/bin/npm-cli.js /usr/bin/npm " + "&& ln -sf /usr/lib/node_modules/npm/bin/npx-cli.js /usr/bin/npx", + ' ENV PATH="${PATH}:/usr/local/bin"', + "The default npm global prefix is /usr; the Python image installs scripts in /usr/local/bin.", + "Keep any existing writable npm prefix. If its bin is missing from PATH, add it before /usr/bin " + "and after your Python scripts (for example /opt/venv/bin or /usr/local/bin).", + "A non-root user without a writable prefix can configure one with " + "`npm config set prefix ~/.npm-global` and add that bin directory to PATH in the same order.", " Official Node.js images: https://hub.docker.com/_/node", ] if env.linux_distro == "ubuntu": diff --git a/tests/smoke/test_installation.py b/tests/smoke/test_installation.py index c8fce5b..fe85155 100644 --- a/tests/smoke/test_installation.py +++ b/tests/smoke/test_installation.py @@ -83,9 +83,10 @@ def test_windows_verification_commands_run_in_git_bash() -> None: @pytest.mark.skipif(sys.platform != "win32", reason="Tests native Windows executable lookup") @pytest.mark.parametrize("shell", ["powershell", "git-bash"]) def test_windows_npx_executable_alternative_works_in_each_shell(tmp_path: Path, shell: str) -> None: - node = shutil.which("node") - assert node - shutil.copy2(node, tmp_path / "npx.exe") + launcher = shutil.which("node") + assert launcher + node = subprocess.run([launcher, "-p", "process.execPath"], capture_output=True, text=True, check=True, timeout=20) + shutil.copy2(node.stdout.strip(), tmp_path / "npx.exe") command = windows_verification_commands("npx.exe")[1] if shell == "powershell": powershell = shutil.which("powershell") diff --git a/tests/test_instructions.py b/tests/test_instructions.py index d4b5a27..9e9a94e 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -94,14 +94,15 @@ def test_ci_container_and_wsl_hints_can_coexist() -> None: assert "FROM node:24-bookworm-slim AS node" in output assert "FROM python:3.12-slim-bookworm" in output assert "Keep the second FROM set to your existing image and Python environment" in output - assert "COPY --from=node /usr/local/bin/node /usr/local/bin/node" in output - assert "npm/bin/npx-cli.js /usr/local/bin/npx" in output - assert "/usr/local/bin/node /usr/bin/node" in output - assert "/usr/local/bin/npm /usr/bin/npm" in output - assert "/usr/local/bin/npx /usr/bin/npx" in output - assert "venv" not in output - assert 'ENV NPM_CONFIG_PREFIX="/opt/npm-global"' in output - assert 'ENV PATH="${PATH}:/usr/local/bin:/opt/npm-global/bin"' in output + assert "COPY --from=node /usr/local/bin/node /usr/bin/node" in output + assert "COPY --from=node /usr/local/lib/node_modules /usr/lib/node_modules" in output + assert "npm/bin/npm-cli.js /usr/bin/npm" in output + assert "npm/bin/npx-cli.js /usr/bin/npx" in output + assert 'ENV PATH="${PATH}:/usr/local/bin"' in output + assert "NPM_CONFIG_PREFIX" not in output + assert "Keep any existing writable npm prefix" in output + assert "add it before /usr/bin and after your Python scripts" in output + assert "npm config set prefix ~/.npm-global" in output assert "nvm install" not in output assert "https://hub.docker.com/_/node" in output assert "install Node.js inside your Linux distribution" in output @@ -140,8 +141,8 @@ def test_trixie_container_uses_matching_supported_node_and_python_images() -> No assert "FROM node:24-trixie-slim AS node" in output assert "FROM python:3.12-slim-trixie" in output assert "/usr/local/bin/node /usr/bin/node" in output - assert 'ENV NPM_CONFIG_PREFIX="/opt/npm-global"' in output - assert 'ENV PATH="${PATH}:/usr/local/bin:/opt/npm-global/bin"' in output + assert 'ENV PATH="${PATH}:/usr/local/bin"' in output + assert "NPM_CONFIG_PREFIX" not in output assert "bookworm" not in output From 2c329b52a76ed115b377d4fe01f4ab09652fb225 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 18 Sep 2026 14:48:57 -0700 Subject: [PATCH 12/12] fix: install Debian Node with apt and exercise npm upgrades --- .github/workflows/test.yml | 6 +++++- src/promptfoo/instructions.py | 38 +++++++++++++++++++++-------------- tests/test_instructions.py | 26 ++++++++++++------------ 3 files changed, 41 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68ab9ee..77a45bf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -120,13 +120,16 @@ jobs: cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' RUN python -c "import sys, promptfoo_help_probe; assert sys.version_info[:2] == (3, 12)" RUN env PATH=/opt/venv/bin:/usr/bin:/bin sh -c "node --version && npm --version && npx --version" + RUN dpkg-query -W -f='${Version}' nodejs | grep -E '^24\.' \ + && apt-get update && apt-get install -y --no-install-recommends nodejs \ + && node --version | grep -E '^v24\.' && rm -rf /var/lib/apt/lists/* RUN mkdir -p /tmp/promptfoo-fixture /opt/venv/bin \ && printf '%s\n' '{"name":"promptfoo","version":"1.0.0","bin":"cli.js"}' > /tmp/promptfoo-fixture/package.json \ && printf '%s\n' '#!/usr/bin/env node' 'console.log("global-promptfoo")' > /tmp/promptfoo-fixture/cli.js \ && chmod +x /tmp/promptfoo-fixture/cli.js \ && printf '%s\n' '#!/bin/sh' 'echo python-promptfoo' > /usr/local/bin/promptfoo \ && chmod +x /usr/local/bin/promptfoo && cp /usr/local/bin/promptfoo /opt/venv/bin/promptfoo \ - && npm pack --offline --pack-destination /tmp /usr/lib/node_modules/npm + && npm pack --offline --ignore-scripts --pack-destination /tmp /usr/lib/node_modules/npm EOF case "$distro" in bookworm-env|bookworm-npmrc) @@ -148,6 +151,7 @@ jobs: && npm install --offline --no-audit --no-fund --global /tmp/promptfoo-fixture \ && test "$(promptfoo)" = python-promptfoo && test "$("$prefix/bin/promptfoo")" = global-promptfoo \ && npm install --offline --no-audit --no-fund --global /tmp/npm-*.tgz \ + && hash -r \ && test "$(readlink -f "$(command -v npm)")" = "$prefix/lib/node_modules/npm/bin/npm-cli.js" \ && test "$(readlink -f "$(command -v npx)")" = "$prefix/lib/node_modules/npm/bin/npx-cli.js" \ && npm --version && npx --version diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index 03ad1e4..0a5c6cd 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -5,6 +5,17 @@ _NODE_DOWNLOAD = "https://nodejs.org/en/download" _NVM = "https://github.com/nvm-sh/nvm#installing-and-updating" +_APT_NODE_24 = ( + " RUN apt-get update && apt-get install -y --no-install-recommends bash ca-certificates curl " + "&& curl -fsSL https://deb.nodesource.com/setup_24.x -o /tmp/nodesource-setup.sh " + "&& bash /tmp/nodesource-setup.sh && rm /tmp/nodesource-setup.sh " + "&& apt-get install -y --no-install-recommends nodejs && rm -rf /var/lib/apt/lists/*" +) +_NODESOURCE = " NodeSource's apt repository: https://github.com/nodesource/distributions" +_APT_ARCHITECTURES = ( + "The NodeSource apt recipe supports amd64 and arm64. For other CPU architectures, select a supported " + f"Node.js 24 installation for your target: {_NODE_DOWNLOAD}" +) _SERVERLESS = { "google": ( "Google Cloud Functions / Cloud Run", @@ -38,30 +49,27 @@ def _container_instructions(env: Environment) -> list[str]: return [ f"CONTAINER: add Node.js 24 to your existing Debian {debian_release.title()} Python image; " "this example uses Python 3.12.", - "Keep the second FROM set to your existing image and Python environment:", + "Keep FROM set to your existing image, platform, and Python environment:", "Run these build steps as root; restore your original USER afterward if needed.", - f" FROM node:24-{debian_release}-slim AS node", + _APT_ARCHITECTURES, f" FROM python:3.12-slim-{debian_release}", - " COPY --from=node /usr/local/bin/node /usr/bin/node", - " COPY --from=node /usr/local/lib/node_modules /usr/lib/node_modules", - " RUN ln -sf /usr/lib/node_modules/npm/bin/npm-cli.js /usr/bin/npm " - "&& ln -sf /usr/lib/node_modules/npm/bin/npx-cli.js /usr/bin/npx", + _APT_NODE_24, ' ENV PATH="${PATH}:/usr/local/bin"', "The default npm global prefix is /usr; the Python image installs scripts in /usr/local/bin.", - "Keep any existing writable npm prefix. If its bin is missing from PATH, add it before /usr/bin " + "Keep an existing writable npm prefix only if its bin is separate from Python's scripts directory. " + "If its bin is missing from PATH, add it before /usr/bin " "and after your Python scripts (for example /opt/venv/bin or /usr/local/bin).", - "A non-root user without a writable prefix can configure one with " - "`npm config set prefix ~/.npm-global` and add that bin directory to PATH in the same order.", - " Official Node.js images: https://hub.docker.com/_/node", + "If an inherited NPM_CONFIG_PREFIX points to an unwritable directory, change it in the Dockerfile " + "(for example `ENV NPM_CONFIG_PREFIX=/home/app/.npm-global`). Without that override, a non-root user " + "can run `npm config set prefix ~/.npm-global`. Add the writable bin to PATH in either case.", + _NODESOURCE, ] if env.linux_distro == "ubuntu": return [ "UBUNTU CONTAINER: keep your existing FROM and install Node.js 24 in place as root:", - " RUN apt-get update && apt-get install -y --no-install-recommends bash ca-certificates curl " - "&& curl -fsSL https://deb.nodesource.com/setup_24.x -o /tmp/nodesource-setup.sh " - "&& bash /tmp/nodesource-setup.sh && rm /tmp/nodesource-setup.sh " - "&& apt-get install -y --no-install-recommends nodejs && rm -rf /var/lib/apt/lists/*", - " NodeSource's apt repository: https://github.com/nodesource/distributions", + _APT_ARCHITECTURES, + _APT_NODE_24, + _NODESOURCE, ] if env.linux_distro == "amzn" and env.linux_distro_version == "2023": return [ diff --git a/tests/test_instructions.py b/tests/test_instructions.py index 9e9a94e..22bea63 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -91,20 +91,19 @@ def test_ci_container_and_wsl_hints_can_coexist() -> None: ) assert "- uses: actions/setup-node@v7\n with:\n node-version: '24'" in output - assert "FROM node:24-bookworm-slim AS node" in output assert "FROM python:3.12-slim-bookworm" in output - assert "Keep the second FROM set to your existing image and Python environment" in output - assert "COPY --from=node /usr/local/bin/node /usr/bin/node" in output - assert "COPY --from=node /usr/local/lib/node_modules /usr/lib/node_modules" in output - assert "npm/bin/npm-cli.js /usr/bin/npm" in output - assert "npm/bin/npx-cli.js /usr/bin/npx" in output + assert "Keep FROM set to your existing image, platform, and Python environment" in output + assert "FROM node:" not in output + assert "https://deb.nodesource.com/setup_24.x" in output + assert "apt-get install -y --no-install-recommends nodejs" in output + assert "apt recipe supports amd64 and arm64. For other CPU architectures" in output assert 'ENV PATH="${PATH}:/usr/local/bin"' in output - assert "NPM_CONFIG_PREFIX" not in output - assert "Keep any existing writable npm prefix" in output + assert "If an inherited NPM_CONFIG_PREFIX points to an unwritable directory, change it" in output + assert "Keep an existing writable npm prefix only if its bin is separate from Python's scripts directory" in output assert "add it before /usr/bin and after your Python scripts" in output assert "npm config set prefix ~/.npm-global" in output assert "nvm install" not in output - assert "https://hub.docker.com/_/node" in output + assert "https://github.com/nodesource/distributions" in output assert "install Node.js inside your Linux distribution" in output @@ -131,18 +130,19 @@ def test_non_bookworm_containers_keep_their_original_base(distribution: str, ver assert len(commands) == 1 assert "apt-get update" in commands[0] assert "rm -rf /var/lib/apt/lists/*" in commands[0] + assert "apt recipe supports amd64 and arm64. For other CPU architectures" in output -def test_trixie_container_uses_matching_supported_node_and_python_images() -> None: +def test_trixie_container_installs_node_on_its_existing_platform() -> None: output = get_installation_instructions( Environment(os_type="linux", linux_distro="debian", linux_distro_version="13", is_docker=True) ) assert "Debian Trixie" in output - assert "FROM node:24-trixie-slim AS node" in output assert "FROM python:3.12-slim-trixie" in output - assert "/usr/local/bin/node /usr/bin/node" in output + assert "FROM node:" not in output + assert "apt-get install -y --no-install-recommends nodejs" in output assert 'ENV PATH="${PATH}:/usr/local/bin"' in output - assert "NPM_CONFIG_PREFIX" not in output + assert "existing image, platform, and Python environment" in output assert "bookworm" not in output