diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46a309d..77a45bf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,28 +71,94 @@ jobs: - name: Check GitHub workflows run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 - - name: Verify Alpine installation instructions + - name: Verify container installation instructions run: | - alpine_dir="$RUNNER_TEMP/promptfoo-alpine" - mkdir -p "$alpine_dir" - uv run python - <<'PY' > "$alpine_dir/Dockerfile" + instruction_dir="$RUNNER_TEMP/promptfoo-docker-help" + uv run python - "$instruction_dir" <<'PY' + import sys + from pathlib import Path + from promptfoo.environment import Environment from promptfoo.instructions import get_installation_instructions - instructions = get_installation_instructions( - Environment(os_type="linux", linux_distro="alpine", is_docker=True) - ) - for line in instructions.splitlines(): - command = line.strip() - if command.startswith(("FROM ", "RUN ", "ENV ")): - print(command) + for label, distro, version in (("alpine", "alpine", None), ("bookworm", "debian", "12"), + ("trixie", "debian", "13"), ("bookworm-env", "debian", "12"), + ("bookworm-npmrc", "debian", "12")): + instructions = get_installation_instructions( + Environment(os_type="linux", linux_distro=distro, linux_distro_version=version, is_docker=True) + ) + commands = [line.strip() for line in instructions.splitlines()] + commands = [command for command in commands if command.startswith(("FROM ", "COPY ", "RUN ", "ENV "))] + if distro == "debian": + runtime = next(index for index, command in enumerate(commands) if command.startswith("FROM python:")) + inherited = ["RUN python -c 'import pathlib, sysconfig; " + "(pathlib.Path(sysconfig.get_path(\"purelib\")) / \"promptfoo_help_probe.py\").touch()'"] + if label in ("bookworm-env", "bookworm-npmrc"): + inherited += ["RUN useradd -m app && mkdir -p /home/app/.npm-global " + "&& chown app:app /home/app/.npm-global", + 'ENV PATH="/usr/local/bin:/home/app/.npm-global/bin:/usr/bin:/bin"'] + if label == "bookworm-env": + inherited.append('ENV NPM_CONFIG_PREFIX="/home/app/.npm-global"') + else: + inherited.append("RUN printf '%s\\n' 'prefix=/home/app/.npm-global' > /home/app/.npmrc " + "&& chown app:app /home/app/.npmrc") + else: + inherited.append('ENV PATH="/opt/venv/bin:/usr/bin:/bin"') + commands[runtime + 1:runtime + 1] = inherited + directory = Path(sys.argv[1]) / label + directory.mkdir(parents=True, exist_ok=True) + (directory / "Dockerfile").write_text("\n".join(commands) + "\n") PY - cat >> "$alpine_dir/Dockerfile" <<'EOF' + for distro in alpine bookworm trixie bookworm-env bookworm-npmrc; do + cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' RUN node --version | grep -E '^v24\.' \ - && npm --version && npx --version && python --version && pip --version \ - && python -c 'import sys; assert sys.prefix == "/opt/venv"' + && npm --version && npx --version && python --version && pip --version + EOF + if [ "$distro" = alpine ]; then + echo 'RUN python -c "import sys; assert sys.prefix == \"/opt/venv\""' >> "$instruction_dir/$distro/Dockerfile" + else + cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' + RUN python -c "import sys, promptfoo_help_probe; assert sys.version_info[:2] == (3, 12)" + RUN env PATH=/opt/venv/bin:/usr/bin:/bin sh -c "node --version && npm --version && npx --version" + RUN dpkg-query -W -f='${Version}' nodejs | grep -E '^24\.' \ + && apt-get update && apt-get install -y --no-install-recommends nodejs \ + && node --version | grep -E '^v24\.' && rm -rf /var/lib/apt/lists/* + RUN mkdir -p /tmp/promptfoo-fixture /opt/venv/bin \ + && printf '%s\n' '{"name":"promptfoo","version":"1.0.0","bin":"cli.js"}' > /tmp/promptfoo-fixture/package.json \ + && printf '%s\n' '#!/usr/bin/env node' 'console.log("global-promptfoo")' > /tmp/promptfoo-fixture/cli.js \ + && chmod +x /tmp/promptfoo-fixture/cli.js \ + && printf '%s\n' '#!/bin/sh' 'echo python-promptfoo' > /usr/local/bin/promptfoo \ + && chmod +x /usr/local/bin/promptfoo && cp /usr/local/bin/promptfoo /opt/venv/bin/promptfoo \ + && npm pack --offline --ignore-scripts --pack-destination /tmp /usr/lib/node_modules/npm + EOF + case "$distro" in + bookworm-env|bookworm-npmrc) + cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' + ENV HOME=/home/app + USER app + RUN test "$(npm prefix --global)" = /home/app/.npm-global && test "$(command -v promptfoo)" = /usr/local/bin/promptfoo EOF - docker build --progress=plain "$alpine_dir" + ;; + *) + cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' + RUN test "$(npm prefix --global)" = /usr && test "$(command -v promptfoo)" = /opt/venv/bin/promptfoo + EOF + ;; + esac + cat >> "$instruction_dir/$distro/Dockerfile" <<'EOF' + RUN prefix="$(npm prefix --global)" \ + && python -c "import sysconfig; assert sysconfig.get_path('scripts') != '$prefix/bin'" \ + && npm install --offline --no-audit --no-fund --global /tmp/promptfoo-fixture \ + && test "$(promptfoo)" = python-promptfoo && test "$("$prefix/bin/promptfoo")" = global-promptfoo \ + && npm install --offline --no-audit --no-fund --global /tmp/npm-*.tgz \ + && hash -r \ + && test "$(readlink -f "$(command -v npm)")" = "$prefix/lib/node_modules/npm/bin/npm-cli.js" \ + && test "$(readlink -f "$(command -v npx)")" = "$prefix/lib/node_modules/npm/bin/npx-cli.js" \ + && npm --version && npx --version + EOF + fi + docker build --progress=plain "$instruction_dir/$distro" + done docker run --rm alpine:3.20 sh -ec ' apk update apk add --no-cache nodejs npm diff --git a/AGENTS.md b/AGENTS.md index 569da0e..fb8915e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -508,12 +508,8 @@ git push --force ### Q: The release-please PR shows the wrong version. How do I fix it? -**A**: The version bump is determined by commit messages: -- Check all commits since the last release -- Ensure they follow conventional commits -- `fix:` commits bump minor version (pre-1.0.0) -- `feat:` commits bump minor version (pre-1.0.0) -- If the version is still wrong, you may need to manually adjust `.release-please-manifest.json` in a new PR +**A**: Check the commits since the last release against [Version Bumping Strategy](#version-bumping-strategy). +That section is the source of truth for the pre-1.0 rules and links to the release-please configuration. ### Q: How do I manually publish to PyPI? diff --git a/src/promptfoo/environment.py b/src/promptfoo/environment.py index 1d9ae43..03fe094 100644 --- a/src/promptfoo/environment.py +++ b/src/promptfoo/environment.py @@ -1,357 +1,106 @@ -""" -Environment detection for providing contextual Node.js installation instructions. - -This module detects the operating system, Linux distribution, cloud provider, -container environment, CI/CD platform, and Python environment to provide -tailored installation instructions for Node.js. -""" +"""Detect only the environment details that change the missing-Node help.""" import os +import platform import sys from dataclasses import dataclass from pathlib import Path -@dataclass +@dataclass(frozen=True) class Environment: - """Information about the current execution environment.""" + """Platform information used by the Node installation instructions.""" - os_type: str # "linux", "darwin", "windows" - linux_distro: str | None = None # "ubuntu", "debian", "rhel", "fedora", "alpine", "arch", etc. - linux_distro_version: str | None = None # e.g., "22.04", "11", "9" - cloud_provider: str | None = None # "aws", "gcp", "azure" - is_lambda: bool = False # AWS Lambda - is_cloud_function: bool = False # GCP Cloud Functions or Azure Functions + os_type: str + linux_distro: str | None = None + linux_distro_version: str | None = None is_docker: bool = False - is_kubernetes: bool = False - is_wsl: bool = False # Windows Subsystem for Linux - is_ci: bool = False - ci_platform: str | None = None # "github", "gitlab", "circleci", "jenkins", etc. - is_venv: bool = False - is_conda: bool = False - has_sudo: bool = False # Best guess if user has sudo access + is_wsl: bool = False + ci_platform: str | None = None + serverless: str | None = None -def _read_probe_file(path: Path) -> str | None: - """ - Read an optional environment probe file. +def _read_probe(path: str) -> str: + try: + return Path(path).read_text(encoding="utf-8") + except (OSError, UnicodeError): + return "" - Returns: - File contents, or None when the probe file does not exist or cannot be read. - """ - if not path.exists(): - return None +def _linux_release() -> tuple[str | None, str | None]: try: - with open(path) as f: - return f.read() + release = platform.freedesktop_os_release() except OSError: - # Environment detection is best-effort. Proc/sys metadata files can be - # unreadable or disappear between exists() and open(), so treat that as - # "signal unavailable" and continue with fallback probes. - return None - - -def _detect_linux_distro() -> tuple[str | None, str | None]: - """ - Detect Linux distribution and version. - - Returns: - Tuple of (distro_id, version) where distro_id is normalized - (e.g., "ubuntu", "debian", "rhel", "alpine", "arch") - """ - # Define known distros for normalization - known_base_distros = {"ubuntu", "debian", "alpine", "arch", "fedora"} - rhel_family = {"rhel", "centos", "rocky", "almalinux", "ol", "amzn"} - suse_family = {"opensuse", "opensuse-leap", "opensuse-tumbleweed", "sles"} - - # Try /etc/os-release first, then /usr/lib/os-release (per freedesktop spec) - for os_release_path in [Path("/etc/os-release"), Path("/usr/lib/os-release")]: - os_release_content = _read_probe_file(os_release_path) - if os_release_content is None: - continue - - os_release = {} - for line in os_release_content.splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - if "=" in line: - key, _, value = line.partition("=") - # Remove quotes - value = value.strip('"').strip("'") - os_release[key] = value - - distro_id = os_release.get("ID", "").lower() - version = os_release.get("VERSION_ID", "") - id_like = os_release.get("ID_LIKE", "").lower().split() - - # Normalize distro IDs - if distro_id in known_base_distros: - return distro_id, version - elif distro_id in rhel_family: - # Oracle Linux (ol), Amazon Linux (amzn) - return "rhel", version - elif distro_id in suse_family: - return "suse", version - - # Check ID_LIKE for derivative distributions (e.g., Pop!_OS, Raspbian, Mint) - if id_like: - for parent in id_like: - if parent in known_base_distros: - return parent, version - elif parent in rhel_family: - return "rhel", version - elif parent in suse_family: - return "suse", version - - # Return the raw distro_id if we couldn't normalize it - return distro_id, version - - # Fallback: check for specific files - if Path("/etc/debian_version").exists(): - return "debian", None - elif Path("/etc/redhat-release").exists(): - return "rhel", None - elif Path("/etc/alpine-release").exists(): - return "alpine", None - elif Path("/etc/arch-release").exists(): - return "arch", None - - return None, None - + release = {} + + distro = release.get("ID", "").lower() + version = release.get("VERSION_ID") or None + if not distro and (alpine_version := _read_probe("/etc/alpine-release").strip()): + return "alpine", alpine_version + family = [distro, *release.get("ID_LIKE", "").lower().split()] + if "alpine" in family: + return "alpine", version + if distro in ("amzn", "amazon"): + return "amzn", version + return distro or None, version + + +def _in_container() -> bool: + if os.environ.get("KUBERNETES_SERVICE_HOST") or Path("/.dockerenv").is_file(): + return True + cgroup = _read_probe("/proc/1/cgroup").lower() + return any(runtime in cgroup for runtime in ("docker", "containerd", "kubepods", "crio")) + + +def _ci_platform() -> str | None: + for variable, name in ( + ("GITHUB_ACTIONS", "GitHub Actions"), + ("GITLAB_CI", "GitLab CI"), + ("CIRCLECI", "CircleCI"), + ("JENKINS_URL", "Jenkins"), + ("JENKINS_HOME", "Jenkins"), + ("BUILDKITE", "Buildkite"), + ("TF_BUILD", "Azure Pipelines"), + ("TEAMCITY_VERSION", "TeamCity"), + ("TRAVIS", "Travis CI"), + ("DRONE", "Drone CI"), + ("BITBUCKET_BUILD_NUMBER", "Bitbucket Pipelines"), + ("CONTINUOUS_INTEGRATION", "CI"), + ("CI", "CI"), + ): + if os.environ.get(variable): + return name + return None -def _detect_cloud_provider() -> str | None: - """ - Detect if running on a cloud provider. - Returns: - One of "aws", "gcp", "azure", or None - """ - # AWS detection - # Check for EC2 metadata - uuid = _read_probe_file(Path("/sys/hypervisor/uuid")) - if uuid and uuid.strip().lower().startswith("ec2"): +def _serverless() -> str | None: + if os.environ.get("AWS_LAMBDA_FUNCTION_NAME"): return "aws" - - # Check AWS environment variables - if os.getenv("AWS_EXECUTION_ENV") or os.getenv("AWS_REGION"): - return "aws" - - # GCP detection - # Check for GCP metadata - product = _read_probe_file(Path("/sys/class/dmi/id/product_name")) - if product: - product = product.strip() - if "Google" in product or "GCE" in product: - return "gcp" - - # Check GCP environment variables - if os.getenv("GOOGLE_CLOUD_PROJECT") or os.getenv("GCP_PROJECT"): - return "gcp" - - # Azure detection - vendor = _read_probe_file(Path("/sys/class/dmi/id/sys_vendor")) - # Could be Azure or Hyper-V, check for Azure-specific - if vendor and "Microsoft Corporation" in vendor.strip() and Path("/var/lib/waagent").exists(): + if os.environ.get("FUNCTIONS_WORKER_RUNTIME") and any( + os.environ.get(name) + for name in ("WEBSITE_INSTANCE_ID", "WEBSITE_SITE_NAME", "CONTAINER_APP_NAME", "KUBERNETES_SERVICE_HOST") + ): return "azure" - - # Check Azure environment variables - if os.getenv("AZURE_SUBSCRIPTION_ID") or os.getenv("WEBSITE_INSTANCE_ID"): - return "azure" - + if os.environ.get("FUNCTION_NAME") or (os.environ.get("FUNCTION_TARGET") and os.environ.get("K_SERVICE")): + return "google" return None -def _detect_container() -> tuple[bool, bool]: - """ - Detect if running in a container. - - Returns: - Tuple of (is_docker, is_kubernetes) - """ - is_docker = False - is_kubernetes = False - - # Docker detection - if Path("/.dockerenv").exists(): - is_docker = True - - # Also check cgroup - cgroup_content = _read_probe_file(Path("/proc/1/cgroup")) - if cgroup_content and ("docker" in cgroup_content or "containerd" in cgroup_content): - is_docker = True - - # Kubernetes detection - if os.getenv("KUBERNETES_SERVICE_HOST"): - is_kubernetes = True - - return is_docker, is_kubernetes - - -def _detect_wsl() -> bool: - """ - Detect if running in Windows Subsystem for Linux (WSL). - - Returns: - True if running in WSL, False otherwise - """ - # Check for WSL environment variable - if os.getenv("WSL_DISTRO_NAME") or os.getenv("WSL_INTEROP"): - return True - - # Check /proc/version for Microsoft/WSL signatures - version_info = _read_probe_file(Path("/proc/version")) - if version_info: - version_info = version_info.lower() - if "microsoft" in version_info or "wsl" in version_info: - return True - - # Check for Windows filesystem mounts (WSL mounts Windows drives at /mnt/) - # This is less reliable but can catch WSL 1 - return Path("/mnt/c").exists() and Path("/proc/version").exists() - - -def _detect_ci() -> tuple[bool, str | None]: - """ - Detect if running in a CI/CD environment. - - Returns: - Tuple of (is_ci, ci_platform) - """ - ci_env_vars = { - "GITHUB_ACTIONS": "github", - "GITLAB_CI": "gitlab", - "CIRCLECI": "circleci", - "JENKINS_HOME": "jenkins", - "TRAVIS": "travis", - "BUILDKITE": "buildkite", - "DRONE": "drone", - "BITBUCKET_BUILD_NUMBER": "bitbucket", - "TEAMCITY_VERSION": "teamcity", - "TF_BUILD": "azure-devops", - } - - for env_var, platform in ci_env_vars.items(): - if os.getenv(env_var): - return True, platform - - # Generic CI detection - if os.getenv("CI"): - return True, None - - return False, None - - -def _detect_python_env() -> tuple[bool, bool]: - """ - Detect Python virtual environment. - - Returns: - Tuple of (is_venv, is_conda) - """ - # venv/virtualenv detection - is_venv = hasattr(sys, "real_prefix") or (hasattr(sys, "base_prefix") and sys.base_prefix != sys.prefix) - - # Conda detection - is_conda = "CONDA_DEFAULT_ENV" in os.environ or "CONDA_PREFIX" in os.environ - - return is_venv, is_conda - - -def _has_sudo_access() -> bool: - """ - Best-effort check if user likely has sudo access. - - Returns: - True if user is root or likely has sudo, False otherwise - """ - # Unix-like systems - if hasattr(os, "geteuid"): - # Root user - if os.geteuid() == 0: - return True - - # Check if sudo command exists - import shutil - - return shutil.which("sudo") is not None - - # Windows - check if admin (requires elevation detection) - if sys.platform == "win32": - try: - import ctypes - - return ctypes.windll.shell32.IsUserAnAdmin() != 0 - except Exception: - return False - - return False - - def detect_environment() -> Environment: - """ - Detect the current execution environment. - - Returns: - Environment object with detected platform information - """ - os_type = sys.platform - if os_type.startswith("linux"): - os_type = "linux" - elif os_type == "darwin": - os_type = "darwin" - elif os_type == "win32": - os_type = "windows" - - # Linux-specific detection - linux_distro = None - linux_distro_version = None - if os_type == "linux": - linux_distro, linux_distro_version = _detect_linux_distro() - - # Cloud provider detection - cloud_provider = _detect_cloud_provider() - - # Lambda and Cloud Functions detection - is_lambda = os.getenv("AWS_LAMBDA_FUNCTION_NAME") is not None - is_cloud_function = ( - os.getenv("FUNCTION_NAME") is not None # GCP Cloud Functions - or os.getenv("FUNCTIONS_WORKER_RUNTIME") is not None # Azure Functions + """Return the current platform and the deployment hints relevant to installation.""" + os_type = "linux" if sys.platform.startswith("linux") else {"win32": "windows"}.get(sys.platform, sys.platform) + distro, version = _linux_release() if os_type == "linux" else (None, None) + is_wsl = os_type == "linux" and bool( + os.environ.get("WSL_DISTRO_NAME") + or os.environ.get("WSL_INTEROP") + or "microsoft" in _read_probe("/proc/sys/kernel/osrelease").lower() ) - - # Container detection - is_docker, is_kubernetes = False, False - if os_type == "linux": - is_docker, is_kubernetes = _detect_container() - - # WSL detection - is_wsl = False - if os_type == "linux": - is_wsl = _detect_wsl() - - # CI detection - is_ci, ci_platform = _detect_ci() - - # Python environment detection - is_venv, is_conda = _detect_python_env() - - # Sudo detection - has_sudo = _has_sudo_access() - return Environment( os_type=os_type, - linux_distro=linux_distro, - linux_distro_version=linux_distro_version, - cloud_provider=cloud_provider, - is_lambda=is_lambda, - is_cloud_function=is_cloud_function, - is_docker=is_docker, - is_kubernetes=is_kubernetes, + linux_distro=distro, + linux_distro_version=version, + is_docker=os_type == "linux" and _in_container(), is_wsl=is_wsl, - is_ci=is_ci, - ci_platform=ci_platform, - is_venv=is_venv, - is_conda=is_conda, - has_sudo=has_sudo, + ci_platform=_ci_platform(), + serverless=_serverless(), ) diff --git a/src/promptfoo/instructions.py b/src/promptfoo/instructions.py index 2662cc5..0a5c6cd 100644 --- a/src/promptfoo/instructions.py +++ b/src/promptfoo/instructions.py @@ -1,432 +1,189 @@ -""" -Platform-specific Node.js installation instructions. - -Generates tailored installation instructions based on the detected environment. -""" +"""Concise Node.js installation guidance for the current platform.""" from .environment import Environment from .node import MIN_NODE_VERSION_TEXT - -def get_installation_instructions(env: Environment) -> str: - """ - Generate Node.js installation instructions for the detected environment. - - Args: - env: Detected environment information - - Returns: - Formatted installation instructions as a multi-line string - """ - lines = [] - lines.append("=" * 70) - lines.append(f"ERROR: promptfoo requires Node.js {MIN_NODE_VERSION_TEXT} or newer but it's not installed") - lines.append("=" * 70) - lines.append("Install a supported version and verify it with: node --version") - lines.append("") - - # Special cases first (Lambda, Cloud Functions, etc.) - if env.is_lambda: - lines.extend(_get_lambda_instructions()) - return "\n".join(lines) - - if env.is_cloud_function: - lines.extend(_get_cloud_function_instructions(env)) - return "\n".join(lines) - - # CI/CD environment - if env.is_ci: - lines.extend(_get_ci_instructions(env)) - lines.append("") - - # Container environment - if env.is_docker: - lines.extend(_get_docker_instructions(env)) - lines.append("") - - # WSL environment - if env.is_wsl: - lines.extend(_get_wsl_instructions()) - lines.append("") - - # Platform-specific instructions - if env.os_type == "linux": - lines.extend(_get_linux_instructions(env)) - elif env.os_type == "darwin": - lines.extend(_get_macos_instructions()) - elif env.os_type == "windows": - lines.extend(_get_windows_instructions()) - - # Virtual environment alternative - if env.is_venv or env.is_conda: - lines.append("") - lines.extend(_get_venv_instructions()) - - # Direct npx usage - lines.append("") - lines.extend(_get_npx_instructions()) - - return "\n".join(lines) - - -def _get_lambda_instructions() -> list[str]: - """Instructions for AWS Lambda environment.""" +_NODE_DOWNLOAD = "https://nodejs.org/en/download" +_NVM = "https://github.com/nvm-sh/nvm#installing-and-updating" +_APT_NODE_24 = ( + " RUN apt-get update && apt-get install -y --no-install-recommends bash ca-certificates curl " + "&& curl -fsSL https://deb.nodesource.com/setup_24.x -o /tmp/nodesource-setup.sh " + "&& bash /tmp/nodesource-setup.sh && rm /tmp/nodesource-setup.sh " + "&& apt-get install -y --no-install-recommends nodejs && rm -rf /var/lib/apt/lists/*" +) +_NODESOURCE = " NodeSource's apt repository: https://github.com/nodesource/distributions" +_APT_ARCHITECTURES = ( + "The NodeSource apt recipe supports amd64 and arm64. For other CPU architectures, select a supported " + f"Node.js 24 installation for your target: {_NODE_DOWNLOAD}" +) +_SERVERLESS = { + "google": ( + "Google Cloud Functions / Cloud Run", + "Functions deployed from source, including first-generation functions, must move to a Cloud Run service " + "to deploy a custom image.", + "https://docs.cloud.google.com/run/docs/building/containers", + ), + "azure": ( + "Azure Functions", + "Consumption and Flex Consumption do not accept custom images. Move to Azure Container Apps " + "or a Linux Premium/Dedicated plan. Select that hosting environment in the documentation. " + "On Kubernetes, keep the Azure Functions base image and add Node.", + "https://learn.microsoft.com/en-us/azure/azure-functions/functions-how-to-custom-container", + ), +} + + +def _container_instructions(env: Environment) -> list[str]: + if env.linux_distro == "alpine": + return [ + "CONTAINER: start from the official Node.js 24 Alpine image and add Python:", + " FROM node:24-alpine", + " RUN apk add --no-cache python3 py3-pip", + " RUN python3 -m venv /opt/venv", + ' ENV PATH="/opt/venv/bin:$PATH"', + ] + debian_release = {"12": "bookworm", "bookworm": "bookworm", "13": "trixie", "trixie": "trixie"}.get( + env.linux_distro_version or "" + ) + if env.linux_distro == "debian" and debian_release: + return [ + f"CONTAINER: add Node.js 24 to your existing Debian {debian_release.title()} Python image; " + "this example uses Python 3.12.", + "Keep FROM set to your existing image, platform, and Python environment:", + "Run these build steps as root; restore your original USER afterward if needed.", + _APT_ARCHITECTURES, + f" FROM python:3.12-slim-{debian_release}", + _APT_NODE_24, + ' ENV PATH="${PATH}:/usr/local/bin"', + "The default npm global prefix is /usr; the Python image installs scripts in /usr/local/bin.", + "Keep an existing writable npm prefix only if its bin is separate from Python's scripts directory. " + "If its bin is missing from PATH, add it before /usr/bin " + "and after your Python scripts (for example /opt/venv/bin or /usr/local/bin).", + "If an inherited NPM_CONFIG_PREFIX points to an unwritable directory, change it in the Dockerfile " + "(for example `ENV NPM_CONFIG_PREFIX=/home/app/.npm-global`). Without that override, a non-root user " + "can run `npm config set prefix ~/.npm-global`. Add the writable bin to PATH in either case.", + _NODESOURCE, + ] + if env.linux_distro == "ubuntu": + return [ + "UBUNTU CONTAINER: keep your existing FROM and install Node.js 24 in place as root:", + _APT_ARCHITECTURES, + _APT_NODE_24, + _NODESOURCE, + ] + if env.linux_distro == "amzn" and env.linux_distro_version == "2023": + return [ + "AMAZON LINUX 2023 CONTAINER: keep your existing FROM and install Node.js 24 as root:", + " RUN dnf --releasever=latest install -y nodejs24 nodejs24-npm " + "&& /usr/sbin/alternatives --set node /usr/bin/node-24 && dnf clean all", + "Node 24 requires repository release 2023.9.20251110 or newer; replace latest with your approved recent " + "snapshot, or update an older base image before installing.", + " https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html", + " https://docs.aws.amazon.com/linux/al2023/ug/managing-repos-os-updates.html", + ] + if env.linux_distro == "amzn" and env.linux_distro_version == "2": + return [ + "AMAZON LINUX 2 CONTAINER: move to an Amazon Linux 2023 or another Node.js 24 compatible base image.", + "Official Node.js 22/24 Linux binaries require newer glibc than Amazon Linux 2 provides.", + " https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html", + ] return [ - "You are running in AWS Lambda with a Python runtime.", - "", - "AWS Lambda Python runtimes do not include Node.js. You have options:", - "", - "1. Use a Lambda Layer with Node.js:", - " https://docs.aws.amazon.com/lambda/latest/dg/chapter-layers.html", - "", - "2. Switch to Node.js runtime:", - " https://docs.aws.amazon.com/lambda/latest/dg/lambda-nodejs.html", - "", - "3. Use Lambda container images with both Python and Node.js:", - " https://docs.aws.amazon.com/lambda/latest/dg/images-create.html", - "", - "Note: promptfoo is primarily designed for local development and CI/CD,", - "not for Lambda runtime execution.", + "CONTAINER: keep your existing base image and install Node.js 24 with npm during the image build.", + "Use installation instructions compatible with your distribution and CPU architecture:", + f" {_NODE_DOWNLOAD}", ] -def _get_cloud_function_instructions(env: Environment) -> list[str]: - """Instructions for Cloud Functions (GCP/Azure).""" - if env.cloud_provider == "gcp": +def _linux_instructions(env: Environment) -> list[str]: + if env.linux_distro == "alpine": return [ - "You are running in Google Cloud Functions with a Python runtime.", - "", - "GCP Cloud Functions Python runtimes do not include Node.js.", - "Consider using Node.js runtime instead:", - " https://cloud.google.com/functions/docs/concepts/nodejs-runtime", + "ALPINE: on a release whose repositories provide Node.js 24, run as root:", + " apk add --no-cache 'nodejs~24' npm", + " node --version", + "If apk cannot find Node.js 24, upgrade Alpine or use the official node:24-alpine container.", ] - else: # Azure or unknown + if env.linux_distro == "amzn" and env.linux_distro_version == "2023": return [ - "You are running in Azure Functions with a Python runtime.", - "", - "Azure Functions Python runtimes do not include Node.js.", - "Consider using Node.js runtime instead:", - " https://learn.microsoft.com/en-us/azure/azure-functions/functions-reference-node", + "AMAZON LINUX 2023: install and select Node.js 24 (omit sudo when running as root):", + " sudo dnf install -y nodejs24 nodejs24-npm", + " sudo /usr/sbin/alternatives --set node /usr/bin/node-24", + " node --version", + " https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html", + "If your pinned repository is older than 2023.9.20251110, update the base/repository or install from a " + "newer approved snapshot: sudo dnf --releasever=latest install -y nodejs24 nodejs24-npm", + f" Without sudo, install nvm: {_NVM}", + " Then run: nvm install 24", + ] + if env.linux_distro == "amzn" and env.linux_distro_version == "2": + return [ + "AMAZON LINUX 2: upgrade to Amazon Linux 2023 or run in a Node.js 24 compatible container.", + "Official Node.js 22/24 Linux binaries require newer glibc than Amazon Linux 2 provides.", + " https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html", ] - - -def _get_ci_instructions(env: Environment) -> list[str]: - """Instructions for CI/CD environments.""" - lines = ["RUNNING IN CI/CD: " + (env.ci_platform or "detected").upper(), ""] - - if env.ci_platform == "github": - lines.extend( - [ - "Add Node.js to your workflow:", - " - uses: actions/setup-node@v7", - " with:", - " node-version: '24'", - ] - ) - elif env.ci_platform == "gitlab": - lines.extend( - [ - "Use a Docker image with Node.js:", - " image: node:24", - ] - ) - elif env.ci_platform == "circleci": - lines.extend( - [ - "Use the CircleCI Node orb with your Python image:", - " orbs:", - " node: circleci/node@5", - " # Add under your job's steps:", - " - node/install:", - " node-version: '24'", - ] - ) - else: - lines.extend( - [ - "Install Node.js in your CI configuration.", - "Most CI platforms provide Node.js images or setup actions.", - ] - ) - - return lines - - -def _get_docker_instructions(env: Environment) -> list[str]: - """Instructions for Docker environments.""" - lines = ["RUNNING IN DOCKER CONTAINER:", ""] - - if env.linux_distro == "alpine": - lines.extend( - [ - "Start your Alpine Dockerfile from the official Node.js 24 image, then add Python:", - " FROM node:24-alpine", - " RUN apk add --no-cache python3 py3-pip", - " RUN python3 -m venv /opt/venv", - ' ENV PATH="/opt/venv/bin:$PATH"', - ] - ) - elif env.linux_distro in ("ubuntu", "debian"): - lines.extend( - [ - "Add to your Dockerfile (Debian/Ubuntu):", - " RUN apt-get update && apt-get install -y ca-certificates curl && \\", - " curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && \\", - " apt-get install -y nodejs && \\", - " rm -rf /var/lib/apt/lists/*", - ] - ) - else: - lines.extend( - [ - "Use matching Debian-based Node.js and Python stages in your Dockerfile:", - " FROM node:24-bookworm-slim AS node", - " FROM python:3.12-slim-bookworm", - " COPY --from=node /usr/local/ /usr/local/", - ] - ) - - return lines - - -def _get_wsl_instructions() -> list[str]: - """Instructions for Windows Subsystem for Linux (WSL).""" return [ - "WINDOWS SUBSYSTEM FOR LINUX (WSL) DETECTED:", - "", - "IMPORTANT: Install Node.js within WSL, not from Windows.", - "Using Windows Node.js from WSL can cause path and performance issues.", - "", - "Recommended approach:", - " 1. Use your Linux distribution's package manager (see below)", - " 2. Or use nvm for version management:", - " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash", - " source ~/.bashrc", - " nvm install 24", - "", - "Tips for WSL:", - " - Store project files in the WSL filesystem (~/), not /mnt/c/", - " - This improves file I/O performance significantly", - " - Use 'wsl --shutdown' to restart WSL if needed", + "LINUX: use a Node.js version manager or your distribution's instructions for Node.js 24.", + f" Install nvm: {_NVM}", + " Then run: nvm install 24", ] -def _get_linux_instructions(env: Environment) -> list[str]: - """Instructions for Linux systems.""" - lines = [] - distro = env.linux_distro - - if distro in ("ubuntu", "debian"): - lines.extend(_get_debian_instructions(env)) - elif distro == "rhel": - lines.extend(_get_rhel_instructions(env)) - elif distro == "alpine": - lines.extend(_get_alpine_instructions()) - elif distro == "arch": - lines.extend(_get_arch_instructions()) - elif distro == "suse": - lines.extend(_get_suse_instructions()) - else: - # Generic Linux instructions - lines.extend(_get_generic_linux_instructions()) - - return lines - - -def _get_debian_instructions(env: Environment) -> list[str]: - """Instructions for Debian/Ubuntu systems.""" - lines = ["UBUNTU/DEBIAN INSTALLATION:", ""] - - if env.has_sudo: - lines.extend( - [ - "Option 1 - Install from NodeSource (recommended for production):", - " curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash -", - " sudo apt install -y nodejs", - "", - "Option 2 - Use the default repository only if it supplies a supported version:", - " sudo apt update", - " sudo apt install -y nodejs npm", - "", - "Option 3 - Install using snap (not recommended for production):", - " sudo snap install node --classic", - " # Note: Snap auto-updates can cause unexpected behavior", - ] - ) - else: - lines.extend( - [ - "You don't have sudo access. Use nvm (Node Version Manager):", - " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash", - " source ~/.bashrc", - " nvm install 24", - ] - ) - - return lines - - -def _get_rhel_instructions(env: Environment) -> list[str]: - """Instructions for RHEL/CentOS/Fedora/Amazon Linux.""" - lines = [] - - # Detect Amazon Linux vs RHEL/CentOS/Fedora - is_amazon_linux = ( - env.linux_distro == "rhel" and env.linux_distro_version and env.linux_distro_version.startswith("202") - ) - - if is_amazon_linux: - lines.extend(["AMAZON LINUX INSTALLATION:", ""]) - if env.has_sudo: - lines.extend( - [ - "Amazon Linux 2023:", - " sudo dnf install -y nodejs", - "", - "Amazon Linux 2:", - " curl -fsSL https://rpm.nodesource.com/setup_24.x | sudo bash -", - " sudo yum install -y nodejs", - ] - ) - else: - lines.extend( - [ - "Use nvm (Node Version Manager) - no sudo needed:", - " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash", - " source ~/.bashrc", - " nvm install 24", - ] - ) - else: - lines.extend(["RHEL/CENTOS/FEDORA INSTALLATION:", ""]) - if env.has_sudo: - lines.extend( - [ - "Using dnf (RHEL 8+/Fedora):", - " sudo dnf install -y nodejs npm", - "", - "Using yum (RHEL 7):", - " sudo yum install -y nodejs npm", - "", - "Or use NodeSource for newer version:", - " curl -fsSL https://rpm.nodesource.com/setup_24.x | sudo bash -", - " sudo yum install -y nodejs", - ] - ) - else: - lines.extend( - [ - "Use nvm (Node Version Manager) - no sudo needed:", - " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash", - " source ~/.bashrc", - " nvm install 24", - ] - ) - - return lines - - -def _get_alpine_instructions() -> list[str]: - """Instructions for Alpine Linux.""" - return [ - "ALPINE LINUX INSTALLATION:", - "", - "On an Alpine release whose repositories offer Node.js 24, run as root:", - " apk add --no-cache 'nodejs~24' npm", +def get_installation_instructions(env: Environment) -> str: + """Explain how to install a supported Node.js runtime without guessing package versions.""" + lines = [ + f"ERROR: promptfoo requires Node.js {MIN_NODE_VERSION_TEXT} or newer, but it was not found.", + f"Install Node.js 24 LTS with npm: {_NODE_DOWNLOAD}", + "Verify with:", " node --version", - "", - "If apk cannot find Node.js 24, upgrade Alpine or use the official node:24-alpine container.", + " npx.cmd --version" if env.os_type == "windows" else " npx --version", ] + if env.os_type == "windows": + lines.append("If your Node manager installs npx.exe instead (such as Volta), use: npx.exe --version") + if env.serverless == "aws": + lines += [ + "", + "AWS Lambda: include Node.js and npm in a compatible Lambda layer or a custom Python container.", + "An existing ZIP function can keep its configuration: attach a layer with Node, npm and npx in /opt/bin.", + "Include npm's supporting files and target the function's Amazon Linux version and CPU architecture.", + " https://docs.aws.amazon.com/lambda/latest/dg/packaging-layers.html", + "If you choose a container image instead, create a new image-based function " + "and install Node at build time.", + " https://docs.aws.amazon.com/lambda/latest/dg/images-create.html", + ] + if env.linux_distro == "amzn" and env.linux_distro_version == "2": + lines.append( + "For an Amazon Linux 2 based function, first upgrade to an Amazon Linux 2023 based Python runtime " + "before packaging the official Node.js 22/24 binaries." + ) + return "\n".join(lines) + if env.serverless and env.serverless in _SERVERLESS: + name, hosting, documentation = _SERVERLESS[env.serverless] + lines += [ + "", + f"{name}: build and deploy a custom container that includes both Node.js and Python.", + hosting, + "Install both runtimes when building the image; they cannot be installed in the running function.", + f" {documentation}", + ] + return "\n".join(lines) -def _get_arch_instructions() -> list[str]: - """Instructions for Arch Linux.""" - return [ - "ARCH LINUX INSTALLATION:", - "", - " sudo pacman -S nodejs npm", - ] - - -def _get_suse_instructions() -> list[str]: - """Instructions for SUSE/openSUSE.""" - return [ - "SUSE/OPENSUSE INSTALLATION:", - "", - " sudo zypper install nodejs npm", - ] - - -def _get_generic_linux_instructions() -> list[str]: - """Fallback instructions for unknown Linux distributions.""" - return [ - "LINUX INSTALLATION:", - "", - "Use your package manager to install Node.js, or use nvm:", - "", - "Option 1 - nvm (Node Version Manager, works on any Linux):", - " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash", - " source ~/.bashrc", - " nvm install 24", - "", - "Option 2 - Download binary from https://nodejs.org/", - ] - - -def _get_macos_instructions() -> list[str]: - """Instructions for macOS.""" - return [ - "MACOS INSTALLATION:", - "", - "Option 1 - Homebrew (recommended):", - " brew install node", - "", - "Option 2 - nvm (Node Version Manager, for version management):", - " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash", - " source ~/.zshrc # or ~/.bashrc", - " nvm install 24", - "", - "Option 3 - Official installer:", - " Download from https://nodejs.org/", - ] - - -def _get_windows_instructions() -> list[str]: - """Instructions for Windows.""" - return [ - "WINDOWS INSTALLATION:", - "", - "Option 1 - Official installer (recommended):", - " Download from https://nodejs.org/", - "", - "Option 2 - winget (Windows 10/11, built-in):", - " winget install OpenJS.NodeJS.LTS # For LTS version", - " # or: winget install OpenJS.NodeJS # For current version", - "", - "Option 3 - Chocolatey:", - " choco install nodejs-lts # For LTS version", - " # or: choco install nodejs # For current version", - "", - "Option 4 - Scoop:", - " scoop install nodejs-lts # For LTS version", - " # or: scoop install nodejs # For current version", - ] - + if env.ci_platform == "GitHub Actions": + lines += ["", "GITHUB ACTIONS: add Node.js to your workflow:", " - uses: actions/setup-node@v7", " with:"] + lines.append(" node-version: '24'") + elif env.ci_platform: + lines += ["", f"{env.ci_platform}: use your CI provider's Node.js setup step or an image with Node.js 24."] -def _get_venv_instructions() -> list[str]: - """Instructions for virtual environment users.""" - return [ - "ALTERNATIVE: Install Node.js in your Python virtualenv (no sudo):", - " pip install nodeenv", - " nodeenv -p # Installs Node.js in current virtualenv", - " # Then run promptfoo again", - ] + if env.is_docker: + lines += ["", *_container_instructions(env)] + if env.is_wsl: + lines += ["", "WSL: install Node.js inside your Linux distribution, not on the Windows host."] + if env.os_type == "linux" and (not env.is_docker or env.linux_distro == "alpine"): + lines += ["", *_linux_instructions(env)] + elif env.os_type == "darwin": + lines += ["", "MACOS: install Node.js with Homebrew (`brew install node`) or the installer linked above."] + elif env.os_type == "windows": + lines += ["", "WINDOWS: use the installer linked above or run: winget install OpenJS.NodeJS.LTS"] -def _get_npx_instructions() -> list[str]: - """Instructions for direct npx usage.""" - return [ - "DIRECT USAGE (bypasses Python wrapper):", - " npx promptfoo@latest eval", - " # This is often faster and always uses the latest version", - ] + lines += ["", "DIRECT USAGE after installing Node.js: npx promptfoo@latest eval"] + return "\n".join(lines) diff --git a/tests/smoke/test_installation.py b/tests/smoke/test_installation.py new file mode 100644 index 0000000..fe85155 --- /dev/null +++ b/tests/smoke/test_installation.py @@ -0,0 +1,109 @@ +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from promptfoo.environment import Environment +from promptfoo.instructions import get_installation_instructions + + +def windows_verification_commands(launcher: str | None = None) -> list[str]: + instructions = get_installation_instructions(Environment(os_type="windows")).splitlines() + start = instructions.index("Verify with:") + 1 + if launcher is None: + launcher = next((name for name in ("npx.cmd", "npx.exe") if shutil.which(name)), None) + assert launcher, "Neither npx.cmd nor npx.exe is available" + npx = instructions[start + 1].strip() + if launcher == "npx.exe": + npx = next(line.partition("use: ")[2] for line in instructions if "use: npx.exe" in line) + return [instructions[start].strip(), npx] + + +def git_bash() -> Path | None: + roots = [] + if git := shutil.which("git"): + location = Path(git).resolve() + roots += [location.parent, location.parent.parent] + for variable, suffix in (("PROGRAMFILES", "Git"), ("LOCALAPPDATA", "Programs/Git")): + if directory := os.environ.get(variable): + roots.append(Path(directory) / suffix) + return next( + ( + candidate + for root in roots + for candidate in (root / "bin/bash.exe", root / "usr/bin/bash.exe") + if candidate.is_file() + ), + None, + ) + + +@pytest.mark.smoke +@pytest.mark.skipif(sys.platform != "win32", reason="Tests native Windows PowerShell's restricted execution policy") +def test_windows_verification_commands_run_with_powershell_scripts_disabled() -> None: + powershell = shutil.which("powershell") + assert powershell + commands = windows_verification_commands() + script = "; ".join(["$ErrorActionPreference = 'Stop'", *commands, "if ($LASTEXITCODE) { exit $LASTEXITCODE }"]) + + result = subprocess.run( + [powershell, "-NoProfile", "-ExecutionPolicy", "Restricted", "-Command", script], + capture_output=True, + text=True, + timeout=20, + ) + + assert result.returncode == 0, result.stderr + assert len([line for line in result.stdout.splitlines() if re.fullmatch(r"v?\d+\.\d+\.\d+", line)]) == 2 + + +@pytest.mark.smoke +@pytest.mark.skipif(sys.platform != "win32", reason="Tests native Windows Git Bash") +def test_windows_verification_commands_run_in_git_bash() -> None: + if not (bash := git_bash()): + pytest.skip("Git for Windows is not installed") + commands = windows_verification_commands() + + result = subprocess.run( + [str(bash), "--noprofile", "--norc", "-c", "set -e; " + "; ".join(commands)], + capture_output=True, + text=True, + timeout=20, + ) + + assert result.returncode == 0, result.stderr + assert len([line for line in result.stdout.splitlines() if re.fullmatch(r"v?\d+\.\d+\.\d+", line)]) == 2 + + +@pytest.mark.smoke +@pytest.mark.skipif(sys.platform != "win32", reason="Tests native Windows executable lookup") +@pytest.mark.parametrize("shell", ["powershell", "git-bash"]) +def test_windows_npx_executable_alternative_works_in_each_shell(tmp_path: Path, shell: str) -> None: + launcher = shutil.which("node") + assert launcher + node = subprocess.run([launcher, "-p", "process.execPath"], capture_output=True, text=True, check=True, timeout=20) + shutil.copy2(node.stdout.strip(), tmp_path / "npx.exe") + command = windows_verification_commands("npx.exe")[1] + if shell == "powershell": + powershell = shutil.which("powershell") + assert powershell + prefix = [powershell, "-NoProfile", "-ExecutionPolicy", "Restricted", "-Command"] + elif bash := git_bash(): + prefix = [str(bash), "--noprofile", "--norc", "-c"] + else: + pytest.skip("Git for Windows is not installed") + + result = subprocess.run( + [*prefix, command], + env=os.environ | {"PATH": str(tmp_path) + os.pathsep + os.environ["PATH"]}, + capture_output=True, + text=True, + timeout=20, + ) + + assert result.returncode == 0, result.stderr + assert re.fullmatch(r"v\d+\.\d+\.\d+", result.stdout.strip()) diff --git a/tests/test_environment.py b/tests/test_environment.py index e99ed7d..9e65bc1 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,530 +1,168 @@ -""" -Tests for environment detection. - -This module tests detection of operating systems, Linux distributions, -cloud providers, containers, CI/CD platforms, and Python environments. -""" - -import os from pathlib import Path -from unittest import mock +from unittest.mock import MagicMock import pytest -from promptfoo.environment import ( - _detect_ci, - _detect_cloud_provider, - _detect_container, - _detect_linux_distro, - _detect_python_env, - _detect_wsl, - _has_sudo_access, - _read_probe_file, - detect_environment, +from promptfoo import environment + + +@pytest.fixture(autouse=True) +def isolated_probes(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "GITHUB_ACTIONS", + "GITLAB_CI", + "CIRCLECI", + "JENKINS_URL", + "JENKINS_HOME", + "BUILDKITE", + "TF_BUILD", + "TEAMCITY_VERSION", + "TRAVIS", + "DRONE", + "BITBUCKET_BUILD_NUMBER", + "CONTINUOUS_INTEGRATION", + "CI", + "AWS_LAMBDA_FUNCTION_NAME", + "FUNCTIONS_WORKER_RUNTIME", + "WEBSITE_INSTANCE_ID", + "WEBSITE_SITE_NAME", + "CONTAINER_APP_NAME", + "FUNCTION_TARGET", + "FUNCTION_NAME", + "K_SERVICE", + "WSL_DISTRO_NAME", + "WSL_INTEROP", + "KUBERNETES_SERVICE_HOST", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(environment, "_read_probe", lambda path: "") + monkeypatch.setattr(Path, "is_file", lambda path: False) + + +@pytest.mark.parametrize( + "release, expected", + [ + ({"ID": "alpine", "VERSION_ID": "3.24"}, ("alpine", "3.24")), + ({"ID": "derivative", "ID_LIKE": "alpine", "VERSION_ID": "1"}, ("alpine", "1")), + ({"ID": "amzn", "ID_LIKE": "fedora", "VERSION_ID": "2023"}, ("amzn", "2023")), + ({"ID": "amzn", "VERSION_ID": "2"}, ("amzn", "2")), + ({"ID": "ubuntu", "VERSION_ID": "24.04"}, ("ubuntu", "24.04")), + ({"ID": "unknown"}, ("unknown", None)), + ({}, (None, None)), + ], ) +def test_reads_the_standard_linux_release( + monkeypatch: pytest.MonkeyPatch, release: dict[str, str], expected: tuple[str | None, str | None] +) -> None: + monkeypatch.setattr(environment.platform, "freedesktop_os_release", lambda: release) + assert environment._linux_release() == expected -class TestProbeFileReads: - """Test best-effort probe file reads.""" - - def test_read_probe_file_returns_none_when_missing(self, tmp_path: Path) -> None: - """Missing probe files return None.""" - assert _read_probe_file(tmp_path / "missing") is None - - def test_read_probe_file_returns_content_when_readable(self, tmp_path: Path) -> None: - """Readable probe files return their text content.""" - probe_file = tmp_path / "probe" - probe_file.write_text("value") - - assert _read_probe_file(probe_file) == "value" - - def test_read_probe_file_returns_none_when_unreadable(self, tmp_path: Path) -> None: - """Unreadable probe files return None instead of raising.""" - probe_file = tmp_path / "probe" - probe_file.write_text("value") - - with mock.patch("builtins.open", side_effect=OSError("permission denied")): - assert _read_probe_file(probe_file) is None - - -class TestLinuxDistroDetection: - """Test Linux distribution detection.""" - - def test_detect_linux_distro_returns_tuple(self) -> None: - """Linux distro detection returns a tuple.""" - distro, version = _detect_linux_distro() - # Should return tuple even if both None - assert isinstance(distro, (str, type(None))) - assert isinstance(version, (str, type(None))) - - def test_detect_derivative_distro_pop_os(self, tmp_path: Path) -> None: - """Detect Pop!_OS as Ubuntu derivative via ID_LIKE.""" - os_release = tmp_path / "os-release" - os_release.write_text('ID=pop\nVERSION_ID="22.04"\nID_LIKE="ubuntu debian"') - - missing_path = mock.Mock() - missing_path.exists.return_value = False - - with mock.patch("promptfoo.environment.Path") as mock_path_class: - - def path_side_effect(path_str: str) -> object: - if path_str == "/etc/os-release": - return os_release - return missing_path - - mock_path_class.side_effect = path_side_effect - - distro, version = _detect_linux_distro() - assert distro == "ubuntu" # Should resolve to parent via ID_LIKE - assert version == "22.04" - - def test_detect_derivative_distro_raspbian(self, tmp_path: Path) -> None: - """Detect Raspbian as Debian derivative via ID_LIKE.""" - os_release_data = 'ID=raspbian\nVERSION_ID="11"\nID_LIKE=debian' - - with ( - mock.patch("builtins.open", mock.mock_open(read_data=os_release_data)), - mock.patch("promptfoo.environment.Path") as mock_path_class, - ): - mock_path_obj = mock.Mock() - mock_path_obj.exists.return_value = True - mock_path_class.return_value = mock_path_obj - - distro, version = _detect_linux_distro() - assert distro == "debian" # Should resolve to parent via ID_LIKE - assert version == "11" - - def test_detect_derivative_distro_linux_mint(self, tmp_path: Path) -> None: - """Detect Linux Mint as Ubuntu derivative via ID_LIKE.""" - os_release_data = 'ID=linuxmint\nVERSION_ID="21"\nID_LIKE="ubuntu debian"' - - with ( - mock.patch("builtins.open", mock.mock_open(read_data=os_release_data)), - mock.patch("promptfoo.environment.Path") as mock_path_class, - ): - mock_path_obj = mock.Mock() - mock_path_obj.exists.return_value = True - mock_path_class.return_value = mock_path_obj - - distro, version = _detect_linux_distro() - assert distro == "ubuntu" # Should resolve to first known parent in ID_LIKE - assert version == "21" - - def test_usr_lib_os_release_fallback(self, tmp_path: Path) -> None: - """Detect distro from /usr/lib/os-release if /etc/os-release missing.""" - with mock.patch("promptfoo.environment.Path") as mock_path_class: - # Create mock Path objects - etc_path = mock.Mock() - etc_path.exists.return_value = False - etc_path.__str__ = lambda self: "/etc/os-release" - - usr_path = mock.Mock() - usr_path.exists.return_value = True - usr_path.__str__ = lambda self: "/usr/lib/os-release" - - def path_constructor(path_str: str) -> mock.Mock: - if path_str == "/etc/os-release": - return etc_path - elif path_str == "/usr/lib/os-release": - return usr_path - return mock.Mock() - - mock_path_class.side_effect = path_constructor - - with mock.patch("builtins.open", mock.mock_open(read_data='ID=ubuntu\nVERSION_ID="22.04"')): - distro, version = _detect_linux_distro() - assert distro == "ubuntu" - assert version == "22.04" - - def test_detect_linux_distro_skips_unreadable_os_release(self) -> None: - """Unreadable /etc/os-release falls back to /usr/lib/os-release.""" - etc_path = mock.Mock() - etc_path.exists.return_value = True - - usr_path = mock.Mock() - usr_path.exists.return_value = True - - def path_constructor(path_str: str) -> mock.Mock: - if path_str == "/etc/os-release": - return etc_path - elif path_str == "/usr/lib/os-release": - return usr_path - fallback_path = mock.Mock() - fallback_path.exists.return_value = False - return fallback_path - - usr_open = mock.mock_open(read_data='ID=ubuntu\nVERSION_ID="22.04"') - - def open_side_effect(path: mock.Mock) -> mock.MagicMock: - if path is etc_path: - raise OSError("permission denied") - if path is usr_path: - return usr_open() - raise AssertionError(f"unexpected probe path: {path!r}") - - with ( - mock.patch("promptfoo.environment.Path", side_effect=path_constructor), - mock.patch("builtins.open", side_effect=open_side_effect), - ): - distro, version = _detect_linux_distro() - assert distro == "ubuntu" - assert version == "22.04" - - -class TestCloudProviderDetection: - """Test cloud provider detection.""" - - def test_detect_aws_from_hypervisor_uuid(self, tmp_path: Path) -> None: - """Detect AWS from hypervisor UUID.""" - uuid_file = tmp_path / "uuid" - uuid_file.write_text("ec2e1916-9099-7caf-fd21-012345abcdef\n") - - with mock.patch("promptfoo.environment.Path") as mock_path: - mock_path_instance = mock_path.return_value - mock_path_instance.exists.return_value = True - mock_path_instance.__truediv__.return_value = uuid_file - - with mock.patch("builtins.open", mock.mock_open(read_data="ec2e1916-9099-7caf-fd21-012345abcdef\n")): - provider = _detect_cloud_provider() - assert provider == "aws" - - def test_detect_aws_from_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect AWS from environment variables.""" - monkeypatch.setenv("AWS_EXECUTION_ENV", "AWS_Lambda_python3.11") - - with mock.patch("promptfoo.environment.Path") as mock_path: - mock_path.return_value.exists.return_value = False - - provider = _detect_cloud_provider() - assert provider == "aws" - - def test_detect_gcp_from_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect GCP from environment variables.""" - monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "my-project") - - with mock.patch("promptfoo.environment.Path") as mock_path: - mock_path.return_value.exists.return_value = False - - provider = _detect_cloud_provider() - assert provider == "gcp" - - def test_detect_azure_from_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect Azure from environment variables.""" - monkeypatch.setenv("AZURE_SUBSCRIPTION_ID", "12345") - - with mock.patch("promptfoo.environment.Path") as mock_path: - mock_path.return_value.exists.return_value = False - - provider = _detect_cloud_provider() - assert provider == "azure" +def test_missing_linux_release_does_not_prevent_generic_help(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(environment.platform, "freedesktop_os_release", MagicMock(side_effect=OSError)) + assert environment._linux_release() == (None, None) - def test_no_cloud_provider_detected(self) -> None: - """Return None when no cloud provider is detected.""" - with mock.patch("promptfoo.environment.Path") as mock_path: - mock_path.return_value.exists.return_value = False - with mock.patch.dict(os.environ, {}, clear=True): - provider = _detect_cloud_provider() - assert provider is None - def test_detect_cloud_provider_ignores_unreadable_probe_files(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Unreadable cloud metadata files fall back to environment variables.""" - monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "my-project") +def test_alpine_marker_survives_a_missing_standard_release(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(environment.platform, "freedesktop_os_release", MagicMock(side_effect=OSError)) + monkeypatch.setattr(environment, "_read_probe", lambda path: "3.24.1\n" if path == "/etc/alpine-release" else "") + assert environment._linux_release() == ("alpine", "3.24.1") - path_mock = mock.Mock() - path_mock.exists.return_value = True - with ( - mock.patch("promptfoo.environment.Path", return_value=path_mock), - mock.patch("builtins.open", side_effect=OSError("permission denied")), - ): - provider = _detect_cloud_provider() - assert provider == "gcp" +@pytest.mark.parametrize("platform, expected", [("win32", "windows"), ("darwin", "darwin"), ("freebsd14", "freebsd14")]) +def test_non_linux_platforms_do_not_probe_linux_files( + monkeypatch: pytest.MonkeyPatch, platform: str, expected: str +) -> None: + monkeypatch.setattr(environment.sys, "platform", platform) + probe = MagicMock(side_effect=AssertionError("Linux probe ran on another OS")) + monkeypatch.setattr(environment, "_linux_release", probe) + monkeypatch.setattr(environment, "_in_container", probe) + monkeypatch.setattr(environment, "_read_probe", probe) + assert environment.detect_environment() == environment.Environment(os_type=expected) -class TestContainerDetection: - """Test container detection.""" +def test_kubernetes_is_used_to_show_container_help(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(environment.sys, "platform", "linux") + monkeypatch.setattr(environment.platform, "freedesktop_os_release", lambda: {"ID": "alpine"}) + monkeypatch.setattr(environment, "_read_probe", lambda path: "0::/" if path == "/proc/1/cgroup" else "") + monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.0.0.1") - def test_detect_kubernetes_from_env(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect Kubernetes from environment variable.""" - monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1") + result = environment.detect_environment() - with mock.patch("promptfoo.environment.Path") as mock_path: - mock_path.return_value.exists.return_value = False + assert result.is_docker + assert result.linux_distro == "alpine" - is_docker, is_k8s = _detect_container() - assert is_docker is False - assert is_k8s is True - def test_detect_container_returns_tuple(self) -> None: - """Container detection returns a tuple of booleans.""" - is_docker, is_k8s = _detect_container() - assert isinstance(is_docker, bool) - assert isinstance(is_k8s, bool) +@pytest.mark.parametrize("runtime", ["docker", "containerd", "kubepods", "crio"]) +def test_container_runtime_can_also_be_detected_from_cgroups(monkeypatch: pytest.MonkeyPatch, runtime: str) -> None: + monkeypatch.setattr(environment, "_read_probe", lambda path: f"0::/{runtime}/container") + assert environment._in_container() - def test_detect_container_ignores_unreadable_cgroup(self) -> None: - """Unreadable cgroup metadata does not raise.""" - def path_constructor(path_str: str) -> mock.Mock: - path_mock = mock.Mock() - path_mock.exists.return_value = path_str == "/proc/1/cgroup" - return path_mock +def test_docker_marker_works_without_cgroup_names(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Path, "is_file", lambda path: str(path).replace("\\", "/").endswith("/.dockerenv")) + assert environment._in_container() - with ( - mock.patch("promptfoo.environment.Path", side_effect=path_constructor), - mock.patch("builtins.open", side_effect=OSError("permission denied")), - mock.patch.dict(os.environ, {}, clear=True), - ): - is_docker, is_k8s = _detect_container() - assert is_docker is False - assert is_k8s is False - -class TestWSLDetection: - """Test WSL detection.""" - - def test_detect_wsl_from_env_var(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect WSL from WSL_DISTRO_NAME environment variable.""" +@pytest.mark.parametrize("use_environment", [True, False]) +def test_detects_wsl_only_on_linux(monkeypatch: pytest.MonkeyPatch, use_environment: bool) -> None: + monkeypatch.setattr(environment.sys, "platform", "linux") + monkeypatch.setattr(environment.platform, "freedesktop_os_release", lambda: {"ID": "ubuntu"}) + if use_environment: monkeypatch.setenv("WSL_DISTRO_NAME", "Ubuntu") - - assert _detect_wsl() is True - - def test_detect_wsl_from_interop_env(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect WSL from WSL_INTEROP environment variable.""" - monkeypatch.setenv("WSL_INTEROP", "/run/WSL/123_interop") - - assert _detect_wsl() is True - - def test_no_wsl_detected(self) -> None: - """Return False when not in WSL.""" - with mock.patch.dict(os.environ, {}, clear=True): - # This will return False unless we're actually in WSL - # Just verify it returns a boolean - result = _detect_wsl() - assert isinstance(result, bool) - - def test_detect_wsl_ignores_unreadable_proc_version(self) -> None: - """Unreadable /proc/version does not raise.""" - - def path_constructor(path_str: str) -> mock.Mock: - path_mock = mock.Mock() - path_mock.exists.return_value = path_str == "/proc/version" - return path_mock - - with ( - mock.patch("promptfoo.environment.Path", side_effect=path_constructor), - mock.patch("builtins.open", side_effect=OSError("permission denied")), - mock.patch.dict(os.environ, {}, clear=True), - ): - assert _detect_wsl() is False - - -class TestCIDetection: - """Test CI/CD platform detection.""" - - @pytest.mark.parametrize( - "env_var,expected_platform", - [ - ("GITHUB_ACTIONS", "github"), - ("GITLAB_CI", "gitlab"), - ("CIRCLECI", "circleci"), - ("JENKINS_HOME", "jenkins"), - ("TRAVIS", "travis"), - ("BUILDKITE", "buildkite"), - ], - ) - def test_detect_specific_ci_platforms( - self, env_var: str, expected_platform: str, monkeypatch: pytest.MonkeyPatch - ) -> None: - """Detect specific CI/CD platforms from environment variables.""" - with mock.patch.dict(os.environ, {}, clear=True): - monkeypatch.setenv(env_var, "true") - is_ci, platform = _detect_ci() - assert is_ci is True - assert platform == expected_platform - - def test_detect_generic_ci(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect generic CI from CI environment variable.""" - with mock.patch.dict(os.environ, {}, clear=True): - monkeypatch.setenv("CI", "true") - is_ci, platform = _detect_ci() - assert is_ci is True - assert platform is None - - def test_no_ci_detected(self) -> None: - """Return False when no CI is detected.""" - with mock.patch.dict(os.environ, {}, clear=True): - is_ci, platform = _detect_ci() - assert is_ci is False - assert platform is None - - -class TestPythonEnvDetection: - """Test Python environment detection.""" - - def test_detect_venv(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect virtualenv from sys.prefix.""" - import sys - - with mock.patch.object(sys, "prefix", "/home/user/venv"), mock.patch.object(sys, "base_prefix", "/usr"): - is_venv, is_conda = _detect_python_env() - assert is_venv is True - - def test_detect_conda(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect conda from environment variable.""" - monkeypatch.setenv("CONDA_DEFAULT_ENV", "base") - is_venv, is_conda = _detect_python_env() - assert is_conda is True - - def test_no_venv_detected(self) -> None: - """Return False when no venv is detected.""" - import sys - - with ( - mock.patch.object(sys, "prefix", "/usr"), - mock.patch.object(sys, "base_prefix", "/usr"), - mock.patch.dict(os.environ, {}, clear=True), - ): - is_venv, is_conda = _detect_python_env() - assert is_venv is False - assert is_conda is False - - -class TestSudoAccess: - """Test sudo access detection.""" - - def test_has_sudo_when_root(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect sudo access when running as root.""" - if hasattr(os, "geteuid"): - with mock.patch("os.geteuid", return_value=0): - assert _has_sudo_access() is True - - def test_has_sudo_when_sudo_command_exists(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect sudo access when sudo command exists.""" - if hasattr(os, "geteuid"): - with ( - mock.patch("os.geteuid", return_value=1000), - mock.patch("shutil.which", return_value="/usr/bin/sudo"), - ): - assert _has_sudo_access() is True - - def test_no_sudo_when_command_missing(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Return False when sudo command doesn't exist.""" - if hasattr(os, "geteuid"): - with mock.patch("os.geteuid", return_value=1000), mock.patch("shutil.which", return_value=None): - assert _has_sudo_access() is False - - -class TestDetectEnvironment: - """Test complete environment detection.""" - - def test_detect_ubuntu_with_docker(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """Detect Ubuntu in Docker container.""" - os_release = tmp_path / "os-release" - os_release.write_text('ID=ubuntu\nVERSION_ID="22.04"') - - with ( - mock.patch("sys.platform", "linux"), - mock.patch("promptfoo.environment._detect_linux_distro", return_value=("ubuntu", "22.04")), - mock.patch("promptfoo.environment._detect_container", return_value=(True, False)), - mock.patch("promptfoo.environment._detect_wsl", return_value=False), - mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), - mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), - mock.patch("promptfoo.environment._detect_python_env", return_value=(True, False)), - mock.patch("promptfoo.environment._has_sudo_access", return_value=False), - ): - env = detect_environment() - - assert env.os_type == "linux" - assert env.linux_distro == "ubuntu" - assert env.linux_distro_version == "22.04" - assert env.is_docker is True - assert env.is_kubernetes is False - assert env.is_wsl is False - assert env.is_venv is True - - def test_detect_macos_environment(self) -> None: - """Detect macOS environment.""" - with ( - mock.patch("sys.platform", "darwin"), - mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), - mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), - mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), - mock.patch("promptfoo.environment._has_sudo_access", return_value=True), - ): - env = detect_environment() - - assert env.os_type == "darwin" - assert env.linux_distro is None - assert env.has_sudo is True - - def test_detect_windows_environment(self) -> None: - """Detect Windows environment.""" - with ( - mock.patch("sys.platform", "win32"), - mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), - mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), - mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), - mock.patch("promptfoo.environment._has_sudo_access", return_value=False), - ): - env = detect_environment() - - assert env.os_type == "windows" - - def test_detect_aws_lambda(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect AWS Lambda environment.""" - monkeypatch.setenv("AWS_LAMBDA_FUNCTION_NAME", "my-function") - - with ( - mock.patch("sys.platform", "linux"), - mock.patch("promptfoo.environment._detect_linux_distro", return_value=("amzn", "2")), - mock.patch("promptfoo.environment._detect_container", return_value=(False, False)), - mock.patch("promptfoo.environment._detect_wsl", return_value=False), - mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), - mock.patch("promptfoo.environment._detect_cloud_provider", return_value="aws"), - mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), - mock.patch("promptfoo.environment._has_sudo_access", return_value=False), - ): - env = detect_environment() - - assert env.is_lambda is True - assert env.cloud_provider == "aws" - - def test_detect_github_actions(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect GitHub Actions environment.""" - monkeypatch.setenv("GITHUB_ACTIONS", "true") - - with ( - mock.patch("sys.platform", "linux"), - mock.patch("promptfoo.environment._detect_linux_distro", return_value=("ubuntu", "22.04")), - mock.patch("promptfoo.environment._detect_container", return_value=(False, False)), - mock.patch("promptfoo.environment._detect_wsl", return_value=False), - mock.patch("promptfoo.environment._detect_ci", return_value=(True, "github")), - mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), - mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), - mock.patch("promptfoo.environment._has_sudo_access", return_value=True), - ): - env = detect_environment() - - assert env.is_ci is True - assert env.ci_platform == "github" - - def test_detect_wsl_ubuntu(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Detect WSL with Ubuntu.""" - monkeypatch.setenv("WSL_DISTRO_NAME", "Ubuntu") - - with ( - mock.patch("sys.platform", "linux"), - mock.patch("promptfoo.environment._detect_linux_distro", return_value=("ubuntu", "22.04")), - mock.patch("promptfoo.environment._detect_container", return_value=(False, False)), - mock.patch("promptfoo.environment._detect_wsl", return_value=True), - mock.patch("promptfoo.environment._detect_ci", return_value=(False, None)), - mock.patch("promptfoo.environment._detect_cloud_provider", return_value=None), - mock.patch("promptfoo.environment._detect_python_env", return_value=(False, False)), - mock.patch("promptfoo.environment._has_sudo_access", return_value=True), - ): - env = detect_environment() - - assert env.os_type == "linux" - assert env.linux_distro == "ubuntu" - assert env.is_wsl is True - assert env.is_docker is False + else: + monkeypatch.setattr(environment, "_read_probe", lambda path: "6.6-Microsoft-standard-WSL2") + assert environment.detect_environment().is_wsl + + +@pytest.mark.parametrize( + "variable, expected", + [ + ("GITHUB_ACTIONS", "GitHub Actions"), + ("GITLAB_CI", "GitLab CI"), + ("CIRCLECI", "CircleCI"), + ("JENKINS_HOME", "Jenkins"), + ("TEAMCITY_VERSION", "TeamCity"), + ("TRAVIS", "Travis CI"), + ("DRONE", "Drone CI"), + ("BITBUCKET_BUILD_NUMBER", "Bitbucket Pipelines"), + ("CONTINUOUS_INTEGRATION", "CI"), + ("CI", "CI"), + ], +) +def test_detects_ci_guidance(monkeypatch: pytest.MonkeyPatch, variable: str, expected: str) -> None: + monkeypatch.setenv(variable, "1") + assert environment._ci_platform() == expected + + +@pytest.mark.parametrize( + "variables, expected", + [ + (["AWS_LAMBDA_FUNCTION_NAME"], "aws"), + (["FUNCTIONS_WORKER_RUNTIME", "WEBSITE_INSTANCE_ID"], "azure"), + (["FUNCTIONS_WORKER_RUNTIME", "CONTAINER_APP_NAME"], "azure"), + (["FUNCTIONS_WORKER_RUNTIME", "KUBERNETES_SERVICE_HOST"], "azure"), + (["FUNCTION_TARGET", "K_SERVICE"], "google"), + (["FUNCTION_NAME"], "google"), + ], +) +def test_detects_serverless_from_provider_supplied_variables( + monkeypatch: pytest.MonkeyPatch, variables: list[str], expected: str +) -> None: + for variable in variables: + monkeypatch.setenv(variable, "configured") + assert environment._serverless() == expected + + +@pytest.mark.parametrize("variable", ["FUNCTION_TARGET", "FUNCTIONS_WORKER_RUNTIME", "K_SERVICE"]) +def test_local_function_framework_settings_do_not_suppress_host_guidance( + monkeypatch: pytest.MonkeyPatch, variable: str +) -> None: + monkeypatch.setattr(environment.sys, "platform", "darwin") + monkeypatch.setenv(variable, "configured") + + from promptfoo.instructions import get_installation_instructions + + output = get_installation_instructions(environment.detect_environment()) + assert "brew install node" in output + assert "custom container" not in output diff --git a/tests/test_instructions.py b/tests/test_instructions.py index c6ec6a0..22bea63 100644 --- a/tests/test_instructions.py +++ b/tests/test_instructions.py @@ -1,456 +1,220 @@ -""" -Tests for platform-specific installation instructions. - -This module tests that appropriate instructions are generated for -different platforms and environments. -""" - import pytest from promptfoo.environment import Environment from promptfoo.instructions import get_installation_instructions +from promptfoo.node import MIN_NODE_VERSION_TEXT -@pytest.mark.parametrize( - ("env", "recommended"), - [ - (Environment(os_type="linux", is_lambda=True), "node --version"), - (Environment(os_type="linux", is_cloud_function=True, cloud_provider="gcp"), "node --version"), - (Environment(os_type="linux", is_ci=True, ci_platform="github"), "node-version: '24'"), - (Environment(os_type="linux", is_ci=True, ci_platform="gitlab"), "image: node:24"), - (Environment(os_type="linux", is_ci=True, ci_platform="circleci"), "node-version: '24'"), - (Environment(os_type="linux", linux_distro="alpine"), "apk add --no-cache 'nodejs~24' npm"), - (Environment(os_type="linux", linux_distro="alpine", is_docker=True), "FROM node:24-alpine"), - (Environment(os_type="linux", linux_distro="ubuntu", is_docker=True), "setup_24.x"), - (Environment(os_type="linux", is_docker=True), "FROM node:24-bookworm-slim"), - (Environment(os_type="linux", is_wsl=True), "nvm install 24"), - (Environment(os_type="linux", linux_distro="rhel", has_sudo=True), "setup_24.x"), - (Environment(os_type="darwin"), "nvm install 24"), - (Environment(os_type="windows"), "OpenJS.NodeJS.LTS"), - ], -) -def test_installation_help_requires_supported_node(env: Environment, recommended: str) -> None: - """Every platform states the exact minimum and versioned examples install a supported runtime.""" - instructions = get_installation_instructions(env) - - assert "requires Node.js 22.22.0 or newer" in instructions - assert recommended in instructions - assert "nvm install 20" not in instructions - assert "setup_20.x" not in instructions - assert "node-version: '20'" not in instructions - assert "image: node:20" not in instructions - - -class TestLambdaInstructions: - """Test instructions for AWS Lambda.""" - - def test_lambda_instructions(self) -> None: - """Generate Lambda-specific instructions.""" - env = Environment( - os_type="linux", - linux_distro="rhel", - cloud_provider="aws", - is_lambda=True, - ) - - instructions = get_installation_instructions(env) - - assert "AWS Lambda" in instructions - assert "Lambda Layer" in instructions - assert "Node.js runtime" in instructions - - -class TestCloudFunctionInstructions: - """Test instructions for Cloud Functions.""" - - def test_gcp_cloud_function_instructions(self) -> None: - """Generate GCP Cloud Functions instructions.""" - env = Environment( - os_type="linux", - cloud_provider="gcp", - is_cloud_function=True, - ) - - instructions = get_installation_instructions(env) - - assert "Google Cloud Functions" in instructions or "GCP" in instructions - - def test_azure_function_instructions(self) -> None: - """Generate Azure Functions instructions.""" - env = Environment( - os_type="linux", - cloud_provider="azure", - is_cloud_function=True, - ) - - instructions = get_installation_instructions(env) - - assert "Azure Functions" in instructions - - -class TestCIInstructions: - """Test instructions for CI/CD environments.""" - - def test_github_actions_instructions(self) -> None: - """Generate GitHub Actions-specific instructions.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_ci=True, - ci_platform="github", - ) - - instructions = get_installation_instructions(env) - - assert "actions/setup-node@v7" in instructions - assert "node-version: '24'" in instructions - assert "GITHUB" in instructions.upper() - - def test_gitlab_ci_instructions(self) -> None: - """Generate GitLab CI instructions.""" - env = Environment( - os_type="linux", - is_ci=True, - ci_platform="gitlab", - ) - - instructions = get_installation_instructions(env) - - assert "gitlab" in instructions.lower() or "GITLAB" in instructions - assert "image:" in instructions or "before_script" in instructions - - def test_circleci_instructions(self) -> None: - """Generate CircleCI instructions.""" - env = Environment( - os_type="linux", - is_ci=True, - ci_platform="circleci", - ) - - instructions = get_installation_instructions(env) - - assert "circleci" in instructions.lower() or "CIRCLECI" in instructions - - -class TestDockerInstructions: - """Test instructions for Docker containers.""" - - def test_docker_alpine_instructions(self) -> None: - """Generate Docker instructions for Alpine.""" - env = Environment( - os_type="linux", - linux_distro="alpine", - is_docker=True, - ) - - instructions = get_installation_instructions(env) - - assert "FROM node:24-alpine" in instructions - assert "RUN apk add --no-cache python3 py3-pip" in instructions - assert "python3 -m venv /opt/venv" in instructions - assert "apk add --no-cache nodejs npm" not in instructions - assert "Dockerfile" in instructions - - def test_docker_ubuntu_instructions(self) -> None: - """Generate Docker instructions for Ubuntu.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_docker=True, - ) - - instructions = get_installation_instructions(env) - - assert "apt-get" in instructions - assert "Dockerfile" in instructions - - -class TestWSLInstructions: - """Test instructions for WSL (Windows Subsystem for Linux).""" - - def test_wsl_instructions(self) -> None: - """Generate WSL-specific instructions.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_wsl=True, - ) - - instructions = get_installation_instructions(env) - - assert "WSL" in instructions or "Windows Subsystem for Linux" in instructions - assert "nvm" in instructions - assert "/mnt/c" in instructions # Should mention Windows filesystem - assert "performance" in instructions.lower() - - def test_wsl_with_ubuntu_shows_both(self) -> None: - """WSL instructions should show both WSL tips and Ubuntu instructions.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_wsl=True, - has_sudo=True, - ) +@pytest.mark.parametrize("platform", ["linux", "darwin", "windows", "freebsd14"]) +def test_every_platform_gets_the_runtime_requirement_and_a_working_fallback(platform: str) -> None: + output = get_installation_instructions(Environment(os_type=platform)) - instructions = get_installation_instructions(env) + assert f"requires Node.js {MIN_NODE_VERSION_TEXT} or newer" in output + assert "Node.js 24 LTS with npm" in output + assert "https://nodejs.org/en/download" in output + npx = "npx.cmd" if platform == "windows" else "npx" + assert f" node --version\n {npx} --version" in output + if platform == "windows": + assert "use: npx.exe --version" in output + assert "node --version &&" not in output + assert "DIRECT USAGE after installing Node.js: npx promptfoo@latest eval" in output - # Should have WSL-specific guidance - assert "WSL" in instructions - # Should also have Ubuntu/Debian instructions - assert "UBUNTU" in instructions or "DEBIAN" in instructions - - -class TestLinuxInstructions: - """Test instructions for various Linux distributions.""" - - def test_ubuntu_instructions_with_sudo(self) -> None: - """Generate Ubuntu instructions with sudo access.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - has_sudo=True, - ) - instructions = get_installation_instructions(env) - - assert "UBUNTU/DEBIAN" in instructions - assert "sudo apt" in instructions - assert "NodeSource" in instructions - - def test_ubuntu_instructions_without_sudo(self) -> None: - """Generate Ubuntu instructions without sudo access.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - has_sudo=False, - ) - - instructions = get_installation_instructions(env) - - assert "nvm" in instructions - # Should NOT suggest sudo apt commands when user doesn't have sudo - assert "sudo apt" not in instructions - assert "sudo snap" not in instructions - - def test_debian_instructions(self) -> None: - """Generate Debian instructions.""" - env = Environment( +@pytest.mark.parametrize( + "platform, hint", + [("linux", "nvm install 24"), ("darwin", "brew install node"), ("windows", "winget install OpenJS.NodeJS.LTS")], +) +def test_common_platforms_get_one_relevant_installation_hint(platform: str, hint: str) -> None: + assert hint in get_installation_instructions(Environment(os_type=platform)) + + +def test_alpine_dockerfile_contains_only_the_verified_setup_steps() -> None: + output = get_installation_instructions(Environment(os_type="linux", linux_distro="alpine", is_docker=True)) + commands = [line.strip() for line in output.splitlines() if line.strip().startswith(("FROM ", "RUN ", "ENV "))] + + assert commands == [ + "FROM node:24-alpine", + "RUN apk add --no-cache python3 py3-pip", + "RUN python3 -m venv /opt/venv", + 'ENV PATH="/opt/venv/bin:$PATH"', + ] + assert "apk add --no-cache 'nodejs~24' npm" in output + assert "upgrade Alpine or use the official node:24-alpine container" in output + assert "apk add --no-cache nodejs npm" not in output + + +def test_amazon_linux_2023_installs_both_versioned_packages_and_selects_the_active_node() -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="amzn", linux_distro_version="2023") + ) + + assert "sudo dnf install -y nodejs24 nodejs24-npm" in output + assert "sudo /usr/sbin/alternatives --set node /usr/bin/node-24" in output + assert "omit sudo when running as root" in output + assert "https://docs.aws.amazon.com/linux/al2023/ug/nodejs.html" in output + assert "2023.9.20251110" in output + assert "dnf --releasever=latest install" in output + assert "Without sudo, install nvm: https://github.com/nvm-sh/nvm#installing-and-updating" in output + assert "nvm install 24" in output + assert "dnf install -y nodejs\n" not in output + + +def test_unknown_amazon_release_is_not_given_amazon_linux_2023_commands() -> None: + output = get_installation_instructions(Environment(os_type="linux", linux_distro="amzn")) + assert "nvm install 24" in output + assert "nodejs24-npm" not in output + + +@pytest.mark.parametrize("container", [False, True]) +def test_amazon_linux_2_recommends_a_compatible_os(container: bool) -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="amzn", linux_distro_version="2", is_docker=container) + ) + assert "Amazon Linux 2023" in output + assert "require newer glibc" in output + assert "nvm install 24" not in output + if container: + assert "compatible base image" in output + assert "keep your existing base image" not in output + + +def test_ci_container_and_wsl_hints_can_coexist() -> None: + output = get_installation_instructions( + Environment( os_type="linux", linux_distro="debian", - has_sudo=True, - ) - - instructions = get_installation_instructions(env) - - assert "UBUNTU/DEBIAN" in instructions - assert "apt" in instructions - - def test_rhel_instructions_with_sudo(self) -> None: - """Generate RHEL instructions with sudo.""" - env = Environment( - os_type="linux", - linux_distro="rhel", - has_sudo=True, - ) - - instructions = get_installation_instructions(env) - - assert "RHEL" in instructions or "CENTOS" in instructions or "FEDORA" in instructions - assert "dnf" in instructions or "yum" in instructions - - def test_amazon_linux_instructions(self) -> None: - """Generate Amazon Linux instructions.""" - env = Environment( - os_type="linux", - linux_distro="rhel", - linux_distro_version="2023", - has_sudo=True, - ) - - instructions = get_installation_instructions(env) - - assert "AMAZON LINUX" in instructions - assert "dnf" in instructions or "yum" in instructions - - def test_alpine_instructions(self) -> None: - """Generate Alpine Linux instructions.""" - env = Environment( - os_type="linux", - linux_distro="alpine", - linux_distro_version="3.20", - ) - - instructions = get_installation_instructions(env) - - assert "ALPINE" in instructions - assert "apk add --no-cache 'nodejs~24' npm" in instructions - assert "upgrade Alpine or use the official node:24-alpine container" in instructions - assert "node --version" in instructions - assert "apk add --update nodejs npm" not in instructions - assert "apk add --no-cache nodejs npm" not in instructions - - def test_arch_instructions(self) -> None: - """Generate Arch Linux instructions.""" - env = Environment( - os_type="linux", - linux_distro="arch", - ) - - instructions = get_installation_instructions(env) - - assert "ARCH" in instructions - assert "pacman" in instructions - - def test_suse_instructions(self) -> None: - """Generate SUSE instructions.""" - env = Environment( - os_type="linux", - linux_distro="suse", - ) - - instructions = get_installation_instructions(env) - - assert "SUSE" in instructions or "OPENSUSE" in instructions - assert "zypper" in instructions - - def test_generic_linux_instructions(self) -> None: - """Generate generic Linux instructions for unknown distro.""" - env = Environment( - os_type="linux", - linux_distro="unknown", - ) - - instructions = get_installation_instructions(env) - - assert "nvm" in instructions - - -class TestMacOSInstructions: - """Test instructions for macOS.""" - - def test_macos_instructions(self) -> None: - """Generate macOS instructions.""" - env = Environment(os_type="darwin") - - instructions = get_installation_instructions(env) - - assert "MACOS" in instructions - assert "brew install node" in instructions - assert "Official installer" in instructions - assert "nvm" in instructions - assert "nodejs.org" in instructions - - -class TestWindowsInstructions: - """Test instructions for Windows.""" - - def test_windows_instructions(self) -> None: - """Generate Windows instructions.""" - env = Environment(os_type="windows") - - instructions = get_installation_instructions(env) - - assert "WINDOWS" in instructions - assert "winget" in instructions - assert "Chocolatey" in instructions or "choco" in instructions - assert "Scoop" in instructions - - -class TestVenvInstructions: - """Test virtual environment instructions.""" - - def test_venv_instructions_included(self) -> None: - """Include venv instructions when in virtualenv.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_venv=True, - ) - - instructions = get_installation_instructions(env) - - assert "nodeenv" in instructions - assert "virtualenv" in instructions.lower() - - def test_conda_instructions_included(self) -> None: - """Include venv instructions when in conda.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", - is_conda=True, - ) - - instructions = get_installation_instructions(env) - - assert "nodeenv" in instructions - - -class TestNpxInstructions: - """Test npx direct usage instructions.""" - - def test_npx_instructions_always_included(self) -> None: - """NPX instructions should always be included.""" - env = Environment(os_type="linux", linux_distro="ubuntu") - - instructions = get_installation_instructions(env) - - assert "npx promptfoo@latest" in instructions - assert "DIRECT USAGE" in instructions - - -class TestErrorMessageFormat: - """Test error message formatting.""" - - def test_error_message_has_clear_header(self) -> None: - """Error message should have a clear header.""" - env = Environment(os_type="linux", linux_distro="ubuntu") - - instructions = get_installation_instructions(env) - - assert "ERROR: promptfoo requires Node.js" in instructions - assert "=" * 70 in instructions - - def test_multiline_output(self) -> None: - """Instructions should be multi-line.""" - env = Environment(os_type="linux", linux_distro="ubuntu") - - instructions = get_installation_instructions(env) - - lines = instructions.split("\n") - assert len(lines) > 5 # Should have multiple lines - - -class TestComplexEnvironments: - """Test instructions for complex, combined environments.""" - - def test_docker_github_actions_ubuntu(self) -> None: - """Generate instructions for Docker in GitHub Actions on Ubuntu.""" - env = Environment( - os_type="linux", - linux_distro="ubuntu", + linux_distro_version="12", + ci_platform="GitHub Actions", is_docker=True, - is_ci=True, - ci_platform="github", + is_wsl=True, ) + ) + + assert "- uses: actions/setup-node@v7\n with:\n node-version: '24'" in output + assert "FROM python:3.12-slim-bookworm" in output + assert "Keep FROM set to your existing image, platform, and Python environment" in output + assert "FROM node:" not in output + assert "https://deb.nodesource.com/setup_24.x" in output + assert "apt-get install -y --no-install-recommends nodejs" in output + assert "apt recipe supports amd64 and arm64. For other CPU architectures" in output + assert 'ENV PATH="${PATH}:/usr/local/bin"' in output + assert "If an inherited NPM_CONFIG_PREFIX points to an unwritable directory, change it" in output + assert "Keep an existing writable npm prefix only if its bin is separate from Python's scripts directory" in output + assert "add it before /usr/bin and after your Python scripts" in output + assert "npm config set prefix ~/.npm-global" in output + assert "nvm install" not in output + assert "https://github.com/nodesource/distributions" in output + assert "install Node.js inside your Linux distribution" in output - instructions = get_installation_instructions(env) - - # Should include both CI and Docker instructions - assert "GITHUB" in instructions.upper() - assert "DOCKER" in instructions.upper() - def test_aws_ec2_rhel_with_venv(self) -> None: - """Generate instructions for AWS EC2 RHEL with virtualenv.""" - env = Environment( - os_type="linux", - linux_distro="rhel", - cloud_provider="aws", - is_venv=True, - has_sudo=True, - ) +@pytest.mark.parametrize( + ("distribution", "version", "command"), + [ + ("ubuntu", "24.04", "https://deb.nodesource.com/setup_24.x"), + ("amzn", "2023", "dnf --releasever=latest install -y nodejs24 nodejs24-npm"), + ], +) +def test_non_bookworm_containers_keep_their_original_base(distribution: str, version: str, command: str) -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro=distribution, linux_distro_version=version, is_docker=True) + ) + assert "keep your existing FROM" in output + assert command in output + assert "FROM python:" not in output + assert "bookworm" not in output + if distribution == "amzn": + assert "/usr/sbin/alternatives --set node" in output + assert "2023.9.20251110" in output + if distribution == "ubuntu": + commands = [line for line in output.splitlines() if line.strip().startswith("RUN ")] + assert len(commands) == 1 + assert "apt-get update" in commands[0] + assert "rm -rf /var/lib/apt/lists/*" in commands[0] + assert "apt recipe supports amd64 and arm64. For other CPU architectures" in output + + +def test_trixie_container_installs_node_on_its_existing_platform() -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="debian", linux_distro_version="13", is_docker=True) + ) + assert "Debian Trixie" in output + assert "FROM python:3.12-slim-trixie" in output + assert "FROM node:" not in output + assert "apt-get install -y --no-install-recommends nodejs" in output + assert 'ENV PATH="${PATH}:/usr/local/bin"' in output + assert "existing image, platform, and Python environment" in output + assert "bookworm" not in output + + +def test_unknown_container_does_not_claim_to_be_bookworm() -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="debian", linux_distro_version="14", is_docker=True) + ) + assert "keep your existing base image" in output + assert "distribution and CPU architecture" in output + assert "FROM python:" not in output + + +def test_other_ci_uses_the_provider_setup_instructions() -> None: + output = get_installation_instructions(Environment(os_type="linux", ci_platform="GitLab CI")) + assert "GitLab CI: use your CI provider's Node.js setup step or an image with Node.js 24." in output + assert "actions/setup-node" not in output - instructions = get_installation_instructions(env) - # Should include RHEL and venv instructions - assert "RHEL" in instructions or "CENTOS" in instructions or "FEDORA" in instructions - assert "nodeenv" in instructions +@pytest.mark.parametrize( + "provider, label, documentation, hosting", + [ + ( + "google", + "Google Cloud Functions / Cloud Run", + "docs.cloud.google.com/run/docs/building/containers", + "first-generation functions, must move to a Cloud Run service", + ), + ( + "azure", + "Azure Functions", + "learn.microsoft.com/en-us/azure/azure-functions/functions-how-to-custom-container", + "Flex Consumption do not accept custom images. Move to Azure Container Apps", + ), + ], +) +def test_serverless_links_explain_how_to_build_both_runtimes( + provider: str, label: str, documentation: str, hosting: str +) -> None: + output = get_installation_instructions( + Environment( + os_type="linux", linux_distro="amzn", linux_distro_version="2023", is_docker=True, serverless=provider + ) + ) + + assert f"{label}: build and deploy a custom container that includes both Node.js and Python" in output + assert documentation in output + assert hosting in output + assert "https://nodejs.org/en/download" in output + assert "when building the image" in output + assert "sudo" not in output + assert "nvm" not in output + assert "npx promptfoo" not in output + if provider == "azure": + assert "Select that hosting environment in the documentation" in output + assert "?pivots=" not in output + + +def test_lambda_zip_can_use_a_layer_or_choose_to_create_an_image_based_function() -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="amzn", linux_distro_version="2023", is_docker=True, serverless="aws") + ) + assert "existing ZIP function can keep its configuration" in output + assert "Node, npm and npx in /opt/bin" in output + assert "supporting files" in output + assert "https://docs.aws.amazon.com/lambda/latest/dg/packaging-layers.html" in output + assert "If you choose a container image instead, create a new image-based function" in output + assert "https://docs.aws.amazon.com/lambda/latest/dg/images-create.html" in output + assert "sudo" not in output + + +def test_lambda_on_amazon_linux_2_requires_a_compatible_runtime_before_packaging() -> None: + output = get_installation_instructions( + Environment(os_type="linux", linux_distro="amzn", linux_distro_version="2", is_docker=True, serverless="aws") + ) + assert "first upgrade to an Amazon Linux 2023 based Python runtime" in output