diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01e360cfd..1ebe25faf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -546,24 +546,34 @@ jobs: # altimate_change end # --------------------------------------------------------------------------- - # dbt-tools E2E — slow (~3 min), only on push to main. + # dbt-tools E2E — slow (~3 min): on push to main, and on pull requests that + # touch packages/dbt-tools (the `dbt-tools` change filter above). Without the + # PR path a fix to this job's own environment setup could only be proven + # after merging. # Tests dbt CLI fallbacks against real dbt versions (1.8, 1.10, 1.11) and # real Python environments (venv, uv, system). # --------------------------------------------------------------------------- dbt-tools-e2e: name: "dbt-tools E2E" needs: changes - if: github.event_name == 'push' + if: github.event_name == 'push' || needs.changes.outputs.dbt-tools == 'true' runs-on: ubuntu-latest timeout-minutes: 10 + # Runs pull-request code (setup scripts, tests), so it gets the same + # read-only token scope and no persisted credentials as tracker-leaks. + permissions: + contents: read steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2 with: bun-version: "1.3.14" - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + id: python with: python-version: "3.11" @@ -583,7 +593,10 @@ jobs: uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: packages/dbt-tools/test/.dbt-resolve-envs - key: dbt-resolve-envs-${{ runner.os }}-v1 + # v2: the v1 cache held a uv venv linked to a uv-managed Python that + # fresh runners lack (bin/dbt present, exec ENOENT). setup-resolve.sh + # now validates cached envs and pins uv to the system interpreter. + key: dbt-resolve-envs-${{ runner.os }}-v2 - name: Set up dbt versions run: ./test/e2e/setup-versions.sh 1.8 1.10 1.11 @@ -592,6 +605,10 @@ jobs: - name: Set up Python env scenarios run: ./test/e2e/setup-resolve.sh venv uv system working-directory: packages/dbt-tools + env: + # Build the scenario venvs on the interpreter setup-python installed, + # not the image's /usr/bin/python3; setup-resolve.sh honours this. + DBT_E2E_PYTHON: ${{ steps.python.outputs.python-path }} - name: Run dbt-tools E2E tests run: bun run test:e2e diff --git a/packages/dbt-tools/test/e2e/setup-resolve.sh b/packages/dbt-tools/test/e2e/setup-resolve.sh index 111c6eff2..d00c8d288 100755 --- a/packages/dbt-tools/test/e2e/setup-resolve.sh +++ b/packages/dbt-tools/test/e2e/setup-resolve.sh @@ -33,8 +33,39 @@ ok() { echo " ✓ $1"; } skip() { echo " ⊘ $1 — skipped ($2)"; } fail() { echo " ✗ $1 — $2"; } +# A restored cache can hold an environment whose interpreter no longer exists: +# CI caches only this directory, and a venv's `bin/python` is a symlink (or a +# launcher shebang) into an interpreter outside it. uv in particular links to +# its own managed Python under ~/.local/share/uv, which a fresh runner lacks, +# so `bin/dbt` is present on disk yet fails with ENOENT when executed. The +# `.done` marker alone therefore proves nothing; a cached environment counts +# only if its dbt actually runs. +with_timeout() { + # GNU timeout on Linux; Homebrew coreutils installs it as gtimeout on macOS. + # Without either, the check still runs, just unbounded. + if has timeout; then timeout "$TIMEOUT" "$@" + elif has gtimeout; then gtimeout "$TIMEOUT" "$@" + else "$@"; fi +} +env_ok() { + [ -x "$1" ] && with_timeout "$1" --version >/dev/null 2>&1 +} +# Usage: cached_or_rebuild ; returns 0 when the cached +# environment is usable (caller returns), 1 when it must be rebuilt. +cached_or_rebuild() { + local dir="$ENVS_DIR/$1" + [ -f "$dir/.done" ] || return 1 + if env_ok "$2"; then ok "$1 (cached)"; return 0; fi + echo " ↻ $1 cache is stale (dbt does not run) — rebuilding..." + return 1 +} + # Find a real (non-shim) python3 for venv creation find_real_python() { + # An explicit interpreter wins. CI sets DBT_E2E_PYTHON to the interpreter + # actions/setup-python installed, so the venvs build on the Python the + # workflow chose rather than whatever the runner image happens to ship. + if [ -n "${DBT_E2E_PYTHON:-}" ] && [ -x "$DBT_E2E_PYTHON" ]; then echo "$DBT_E2E_PYTHON"; return; fi # Try pyenv's actual python first if has pyenv; then local p @@ -55,7 +86,7 @@ echo "Using Python: $REAL_PYTHON ($($REAL_PYTHON --version 2>&1))" setup_venv() { local dir="$ENVS_DIR/venv" - if [ -f "$dir/.done" ]; then ok "venv (cached)"; return; fi + cached_or_rebuild venv "$dir/bin/dbt" && return rm -rf "$dir" echo " → Setting up venv..." "$REAL_PYTHON" -m venv "$dir" @@ -68,12 +99,14 @@ setup_venv() { setup_uv() { local dir="$ENVS_DIR/uv" if ! has uv; then skip "uv" "uv not installed"; return; fi - if [ -f "$dir/.done" ]; then ok "uv (cached)"; return; fi + cached_or_rebuild uv "$dir/.venv/bin/dbt" && return rm -rf "$dir" echo " → Setting up uv..." mkdir -p "$dir" - # uv project mode: create .venv in dir - uv venv "$dir/.venv" --quiet + # uv project mode: create .venv in dir. Pin the interpreter to the system + # Python so the venv survives a cache restore on a fresh runner; left to + # itself uv links a managed Python that lives outside the cached directory. + uv venv "$dir/.venv" --python "$REAL_PYTHON" --quiet uv pip install --quiet --python "$dir/.venv/bin/python" "$DBT_SPEC" touch "$dir/.done" ok "uv ($("$dir/.venv/bin/dbt" --version 2>&1 | grep -oE 'installed:\s+\S+' | head -1))" @@ -82,7 +115,7 @@ setup_uv() { setup_pipx() { local dir="$ENVS_DIR/pipx" if ! has pipx; then skip "pipx" "pipx not installed"; return; fi - if [ -f "$dir/.done" ]; then ok "pipx (cached)"; return; fi + cached_or_rebuild pipx "$dir/bin/dbt" && return rm -rf "$dir" echo " → Setting up pipx..." mkdir -p "$dir/bin" "$dir/venvs" @@ -100,7 +133,7 @@ setup_pipx() { setup_conda() { local dir="$ENVS_DIR/conda" if ! has conda; then skip "conda" "conda not installed"; return; fi - if [ -f "$dir/.done" ]; then ok "conda (cached)"; return; fi + cached_or_rebuild conda "$dir/bin/dbt" && return rm -rf "$dir" echo " → Setting up conda..." conda create -y -p "$dir" python=3.11 --quiet 2>/dev/null @@ -117,7 +150,7 @@ setup_conda() { setup_poetry() { local dir="$ENVS_DIR/poetry" if ! has poetry; then skip "poetry" "poetry not installed"; return; fi - if [ -f "$dir/.done" ]; then ok "poetry (cached)"; return; fi + cached_or_rebuild poetry "$dir/.venv/bin/dbt" && return rm -rf "$dir" echo " → Setting up poetry (in-project venv)..." mkdir -p "$dir" @@ -153,7 +186,7 @@ setup_pyenv_venv() { # Simulates a pyenv user who creates a venv with their pyenv-managed python local dir="$ENVS_DIR/pyenv-venv" if ! has pyenv; then skip "pyenv-venv" "pyenv not installed"; return; fi - if [ -f "$dir/.done" ]; then ok "pyenv-venv (cached)"; return; fi + cached_or_rebuild pyenv-venv "$dir/bin/dbt" && return rm -rf "$dir" echo " → Setting up pyenv + venv..." local pyenv_python