diff --git a/.github/workflows/python_push.yml b/.github/workflows/python_push.yml index ba4cd90473..abb0f2ca6b 100644 --- a/.github/workflows/python_push.yml +++ b/.github/workflows/python_push.yml @@ -74,6 +74,13 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # The local backend resolves bundle config by running the in-repo Go helper + # experimental/bundletest/cmd/offline-resolve, so the suite needs the Go toolchain. + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: diff --git a/experimental/bundletest/README.md b/experimental/bundletest/README.md index 212b41a1ae..d15cdb955f 100644 --- a/experimental/bundletest/README.md +++ b/experimental/bundletest/README.md @@ -64,6 +64,12 @@ The same test runs against either backend, chosen by the `BUNDLETEST_BACKEND` en (`main`/`temp`/`system`) can't be judged locally → `LocalUnsupported` → the test **skips with a reason**; - anything that runs and disagrees with an assertion → **red**. +- **Config resolved by the CLI's own engine, online references skipped loudly.** Deploy + runs the real offline resolution (`cmd/offline-resolve`) — includes, target overrides, + presets, and `${var.*}`/`${bundle.*}` all resolve exactly as `bundle validate` renders + them, with no auth or network and no reimplementation. A reference only the workspace can + resolve — `${workspace.*}`, a `lookup` variable, or an unset variable — is + `LocalUnsupported` at the use site, never silently passed through. Assertions you *know* are cloud-only (Databricks type naming, SLA timing, permissions) can also be fenced explicitly with `@pytest.mark.cloud_only`, which skips them on any @@ -71,6 +77,10 @@ non-cloud backend. ## Run it +The local backend resolves bundle config with the in-repo Go helper +`cmd/offline-resolve`, so a Go toolchain (matching the repo's `go.mod`) and a checkout of +the CLI repo are required in addition to Python. + ```sh uv venv --python 3.12 uv pip install -e ".[dev]" diff --git a/experimental/bundletest/cmd/offline-resolve/main.go b/experimental/bundletest/cmd/offline-resolve/main.go new file mode 100644 index 0000000000..1d044c9419 --- /dev/null +++ b/experimental/bundletest/cmd/offline-resolve/main.go @@ -0,0 +1,162 @@ +// Command offline-resolve loads a Declarative Automation Bundle and resolves its +// configuration entirely offline — no workspace client, no auth — then prints the +// resolved config as JSON on stdout. +// +// bundletest's local (DuckDB) backend subprocesses this instead of reimplementing +// variable/include/target resolution in Python. It reuses the CLI's own mutators, so +// includes, target overrides, presets, and ${var.*}/${bundle.*} references resolve +// exactly as `databricks bundle validate` would. +// +// Deliberately offline: it applies only the mutators that need no workspace. In +// particular it never runs PopulateCurrentUser (the first auth call) or +// ResolveLookupVariables (needs the workspace). References only the workspace can +// resolve are preserved for the caller to reject loudly at the use site: ${workspace.*} +// stays literal, and a lookup or unset variable comes through as a sentinel marker (see +// SentinelFormat). +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/loader" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/validate" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/logdiag" +) + +// SentinelFormat encodes a variable that can't be resolved offline. The two %s are the +// reason ("lookup" or "unset") and the variable name. The caller (bundletest's local +// backend) matches this marker in the resolved output and rejects the referencing +// resource loudly, since resolving it needs a workspace. +const SentinelFormat = "__bundletest_unresolved__%s__%s__" + +// seedOfflineUnresolvable gives each variable that has no offline value a sentinel +// default and drops any lookup, so the real resolver can complete instead of aborting. +// +// Two things would otherwise fail offline: SetVariables errors on a required variable +// with no value (set_variables.go), and dynvar errors resolving a reference to a lookup +// variable whose value only a workspace can supply. Assigning a sentinel *default* (not a +// value, so a BUNDLE_VAR_* / variable-file override still wins) and clearing lookup (so +// SetVariables uses the default rather than deferring to ResolveLookupVariables) lets +// ${var.} resolve to the sentinel, which the caller detects. Must run before SetVariables. +type seedOfflineUnresolvable struct{} + +func (seedOfflineUnresolvable) Name() string { return "seedOfflineUnresolvable" } + +func (seedOfflineUnresolvable) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { + for name, v := range b.Config.Variables { + var reason string + switch { + case v.Lookup != nil: + reason = "lookup" + case !v.HasValue() && !v.HasDefault(): + reason = "unset" + default: + continue + } + sentinel := fmt.Sprintf(SentinelFormat, reason, name) + var err error + root, err = dyn.Set(root, "variables."+name+".default", dyn.V(sentinel)) + if err != nil { + return dyn.InvalidValue, err + } + } + // Drop every variables.*.lookup: the only ones present are on vars we just seeded. + return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if len(p) == 3 && p[0] == dyn.Key("variables") && p[2] == dyn.Key("lookup") { + return v, dyn.ErrDrop + } + return v, nil + }) + }) + return diag.FromErr(err) +} + +// offlinePrefixes are the variable-reference prefixes that resolve without a workspace. +// "workspace" is deliberately excluded: workspace fields are unset offline and would +// otherwise resolve to empty strings, hiding references the caller must reject. +var offlinePrefixes = []string{"bundle", "variables"} + +func run(ctx context.Context, path, target string) error { + b, err := bundle.Load(ctx, path) + if err != nil { + return err + } + + selectTarget := mutator.SelectDefaultTarget() + if target != "" { + selectTarget = mutator.SelectTarget(target) + } + + ctx = logdiag.InitContext(ctx) + logdiag.SetCollect(ctx, true) + bundle.ApplySeqContext(ctx, b, + // --- load phase (offline subset of mutator.DefaultMutators) --- + loader.EntryPoint(), + loader.ProcessRootIncludes(), + mutator.EnvironmentsToTargets(), + mutator.ComputeIdToClusterId(), + mutator.InitializeVariables(), + mutator.DefineDefaultTarget(), + selectTarget, + bundle.Mutator(seedOfflineUnresolvable{}), + + // --- initialize phase, everything before the first auth call + // (PopulateCurrentUser) that is safe offline --- + mutator.RejectInternalResources(), + validate.AllResourcesHaveValues(), + validate.ValidateEngine(), + validate.Scripts(), + mutator.RewriteSyncPaths(), + mutator.SyncDefaultPath(), + mutator.SyncInferRoot(), + mutator.InitializeCache(), + + // Variable resolution. SetVariables assigns values from BUNDLE_VAR_*, variable + // files, and defaults. The two ResolveVariableReferences* mutators are the real + // engine, restricted to offline prefixes so workspace/lookup/unset refs stay literal. + mutator.SetVariables(), + mutator.ResolveVariableReferencesInLookup(), + mutator.ResolveVariableReferencesWithoutResources(offlinePrefixes...), + mutator.ResolveVariableReferencesOnlyResources(offlinePrefixes...), + ) + + diags := logdiag.FlushCollected(ctx) + if diags.HasError() { + return fmt.Errorf("offline resolution failed: %w", diags.Error()) + } + + converted, err := convert.FromTyped(&b.Config, b.Config.Value()) + if err != nil { + return err + } + buf, err := json.Marshal(converted.AsAny()) + if err != nil { + return err + } + os.Stdout.Write(buf) + return nil +} + +func main() { + target := flag.String("target", "", "bundle target to select (default target if empty)") + flag.Parse() + if flag.NArg() != 1 { + fmt.Fprintln(os.Stderr, "usage: offline-resolve [--target NAME] ") + os.Exit(1) + } + + if err := run(context.Background(), flag.Arg(0), *target); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/experimental/bundletest/examples/orders_bundle/databricks.yml b/experimental/bundletest/examples/orders_bundle/databricks.yml index 3beb27fa64..f241313351 100644 --- a/experimental/bundletest/examples/orders_bundle/databricks.yml +++ b/experimental/bundletest/examples/orders_bundle/databricks.yml @@ -12,6 +12,9 @@ bundle: variables: warehouse_id: description: SQL warehouse the transform runs on + # A default so ${var.warehouse_id} resolves offline; the local backend never contacts a + # warehouse, so the value is only a placeholder (real runs override via BUNDLE_VAR_* or target). + default: sql-warehouse-placeholder resources: jobs: diff --git a/experimental/bundletest/src/bundletest/backends/duckdb.py b/experimental/bundletest/src/bundletest/backends/duckdb.py index be2eeac163..0da5bafc40 100644 --- a/experimental/bundletest/src/bundletest/backends/duckdb.py +++ b/experimental/bundletest/src/bundletest/backends/duckdb.py @@ -20,16 +20,17 @@ from __future__ import annotations +import json import os import re import shutil +import subprocess import tempfile import time from pathlib import Path from typing import Any import duckdb -import yaml from bundletest.backend import LocalUnsupported, RunResult @@ -68,6 +69,66 @@ def _first_line(err: Exception) -> str: return str(err).splitlines()[0] if str(err) else type(err).__name__ +# Bundle config is resolved offline by the CLI's own engine (cmd/offline-resolve), which +# reuses the real load/target/variable mutators — no reimplementation here. References only a +# workspace can resolve come back two ways: ${workspace.*}/${resources.*} stay literal ($${...} +# is the escape, so a ${ not preceded by $ is a real reference), and a lookup or unset variable +# comes back as a sentinel marker. Both are rejected loudly at the use site. +_VAR_REF = re.compile(r"(? Path: + for p in Path(__file__).resolve().parents: + if (p / "go.mod").exists(): + return p + raise RuntimeError("could not locate the repo root (go.mod) for the offline-resolve helper") + + +def _resolve_config(bundle_path: str) -> dict[str, Any]: + """Load and resolve the bundle at ``bundle_path`` offline via the Go helper, returning the + resolved config as a dict.""" + result = subprocess.run( + ["go", "run", _OFFLINE_RESOLVE_PKG, bundle_path], + cwd=_repo_root(), + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(f"offline bundle resolution failed:\n{result.stderr.strip()}") + return json.loads(result.stdout) + + +def _unresolved_in_str(s: str) -> str | None: + """A message describing why ``s`` can't be resolved locally (a sentinel-marked variable or a + residual ${...} reference), or None if it is fully resolved.""" + m = _SENTINEL.search(s) + if m: + reason, name = m.group(1), m.group(2) + if reason == "lookup": + return f"variable {name!r} is a lookup variable that needs a workspace to resolve" + return f"variable {name!r} has no value offline — set BUNDLE_VAR_{name} or run on the cloud backend" + m = _VAR_REF.search(s) + if m: + return f"{m.group(0)} needs a workspace to resolve" + return None + + +def _unresolved(node: Any) -> str | None: + """First offline-unresolvable reference anywhere in the subtree, as a message, or None.""" + if isinstance(node, dict): + return next((r for r in map(_unresolved, node.values()) if r), None) + if isinstance(node, list): + return next((r for r in map(_unresolved, node) if r), None) + if isinstance(node, str): + return _unresolved_in_str(node) + return None + + class _Task: """One task of a job, as declared in databricks.yml.""" @@ -91,8 +152,7 @@ def __init__(self) -> None: # --- lifecycle --- def deploy(self, bundle_path: str) -> None: self._bundle_path = bundle_path - path = os.path.join(bundle_path, "databricks.yml") - self._config = yaml.safe_load(Path(path).read_text()) if os.path.exists(path) else {} + self._config = _resolve_config(bundle_path) self._jobs = self._extract_jobs(self._config) def teardown(self) -> None: @@ -126,6 +186,12 @@ def run_job(self, name: str, params: dict[str, Any] | None = None) -> RunResult: f"job {name!r} task {task.key!r} is a {task.kind} task; the " f"local backend runs sql_task only — run it on the cloud backend" ) + if task.sql_file: + reason = _unresolved_in_str(task.sql_file) + if reason is not None: + raise LocalUnsupported( + f"job {name!r} task {task.key!r} sql path can't be resolved locally: {reason}" + ) sql = Path(self._bundle_path, task.sql_file).read_text() self._prepare_namespaces(sql) for statement in _split_statements(sql): @@ -154,7 +220,14 @@ def table_schema(self, fqn: str) -> dict[str, str]: # --- control plane --- def get_resource(self, kind: str, name: str) -> dict[str, Any]: - return self._config.get("resources", {}).get(kind, {})[name] + cfg = self._config.get("resources", {}).get(kind, {})[name] + reason = _unresolved(cfg) + if reason is not None: + raise LocalUnsupported( + f"resource {kind}.{name} can't be introspected locally: {reason}; " + f"use the cloud backend" + ) + return cfg def put_file(self, dst: str, src: str) -> None: if not os.path.exists(src): diff --git a/experimental/bundletest/tests/test_variables.py b/experimental/bundletest/tests/test_variables.py new file mode 100644 index 0000000000..1c2f218277 --- /dev/null +++ b/experimental/bundletest/tests/test_variables.py @@ -0,0 +1,125 @@ +"""Config resolution in the local backend. + +Resolution reuses the CLI's own offline engine, so includes, target overrides, and offline +references (${var.name} from BUNDLE_VAR_* or a declared default) resolve exactly as deploy +would render them. References only the workspace can resolve (${workspace.*}, a lookup +variable, an unset variable) are rejected loudly via LocalUnsupported at the use site — +never silently passed through. +""" + +import pytest +from bundletest import LocalUnsupported, bundle_env + + +def _bundle(tmp_path, yml: str) -> str: + (tmp_path / "databricks.yml").write_text(yml) + return str(tmp_path) + + +def test_default_is_resolved(tmp_path): + b = _bundle( + tmp_path, + "variables:\n catalog:\n default: shop\n" + "resources:\n jobs:\n j:\n name: ${var.catalog}_job\n" + " tasks:\n - task_key: t\n", + ) + with bundle_env(b) as env: + assert env.backend.get_resource("jobs", "j")["name"] == "shop_job" + + +def test_env_override_wins(tmp_path, monkeypatch): + monkeypatch.setenv("BUNDLE_VAR_catalog", "prod") + b = _bundle( + tmp_path, + "variables:\n catalog:\n default: shop\n" + "resources:\n jobs:\n j:\n name: ${var.catalog}_job\n" + " tasks:\n - task_key: t\n", + ) + with bundle_env(b) as env: + assert env.backend.get_resource("jobs", "j")["name"] == "prod_job" + + +def test_nested_variable_is_resolved(tmp_path): + b = _bundle( + tmp_path, + "variables:\n env:\n default: dev\n catalog:\n default: shop_${var.env}\n" + "resources:\n jobs:\n j:\n name: ${var.catalog}\n" + " tasks:\n - task_key: t\n", + ) + with bundle_env(b) as env: + assert env.backend.get_resource("jobs", "j")["name"] == "shop_dev" + + +def test_workspace_reference_is_loud(tmp_path): + b = _bundle( + tmp_path, + "resources:\n jobs:\n j:\n name: job\n" + " run_as:\n user_name: ${workspace.current_user.userName}\n" + " tasks:\n - task_key: t\n", + ) + with bundle_env(b) as env, pytest.raises(LocalUnsupported): + env.backend.get_resource("jobs", "j") + + +def test_lookup_variable_is_loud(tmp_path): + b = _bundle( + tmp_path, + "variables:\n wh:\n lookup:\n warehouse: my-warehouse\n" + "resources:\n jobs:\n j:\n name: ${var.wh}\n" + " tasks:\n - task_key: t\n", + ) + with bundle_env(b) as env, pytest.raises(LocalUnsupported): + env.backend.get_resource("jobs", "j") + + +def test_unset_variable_is_loud(tmp_path): + b = _bundle( + tmp_path, + "variables:\n warehouse_id:\n description: set at deploy time\n" + "resources:\n jobs:\n j:\n name: ${var.warehouse_id}\n" + " tasks:\n - task_key: t\n", + ) + with bundle_env(b) as env, pytest.raises(LocalUnsupported): + env.backend.get_resource("jobs", "j") + + +def test_included_file_is_resolved(tmp_path): + # A resource defined in an included file, referencing a variable. The old single-file + # resolver never read includes, so it couldn't see this job at all. + (tmp_path / "databricks.yml").write_text( + "bundle:\n name: b\ninclude:\n - resources/*.yml\n" + "variables:\n catalog:\n default: shop\n" + ) + (tmp_path / "resources").mkdir() + (tmp_path / "resources" / "jobs.yml").write_text( + "resources:\n jobs:\n j:\n name: ${var.catalog}_job\n" + " tasks:\n - task_key: t\n" + ) + with bundle_env(str(tmp_path)) as env: + assert env.backend.get_resource("jobs", "j")["name"] == "shop_job" + + +def test_target_override_is_resolved(tmp_path): + # The default target overrides a variable. The old resolver ignored targets and would + # have rendered the base default ("base_job"). + b = _bundle( + tmp_path, + "bundle:\n name: b\n" + "variables:\n catalog:\n default: base\n" + "resources:\n jobs:\n j:\n name: ${var.catalog}_job\n" + " tasks:\n - task_key: t\n" + "targets:\n dev:\n default: true\n variables:\n catalog: devcat\n", + ) + with bundle_env(b) as env: + assert env.backend.get_resource("jobs", "j")["name"] == "devcat_job" + + +def test_online_variable_in_sql_path_is_loud(tmp_path): + b = _bundle( + tmp_path, + "variables:\n q:\n lookup:\n warehouse: w\n" + "resources:\n jobs:\n j:\n tasks:\n - task_key: t\n" + " sql_task:\n file:\n path: ${var.q}.sql\n", + ) + with bundle_env(b) as env, pytest.raises(LocalUnsupported): + env.run_job("j")