air: add convert-to-dabs (run YAML -> Databricks Asset Bundle)

.gitignore

@@ -78,3 +78,4 @@ tools/gofumpt
 .claude/worktrees/
 .worktrees/
 .isaac/
+/databricks

.nextchanges/bundles/5818.md

@@ -0,0 +1 @@
+* `bundle validate` now reports a clear error when a `sql_warehouse` is missing a `name` (including whitespace-only names), and a warning when a grant is missing a `principal` ([#5818](https://github.com/databricks/cli/pull/5818)).

.nextchanges/bundles/bundle-generate-job-download-script.md

@@ -0,0 +1 @@
+* `bundle generate job` can now download workspace files referenced by `spark_python_task`, rewriting them to a relative path like it already does for notebooks. This is opt-in via the `--download-spark-python-files` flag ([#5799](https://github.com/databricks/cli/pull/5799)).

.nextchanges/cli/filer-gcs-recursive-delete.md

@@ -0,0 +1 @@
+Fixed `databricks fs rm -r` failing on UC Volumes backed by GCS when a directory becomes empty during recursive deletion ([#5958](https://github.com/databricks/cli/pull/5958)).

acceptance/experimental/air/cancel/test.toml

@@ -1,6 +1,3 @@
-# This command does not deploy a bundle, so no engine matrix is needed.
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
-
 # The SDK occasionally probes host reachability with a HEAD request; stub it so
 # the test is deterministic.
 [[Server]]

acceptance/experimental/air/convert-to-dabs/docker.yaml

@@ -0,0 +1,8 @@
+experiment_name: docker-test
+command: python train.py
+compute:
+  accelerator_type: GPU_1xA10
+  num_accelerators: 1
+environment:
+  docker_image:
+    url: myregistry/img:tag

acceptance/experimental/air/convert-to-dabs/output.txt

@@ -0,0 +1,82 @@
+
+=== convert an AIR run YAML into a DABs bundle
+>>> [CLI] experimental air convert-to-dabs train.yaml --output-dir generated
+Wrote a Databricks Asset Bundle to generated:
+  databricks.yml
+  training_config.yaml
+  command.sh
+  requirements.yaml
+  code_source.tar.gz
+
+Notes:
+  - code_source was packaged into code_source.tar.gz; re-run convert-to-dabs to re-snapshot after code changes.
+
+To deploy and run this workload as a bundle:
+  1. cd generated
+  2. databricks bundle validate
+  3. databricks bundle deploy
+  4. databricks bundle run torchrun-a10-smoke-test
+
+bundle deploy uploads the code source and launch scripts automatically.
+
+Unlike `air run` (which submits an ephemeral run), bundle deploy creates a
+persistent job that is not garbage-collected. When you are done, remove the
+job and its uploaded files with:
+  databricks bundle destroy
+
+=== emitted databricks.yml
+>>> cat generated/databricks.yml
+bundle:
+  name: torchrun-a10-smoke-test
+targets:
+  dev:
+    mode: development
+    default: true
+resources:
+  jobs:
+    torchrun-a10-smoke-test:
+      name: torchrun-a10-smoke-test
+      tasks:
+        - task_key: torchrun-a10-smoke-test
+          environment_key: default
+          ai_runtime_task:
+            experiment: torchrun-a10-smoke-test
+            deployments:
+              - command_path: ./command.sh
+                compute:
+                  accelerator_type: GPU_1xA10
+                  accelerator_count: 1
+            code_source_path: ./code_source.tar.gz
+      environments:
+        - environment_key: default
+          spec:
+            environment_version: "5"
+            dependencies:
+              - numpy
+
+=== the generated command.sh carries the run command
+>>> cat generated/command.sh
+torchrun --nproc_per_node=1 train.py
+=== the code source is packaged as a tarball
+>>> ls generated
+code_source.tar.gz
+command.sh
+databricks.yml
+requirements.yaml
+training_config.yaml
+
+=== the emitted bundle validates
+>>> [CLI] bundle validate
+Name: torchrun-a10-smoke-test
+Target: dev
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/torchrun-a10-smoke-test/dev
+
+Validation OK!
+
+=== docker_image is not supported yet
+>>> [CLI] experimental air convert-to-dabs docker.yaml --output-dir generated-docker
+Error: environment.docker_image is not yet supported by convert-to-dabs
+
+Exit code: 1

acceptance/experimental/air/convert-to-dabs/script

@@ -0,0 +1,19 @@
+title "convert an AIR run YAML into a DABs bundle"
+trace $CLI experimental air convert-to-dabs train.yaml --output-dir generated
+
+title "emitted databricks.yml"
+trace cat generated/databricks.yml
+
+title "the generated command.sh carries the run command"
+trace cat generated/command.sh
+
+title "the code source is packaged as a tarball"
+trace ls generated && true
+
+title "the emitted bundle validates"
+cd generated
+trace $CLI bundle validate
+cd ..
+
+title "docker_image is not supported yet"
+errcode trace $CLI experimental air convert-to-dabs docker.yaml --output-dir generated-docker

acceptance/experimental/air/convert-to-dabs/src/train.py

@@ -0,0 +1 @@
+print("train")

acceptance/experimental/air/convert-to-dabs/test.toml

@@ -0,0 +1,3 @@
+# The command writes a bundle under generated/; those are generated artifacts,
+# not committed inputs, so exclude them from the repo-diff check.
+Ignore = ["generated", "generated-docker"]

acceptance/experimental/air/convert-to-dabs/train.yaml

@@ -0,0 +1,13 @@
+experiment_name: torchrun-a10-smoke-test
+command: torchrun --nproc_per_node=1 train.py
+compute:
+  accelerator_type: GPU_1xA10
+  num_accelerators: 1
+environment:
+  version: 5
+  dependencies:
+    - numpy
+code_source:
+  type: snapshot
+  snapshot:
+    root_path: ./src

acceptance/experimental/air/get-ai-runtime/test.toml

@@ -1,6 +1,3 @@
-# This command does not deploy a bundle, so no engine matrix is needed.
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
-
 # On Windows, Git Bash rewrites the leading-/ workspace paths passed to
 # `workspace mkdirs`/`import` into C:/... paths; disable that conversion.
 [Env]

acceptance/experimental/air/get/test.toml

@@ -1,6 +1,3 @@
-# This command does not deploy a bundle, so no engine matrix is needed.
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
-
 # The SDK occasionally probes host reachability with a HEAD request; stub it so
 # the test is deterministic.
 [[Server]]

acceptance/experimental/air/help/output.txt

@@ -10,12 +10,13 @@ Usage:
   databricks experimental air [command]
 
 Available Commands:
-  cancel         Cancel one or more runs
-  get            Show status, configuration, and timing details for a specific run
-  list           List your active runs for the current profile (use --all-status for finished runs)
-  logs           Stream or fetch logs for a run
-  register-image Mirror a Docker image into the workspace registry
-  run            Submit a training workload from a YAML config
+  cancel          Cancel one or more runs
+  convert-to-dabs Convert an AIR run YAML into a Databricks Asset Bundle
+  get             Show status, configuration, and timing details for a specific run
+  list            List your active runs for the current profile (use --all-status for finished runs)
+  logs            Stream or fetch logs for a run
+  register-image  Mirror a Docker image into the workspace registry
+  run             Submit a training workload from a YAML config
 
 Flags:
   -h, --help   help for air
@@ -47,3 +48,24 @@ Global Flags:
   -o, --output type      output type: text or json (default text)
   -p, --profile string   ~/.databrickscfg profile
   -t, --target string    bundle target to use (if applicable)
+
+=== logs help
+>>> [CLI] experimental air logs --help
+Stream logs from an active run, or fetch logs from a completed run.
+
+Usage:
+  databricks experimental air logs JOB_RUN_ID [flags]
+
+Flags:
+      --download-to string   Download all logs to this directory instead of printing
+  -h, --help                 help for logs
+      --lines int            For completed runs, print the last N lines (default 10000)
+      --minutes int          Fetch only logs from the last N minutes
+      --node int             Fetch logs from this node
+      --retry int            View logs from a specific retry attempt; -1 means latest (default -1)
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)

acceptance/experimental/air/help/script

@@ -6,3 +6,6 @@ trace $CLI experimental air --help
 
 title "list help"
 trace $CLI experimental air list --help
+
+title "logs help"
+trace $CLI experimental air logs --help

acceptance/experimental/air/list/test.toml

@@ -1,6 +1,3 @@
-# This command does not deploy a bundle, so no engine matrix is needed.
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
-
 # Disable the on-disk run cache so --all-status output is deterministic across runs.
 [Env]
 DATABRICKS_CACHE_ENABLED = "false"

acceptance/experimental/air/logs-mlflow-fallback/output.txt

@@ -0,0 +1,8 @@
+
+=== logs falls back to mlflow (no logs)
+>>> [CLI] experimental air logs 123
+No logs available for run 123. Run terminated in state SUCCESS
+
+=== logs falls back to mlflow (json)
+>>> [CLI] experimental air logs 123 -o json
+{"type":"ERROR","ts":"[TIMESTAMP]","node":0,"line":"No logs available for run 123. Run terminated in state SUCCESS"}

acceptance/experimental/air/logs-mlflow-fallback/script

@@ -0,0 +1,9 @@
+# Bricklens is gated off (FEATURE_DISABLED), so the command falls back to the
+# MLflow log path. With no MLflow run id resolvable, the fallback reports no
+# logs and exits non-zero — proving the try/catch routes to MLflow.
+
+title "logs falls back to mlflow (no logs)"
+errcode trace $CLI experimental air logs 123
+
+title "logs falls back to mlflow (json)"
+errcode trace $CLI experimental air logs 123 -o json

acceptance/experimental/air/logs-mlflow-fallback/test.toml

@@ -0,0 +1,32 @@
+# The SDK occasionally probes host reachability with a HEAD request; stub it so
+# the test is deterministic.
+[[Server]]
+Pattern = "HEAD /"
+Response.Body = ''
+
+# A completed run.
+[[Server]]
+Pattern = "GET /api/2.2/jobs/runs/get"
+Response.Body = '''
+{
+  "run_id": 123,
+  "start_time": 1700000000000,
+  "end_time": 1700000012000,
+  "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"},
+  "tasks": [{"task_key": "train", "run_id": 456, "attempt_number": 0}]
+}
+'''
+
+# Bricklens is gated off by the backend SAFE flag, forcing the MLflow fallback.
+[[Server]]
+Pattern = "GET /api/2.0/ai-training/workflows/by-run-id/123/logs"
+Response.StatusCode = 403
+Response.Body = '''
+{"error_code": "FEATURE_DISABLED", "message": "training log streaming is not enabled"}
+'''
+
+# The MLflow fallback has no run output to resolve an MLflow run id from, so it
+# reports no logs rather than failing — exercising the fallback wiring end to end.
+[[Server]]
+Pattern = "GET /api/2.2/jobs/runs/get-output"
+Response.Body = '{}'

acceptance/experimental/air/logs/output.txt

@@ -0,0 +1,72 @@
+
+=== logs (text, completed run)
+>>> [CLI] experimental air logs 123
+step 1
+step 2
+CUDA out of memory
+
+=== logs (json)
+>>> [CLI] experimental air logs 123 -o json
+{"type":"LOG","ts":"[TIMESTAMP]","node":0,"line":"step 1"}
+{"type":"LOG","ts":"[TIMESTAMP]","node":0,"line":"step 2"}
+{"type":"ALERT","ts":"[TIMESTAMP]","node":0,"line":"CUDA out of memory"}
+{"type":"LOG","ts":"[TIMESTAMP]","node":0,"line":"CUDA out of memory"}
+
+=== logs with --minutes
+>>> [CLI] experimental air logs 123 --minutes 30
+step 1
+step 2
+CUDA out of memory
+
+=== logs with --lines
+>>> [CLI] experimental air logs 123 --lines 1
+CUDA out of memory
+
+=== logs with --lines 0 prints nothing
+>>> [CLI] experimental air logs 123 --lines 0
+No logs available for run 123. Run terminated in state SUCCESS
+
+=== logs from a specific retry
+>>> [CLI] experimental air logs 123 --retry 0
+step 1
+step 2
+CUDA out of memory
+
+=== logs --lines and --minutes are mutually exclusive
+>>> [CLI] experimental air logs 123 --lines 100 --minutes 30
+Error: cannot combine --lines with --minutes: --lines tails by line count, --minutes by time window
+
+Exit code: 1
+
+=== logs --lines and --minutes are mutually exclusive (json)
+>>> [CLI] experimental air logs 123 --lines 100 --minutes 30 -o json
+{
+  "v": 1,
+  "ts": "[TIMESTAMP]",
+  "error": {
+    "code": "INVALID_ARGS",
+    "kind": "PERMANENT",
+    "message": "cannot combine --lines with --minutes: --lines tails by line count, --minutes by time window",
+    "retryable": false
+  }
+}
+
+Exit code: 1
+
+=== invalid run id
+>>> [CLI] experimental air logs notanumber
+Error: invalid JOB_RUN_ID "notanumber": must be a positive integer
+
+Exit code: 1
+
+=== negative node
+>>> [CLI] experimental air logs 123 --node -1
+Error: invalid --node -1: must not be negative
+
+Exit code: 1
+
+=== --download-to not implemented
+>>> [CLI] experimental air logs 123 --download-to /tmp/out
+Error: --download-to is not implemented yet
+
+Exit code: 1

acceptance/experimental/air/logs/script

@@ -0,0 +1,32 @@
+title "logs (text, completed run)"
+trace $CLI experimental air logs 123
+
+title "logs (json)"
+trace $CLI experimental air logs 123 -o json
+
+title "logs with --minutes"
+trace $CLI experimental air logs 123 --minutes 30
+
+title "logs with --lines"
+trace $CLI experimental air logs 123 --lines 1
+
+title "logs with --lines 0 prints nothing"
+trace $CLI experimental air logs 123 --lines 0
+
+title "logs from a specific retry"
+trace $CLI experimental air logs 123 --retry 0
+
+title "logs --lines and --minutes are mutually exclusive"
+errcode trace $CLI experimental air logs 123 --lines 100 --minutes 30
+
+title "logs --lines and --minutes are mutually exclusive (json)"
+errcode trace $CLI experimental air logs 123 --lines 100 --minutes 30 -o json
+
+title "invalid run id"
+errcode trace $CLI experimental air logs notanumber
+
+title "negative node"
+errcode trace $CLI experimental air logs 123 --node -1
+
+title "--download-to not implemented"
+errcode trace $CLI experimental air logs 123 --download-to /tmp/out