From 87ca82891ac8e03b42702b2a853011b7e505b6ab Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 19 Oct 2026 21:42:37 +0000 Subject: [PATCH 01/15] Add databricks pipelines test command Submits a test-only pipeline update with optional test selections and pytest arguments. Relative selections resolve to deployed workspace paths through the bundle sync root, or against the pipeline's root_path when the pipeline is identified with --pipeline-id. Only -k and -m are accepted in --pytest-arg. Co-authored-by: Isaac --- .nextchanges/cli/pipelines-test.md | 1 + .../pipelines/databricks-cli-help/output.txt | 1 + .../pipelines/test/arguments/out.test.toml | 1 + .../pipelines/test/arguments/output.txt | 61 ++++ acceptance/pipelines/test/arguments/script | 14 + .../pipelines/test/pipeline-id/out.test.toml | 1 + .../pipelines/test/pipeline-id/output.txt | 46 +++ acceptance/pipelines/test/pipeline-id/script | 15 + .../pipelines/test/selection/databricks.yml | 10 + .../pipelines/test/selection/out.test.toml | 1 + .../pipelines/test/selection/output.txt | 94 ++++++ .../pipelines/test/selection/pipeline_file.py | 6 + acceptance/pipelines/test/selection/script | 22 ++ .../test/selection/tests/test_orders.py | 6 + acceptance/pipelines/test/test.toml | 3 + bundle/root.go | 7 + bundle/root_test.go | 21 ++ cmd/pipelines/commands.go | 1 + cmd/pipelines/test.go | 317 ++++++++++++++++++ cmd/pipelines/test_test.go | 116 +++++++ 20 files changed, 744 insertions(+) create mode 100644 .nextchanges/cli/pipelines-test.md create mode 100644 acceptance/pipelines/test/arguments/out.test.toml create mode 100644 acceptance/pipelines/test/arguments/output.txt create mode 100644 acceptance/pipelines/test/arguments/script create mode 100644 acceptance/pipelines/test/pipeline-id/out.test.toml create mode 100644 acceptance/pipelines/test/pipeline-id/output.txt create mode 100644 acceptance/pipelines/test/pipeline-id/script create mode 100644 acceptance/pipelines/test/selection/databricks.yml create mode 100644 acceptance/pipelines/test/selection/out.test.toml create mode 100644 acceptance/pipelines/test/selection/output.txt create mode 100644 acceptance/pipelines/test/selection/pipeline_file.py create mode 100644 acceptance/pipelines/test/selection/script create mode 100644 acceptance/pipelines/test/selection/tests/test_orders.py create mode 100644 acceptance/pipelines/test/test.toml create mode 100644 cmd/pipelines/test.go create mode 100644 cmd/pipelines/test_test.go diff --git a/.nextchanges/cli/pipelines-test.md b/.nextchanges/cli/pipelines-test.md new file mode 100644 index 00000000000..f7ac2beee9b --- /dev/null +++ b/.nextchanges/cli/pipelines-test.md @@ -0,0 +1 @@ +* Add the `databricks pipelines test` command, which runs a pipeline's tests in a test-only update. diff --git a/acceptance/pipelines/databricks-cli-help/output.txt b/acceptance/pipelines/databricks-cli-help/output.txt index 8514fa33b2b..3fb01646d00 100644 --- a/acceptance/pipelines/databricks-cli-help/output.txt +++ b/acceptance/pipelines/databricks-cli-help/output.txt @@ -30,6 +30,7 @@ Main Commands run Run a pipeline show Preview a table's columns and sample rows stop Stop a pipeline + test Run the tests of a pipeline Management Commands apply-environment *Public Preview* Apply the latest environment to the pipeline. diff --git a/acceptance/pipelines/test/arguments/out.test.toml b/acceptance/pipelines/test/arguments/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt new file mode 100644 index 00000000000..dc32fe74d13 --- /dev/null +++ b/acceptance/pipelines/test/arguments/output.txt @@ -0,0 +1,61 @@ + +>>> [CLI] pipelines test --help +Run the tests of a pipeline in a test-only update. + +PIPELINE_KEY is the unique name of the pipeline, as defined in its YAML file. +It is required when test selections are given. If there is only one pipeline +in the project and no arguments are given, it is selected automatically. +Outside a project, identify the pipeline with --pipeline-id; all positional +arguments are then test selections. + +TEST_SELECTION is a test directory, file, or pytest test ID, for example +tests/test_orders.py::test_backfill. Relative paths are converted to their +deployed workspace paths, or outside a project resolved against the pipeline's +root_path. Paths that start with / are workspace paths. Without a selection, +all of the pipeline's tests run. + +Each --pytest-arg passes one argument to pytest, in the order given. Only -k +and -m are supported, for example: + + databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' + +Usage: + databricks pipelines test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...] + +Pipeline Test Flags: + --pipeline-id string ID of the pipeline to test, instead of a PIPELINE_KEY. + --pytest-arg stringArray Argument to pass to pytest (repeatable). Only -k and -m are supported. + +Flags: + -h, --help help for test + +Global Flags: + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in project config. Example: --var="foo=bar" + +=== pytest arguments must be passed with --pytest-arg +>>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders +Error: -k is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' + +>>> musterr [CLI] pipelines test --pipeline-id 1234 -msmoke +Error: -m is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' + +>>> musterr [CLI] pipelines test --pipeline-id 1234 -- -k orders +Error: the -- separator is not supported: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' + +=== Only -k and -m are supported +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x +Error: unsupported pytest argument "-x": only -k and -m are supported + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-k +Error: pytest argument -k requires a value: pass it in the next --pytest-arg + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=tests/test_orders.py +Error: unsupported pytest argument "tests/test_orders.py": pass test paths and test IDs as TEST_SELECTION arguments + +=== JSON output is not supported +>>> musterr [CLI] pipelines test --pipeline-id 1234 -o json +Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script new file mode 100644 index 00000000000..dfe93d5c4b6 --- /dev/null +++ b/acceptance/pipelines/test/arguments/script @@ -0,0 +1,14 @@ +trace $CLI pipelines test --help + +title "pytest arguments must be passed with --pytest-arg" +trace musterr $CLI pipelines test --pipeline-id 1234 -k orders +trace musterr $CLI pipelines test --pipeline-id 1234 -msmoke +trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders + +title "Only -k and -m are supported" +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-k +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=tests/test_orders.py + +title "JSON output is not supported" +trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/acceptance/pipelines/test/pipeline-id/out.test.toml b/acceptance/pipelines/test/pipeline-id/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt new file mode 100644 index 00000000000..9138ee2e806 --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -0,0 +1,46 @@ + +=== Relative test selections resolve against the pipeline root_path +>>> [CLI] pipelines test --pipeline-id [UUID] tests/test_orders.py::test_backfill /Workspace/Shared/tests +Pipeline: [UUID] +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill", + "/Workspace/Shared/tests" + ] + } + } +} + +=== Without a root_path, test selections must be workspace paths +>>> musterr [CLI] pipelines test --pipeline-id [UUID] tests +Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests + +>>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Shared/tests/test_shared.py::test_x +Pipeline: [UUID] +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Shared/tests/test_shared.py::test_x" + ] + } + } +} + +=== Outside a project, --pipeline-id is required +>>> musterr [CLI] pipelines test tests +Error: outside a project, identify the pipeline with --pipeline-id diff --git a/acceptance/pipelines/test/pipeline-id/script b/acceptance/pipelines/test/pipeline-id/script new file mode 100644 index 00000000000..2ef512b1210 --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/script @@ -0,0 +1,15 @@ +pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) +no_root_pipeline_id=$($CLI pipelines create --json '{"name": "no-root"}' | jq -r .pipeline_id) +rm "$OUT_REQUESTS" + +title "Relative test selections resolve against the pipeline root_path" +trace $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Shared/tests +trace print_requests.py //updates + +title "Without a root_path, test selections must be workspace paths" +trace musterr $CLI pipelines test --pipeline-id $no_root_pipeline_id tests +trace $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Shared/tests/test_shared.py::test_x +trace print_requests.py //updates + +title "Outside a project, --pipeline-id is required" +trace musterr $CLI pipelines test tests diff --git a/acceptance/pipelines/test/selection/databricks.yml b/acceptance/pipelines/test/selection/databricks.yml new file mode 100644 index 00000000000..0cf849a7919 --- /dev/null +++ b/acceptance/pipelines/test/selection/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-pipeline-test-selection + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-selection + libraries: + - file: + path: pipeline_file.py diff --git a/acceptance/pipelines/test/selection/out.test.toml b/acceptance/pipelines/test/selection/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/selection/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt new file mode 100644 index 00000000000..8b3bca77bc9 --- /dev/null +++ b/acceptance/pipelines/test/selection/output.txt @@ -0,0 +1,94 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files... +Created pipelines.my_pipeline +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== Run all tests of the only pipeline +>>> [CLI] pipelines test +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true + } +} + +=== Select tests by relative path and test ID, with pytest arguments +>>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg=-msmoke,fast +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests", + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_backfill", + "/Workspace/Shared/tests/test_shared.py" + ], + "pytest_args": [ + "-k", + "orders and not slow", + "-msmoke,fast" + ] + } + } +} + +=== Relative paths resolve from the working directory +>>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_valid_order" + ] + } + } +} + +=== Test selections require the pipeline KEY first +>>> musterr [CLI] pipelines test tests/test_orders.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: resource with key "tests/test_orders.py" not found; the first argument must be a pipeline KEY when test selections are given + +=== Test selections must be in the synchronized files +>>> musterr [CLI] pipelines test my_pipeline ../test_outside.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot resolve test selection "../test_outside.py": [TEST_TMP_DIR]_PARENT/test_outside.py is outside the synchronized bundle files in [TEST_TMP_DIR] diff --git a/acceptance/pipelines/test/selection/pipeline_file.py b/acceptance/pipelines/test/selection/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/selection/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script new file mode 100644 index 00000000000..22348d2962f --- /dev/null +++ b/acceptance/pipelines/test/selection/script @@ -0,0 +1,22 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "Run all tests of the only pipeline" +trace $CLI pipelines test +trace print_requests.py //updates + +title "Select tests by relative path and test ID, with pytest arguments" +trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg=-msmoke,fast +trace print_requests.py //updates + +title "Relative paths resolve from the working directory" +cd tests +trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order +cd .. +trace print_requests.py //updates + +title "Test selections require the pipeline KEY first" +trace musterr $CLI pipelines test tests/test_orders.py + +title "Test selections must be in the synchronized files" +trace musterr $CLI pipelines test my_pipeline ../test_outside.py diff --git a/acceptance/pipelines/test/selection/tests/test_orders.py b/acceptance/pipelines/test/selection/tests/test_orders.py new file mode 100644 index 00000000000..6058cdd0bba --- /dev/null +++ b/acceptance/pipelines/test/selection/tests/test_orders.py @@ -0,0 +1,6 @@ +def test_valid_order(): + assert True + + +def test_backfill(): + assert True diff --git a/acceptance/pipelines/test/test.toml b/acceptance/pipelines/test/test.toml new file mode 100644 index 00000000000..9fc0358f8ac --- /dev/null +++ b/acceptance/pipelines/test/test.toml @@ -0,0 +1,3 @@ +RecordRequests = true +Ignore = ['.databricks'] +Env.MSYS_NO_PATHCONV = "1" diff --git a/bundle/root.go b/bundle/root.go index 5f7317a5e99..c41c52edfb7 100644 --- a/bundle/root.go +++ b/bundle/root.go @@ -70,6 +70,13 @@ func mustGetRoot(ctx context.Context) (string, diag.Diagnostics) { return getRootWithTraversal() } +// HasRoot reports whether a bundle root is configured or can be found without loading it. +// An invalid bundle root environment variable counts as a root so that loading reports the error. +func HasRoot(ctx context.Context) bool { + path, diags := tryGetRoot(ctx) + return path != "" || diags.HasError() +} + // tryGetRoot returns a bundle root or an empty string if one cannot be found. func tryGetRoot(ctx context.Context) (string, diag.Diagnostics) { // Note: an invalid value in the environment variable is still an error. diff --git a/bundle/root_test.go b/bundle/root_test.go index 27109f2aeff..20b160b8b40 100644 --- a/bundle/root_test.go +++ b/bundle/root_test.go @@ -110,3 +110,24 @@ func TestRootLookupError(t *testing.T) { require.True(t, diags.HasError()) require.Contains(t, diags[0].Summary, "Unable to locate the bundle root") } + +func TestHasRoot(t *testing.T) { + ctx := t.Context() + + // Have to set then unset to allow the testing package to revert it to its original value. + t.Setenv(env.RootVariable, "") + os.Unsetenv(env.RootVariable) + + dir := t.TempDir() + t.Chdir(dir) + require.False(t, HasRoot(ctx)) + + f, err := os.Create(filepath.Join(dir, config.FileNames[0])) + require.NoError(t, err) + f.Close() + require.True(t, HasRoot(ctx)) + + // An invalid environment variable is reported when the bundle is loaded. + t.Setenv(env.RootVariable, filepath.Join(dir, "doesntexist")) + require.True(t, HasRoot(ctx)) +} diff --git a/cmd/pipelines/commands.go b/cmd/pipelines/commands.go index 6a6b6931313..24882f97864 100644 --- a/cmd/pipelines/commands.go +++ b/cmd/pipelines/commands.go @@ -17,6 +17,7 @@ func Commands() []*cobra.Command { destroyCommand(), runCommand(), dryRunCommand(), + testCommand(), historyCommand(), logsCommand(), openCommand(), diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go new file mode 100644 index 00000000000..7a9da8bef9d --- /dev/null +++ b/cmd/pipelines/test.go @@ -0,0 +1,317 @@ +package pipelines + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdgroup" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/logdiag" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// startTestUpdateRequest is the Start Update body of a test run. +// test_only and test_options are not in the Go SDK yet, so the request is sent directly. +type startTestUpdateRequest struct { + TestOnly bool `json:"test_only"` + TestOptions *testOptions `json:"test_options,omitempty"` +} + +type testOptions struct { + TestSelection []string `json:"test_selection,omitempty"` + PytestArgs []string `json:"pytest_args,omitempty"` +} + +func newStartTestUpdateRequest(selection, pytestArgs []string) startTestUpdateRequest { + req := startTestUpdateRequest{TestOnly: true} + if len(selection) > 0 || len(pytestArgs) > 0 { + req.TestOptions = &testOptions{TestSelection: selection, PytestArgs: pytestArgs} + } + return req +} + +const pytestArgHint = "pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'" + +// validatePytestArgs accepts only -k and -m, each with a value in the next argument or attached to it. +func validatePytestArgs(args []string) error { + for i := 0; i < len(args); i++ { + arg := args[i] + switch { + case arg == "-k" || arg == "-m": + if i == len(args)-1 { + return fmt.Errorf("pytest argument %s requires a value: pass it in the next --pytest-arg", arg) + } + i++ + case strings.HasPrefix(arg, "-k") || strings.HasPrefix(arg, "-m"): + case !strings.HasPrefix(arg, "-"): + return fmt.Errorf("unsupported pytest argument %q: pass test paths and test IDs as TEST_SELECTION arguments", arg) + default: + return fmt.Errorf("unsupported pytest argument %q: only -k and -m are supported", arg) + } + } + return nil +} + +// testFlagErrorFunc adds guidance when pytest's -k or -m is passed as a CLI flag. +func testFlagErrorFunc(cmd *cobra.Command, err error) error { + if notExist, ok := errors.AsType[*pflag.NotExistError](err); ok { + if name := notExist.GetSpecifiedName(); name == "k" || name == "m" { + return fmt.Errorf("-%s is a pytest argument: %s", name, pytestArgHint) + } + } + return cmd.Parent().FlagErrorFunc()(cmd, err) +} + +// resolveTestSelections converts relative selection paths with toWorkspacePath and keeps +// the pytest test ID after "::" unchanged. Paths that start with "/" are workspace paths. +func resolveTestSelections(selections []string, toWorkspacePath func(string) (string, error)) ([]string, error) { + var resolved []string + for _, selection := range selections { + p, testID, _ := strings.Cut(selection, "::") + if p == "" { + return nil, fmt.Errorf("test selection %q has no path", selection) + } + if strings.HasPrefix(p, "/") { + p = path.Clean(p) + } else { + var err error + p, err = toWorkspacePath(p) + if err != nil { + return nil, fmt.Errorf("cannot resolve test selection %q: %w", selection, err) + } + } + if strings.Contains(selection, "::") { + p += "::" + testID + } + resolved = append(resolved, p) + } + return resolved, nil +} + +// bundleWorkspacePath maps a local path, relative to the working directory, to its deployed +// path under the bundle's workspace file path. +func bundleWorkspacePath(cwd, syncRoot, workspaceFilePath, localPath string) (string, error) { + if !filepath.IsAbs(localPath) { + localPath = filepath.Join(cwd, localPath) + } + rel, err := filepath.Rel(syncRoot, localPath) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("%s is outside the synchronized bundle files in %s", localPath, syncRoot) + } + return path.Join(workspaceFilePath, filepath.ToSlash(rel)), nil +} + +// rootPathWorkspacePath resolves a relative path against the pipeline's root_path. +func rootPathWorkspacePath(rootPath, relPath string) (string, error) { + if rootPath == "" { + return "", errors.New("the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") + } + return path.Join(rootPath, filepath.ToSlash(relPath)), nil +} + +// startTestUpdate starts a test-only update and returns its ID. +func startTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string, req startTestUpdateRequest) (string, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return "", fmt.Errorf("failed to create API client: %w", err) + } + + var response pipelines.StartUpdateResponse + err = apiClient.Do( + ctx, + http.MethodPost, + fmt.Sprintf("/api/2.0/pipelines/%s/updates", pipelineID), + auth.WorkspaceIDHeaders(w.Config), + nil, + req, + &response, + ) + if err != nil { + return "", err + } + return response.UpdateId, nil +} + +// testTarget is the pipeline a test run is submitted to. +type testTarget struct { + key string + pipelineID string + selections []string + w *databricks.WorkspaceClient +} + +func (t testTarget) printHeader(out io.Writer, updateID string) { + if t.key != "" { + fmt.Fprintf(out, "Pipeline: %s (%s)\n", t.key, t.pipelineID) + } else { + fmt.Fprintf(out, "Pipeline: %s\n", t.pipelineID) + } + fmt.Fprintf(out, "Update: %s\n", updateID) +} + +// resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. +func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) (testTarget, error) { + if cmd.Flag("target").Value.String() == "" { + cmd.SetContext(root.SkipLoadBundle(cmd.Context())) + } + if err := root.MustWorkspaceClient(cmd, args); err != nil { + return testTarget{}, err + } + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + var rootPath *string + selections, err := resolveTestSelections(args, func(p string) (string, error) { + if rootPath == nil { + pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineID}) + if err != nil { + return "", err + } + rootPath = new(string) + if pipeline.Spec != nil { + *rootPath = pipeline.Spec.RootPath + } + } + return rootPathWorkspacePath(*rootPath, p) + }) + if err != nil { + return testTarget{}, err + } + return testTarget{pipelineID: pipelineID, selections: selections, w: w}, nil +} + +// resolveTestTargetInBundle identifies the pipeline by its bundle KEY, which must precede any selections. +func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, error) { + var key string + b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ + PostInitFunc: func(ctx context.Context, b *bundle.Bundle) error { + var err error + key, err = resolvePipelineArgument(ctx, b, args[:min(len(args), 1)]) + return err + }, + ErrorOnEmptyState: true, + SkipInitContext: true, + }) + if err != nil { + return testTarget{}, err + } + ctx := cmd.Context() + suggestPipelineDeploy(ctx, cmd) + + if _, err := resources.Lookup(b, key, isPipeline); err != nil { + return testTarget{}, fmt.Errorf("%w; the first argument must be a pipeline KEY when test selections are given", err) + } + pipelineID, err := resolvePipelineIdFromKey(ctx, b, key) + if err != nil { + return testTarget{}, err + } + + cwd, err := os.Getwd() + if err != nil { + return testTarget{}, err + } + var selectionArgs []string + if len(args) > 1 { + selectionArgs = args[1:] + } + selections, err := resolveTestSelections(selectionArgs, func(p string) (string, error) { + return bundleWorkspacePath(cwd, b.SyncRootPath, b.Config.Workspace.FilePath, p) + }) + if err != nil { + return testTarget{}, err + } + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx)}, nil +} + +func testCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...]", + Short: "Run the tests of a pipeline", + Long: `Run the tests of a pipeline in a test-only update. + +PIPELINE_KEY is the unique name of the pipeline, as defined in its YAML file. +It is required when test selections are given. If there is only one pipeline +in the project and no arguments are given, it is selected automatically. +Outside a project, identify the pipeline with --pipeline-id; all positional +arguments are then test selections. + +TEST_SELECTION is a test directory, file, or pytest test ID, for example +tests/test_orders.py::test_backfill. Relative paths are converted to their +deployed workspace paths, or outside a project resolved against the pipeline's +root_path. Paths that start with / are workspace paths. Without a selection, +all of the pipeline's tests run. + +Each --pytest-arg passes one argument to pytest, in the order given. Only -k +and -m are supported, for example: + + databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow'`, + } + + var pipelineID string + var pytestArgs []string + + testGroup := cmdgroup.NewFlagGroup("Pipeline Test") + testGroup.FlagSet().StringVar(&pipelineID, "pipeline-id", "", "ID of the pipeline to test, instead of a PIPELINE_KEY.") + testGroup.FlagSet().StringArrayVar(&pytestArgs, "pytest-arg", nil, "Argument to pass to pytest (repeatable). Only -k and -m are supported.") + + wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) + wrappedCmd.AddFlagGroup(testGroup) + + cmd.SetFlagErrorFunc(testFlagErrorFunc) + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + if cmd.ArgsLenAtDash() >= 0 { + return errors.New("the -- separator is not supported: " + pytestArgHint) + } + if root.OutputType(cmd) != flags.OutputText { + return fmt.Errorf("--output %s is not supported by pipelines test", root.OutputType(cmd)) + } + if err := validatePytestArgs(pytestArgs); err != nil { + return err + } + + var target testTarget + var err error + if pipelineID != "" { + target, err = resolveTestTargetByID(cmd, args, pipelineID) + } else { + ctx := logdiag.InitContext(cmd.Context()) + cmd.SetContext(ctx) + if !bundle.HasRoot(ctx) { + return errors.New("outside a project, identify the pipeline with --pipeline-id") + } + target, err = resolveTestTargetInBundle(cmd, args) + } + if err != nil { + return err + } + ctx := cmd.Context() + + req := newStartTestUpdateRequest(target.selections, pytestArgs) + updateID, err := startTestUpdate(ctx, target.w, target.pipelineID, req) + if err != nil { + return err + } + target.printHeader(cmd.OutOrStdout(), updateID) + return nil + } + + return cmd +} diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go new file mode 100644 index 00000000000..1e12fa19094 --- /dev/null +++ b/cmd/pipelines/test_test.go @@ -0,0 +1,116 @@ +package pipelines + +import ( + "encoding/json" + "errors" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidatePytestArgs(t *testing.T) { + tests := []struct { + args []string + wantErr string + }{ + {args: nil}, + {args: []string{"-k", "orders and not slow", "-m", "smoke,fast"}}, + {args: []string{"-korders", "-msmoke"}}, + {args: []string{"-k", "-m"}}, + {args: []string{"-k"}, wantErr: "pytest argument -k requires a value: pass it in the next --pytest-arg"}, + {args: []string{"-m", "smoke", "-m"}, wantErr: "pytest argument -m requires a value: pass it in the next --pytest-arg"}, + {args: []string{"-x"}, wantErr: `unsupported pytest argument "-x": only -k and -m are supported`}, + {args: []string{"--maxfail=1"}, wantErr: `unsupported pytest argument "--maxfail=1": only -k and -m are supported`}, + {args: []string{"tests/test_orders.py"}, wantErr: `unsupported pytest argument "tests/test_orders.py": pass test paths and test IDs as TEST_SELECTION arguments`}, + } + for _, tc := range tests { + err := validatePytestArgs(tc.args) + if tc.wantErr == "" { + assert.NoError(t, err, tc.args) + } else { + assert.EqualError(t, err, tc.wantErr, tc.args) + } + } +} + +func TestResolveTestSelections(t *testing.T) { + toWorkspacePath := func(p string) (string, error) { + if p == "missing" { + return "", errors.New("not found") + } + return "/Workspace/root/" + p, nil + } + + resolved, err := resolveTestSelections([]string{ + "tests", + "tests/test_orders.py::TestOrders::test_backfill[a::b]", + "/Workspace/Users/someone/../someone/tests/test_x.py::test_x", + }, toWorkspacePath) + require.NoError(t, err) + assert.Equal(t, []string{ + "/Workspace/root/tests", + "/Workspace/root/tests/test_orders.py::TestOrders::test_backfill[a::b]", + "/Workspace/Users/someone/tests/test_x.py::test_x", + }, resolved) + + _, err = resolveTestSelections([]string{"::test_x"}, toWorkspacePath) + assert.EqualError(t, err, `test selection "::test_x" has no path`) + + _, err = resolveTestSelections([]string{"missing::test_x"}, toWorkspacePath) + assert.EqualError(t, err, `cannot resolve test selection "missing::test_x": not found`) +} + +func TestBundleWorkspacePath(t *testing.T) { + syncRoot := t.TempDir() + bundleRoot := filepath.Join(syncRoot, "orders") + filePath := "/Workspace/Users/someone/.bundle/orders/dev/files" + + got, err := bundleWorkspacePath(bundleRoot, syncRoot, filePath, filepath.Join("tests", "test_orders.py")) + require.NoError(t, err) + assert.Equal(t, filePath+"/orders/tests/test_orders.py", got) + + got, err = bundleWorkspacePath(filepath.Join(bundleRoot, "tests"), syncRoot, filePath, filepath.Join("..", "conftest.py")) + require.NoError(t, err) + assert.Equal(t, filePath+"/orders/conftest.py", got) + + got, err = bundleWorkspacePath(syncRoot, syncRoot, filePath, ".") + require.NoError(t, err) + assert.Equal(t, filePath, got) + + _, err = bundleWorkspacePath(syncRoot, syncRoot, filePath, filepath.Join("..", "tests")) + assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+syncRoot) +} + +func TestRootPathWorkspacePath(t *testing.T) { + got, err := rootPathWorkspacePath("/Workspace/Users/someone/orders", "tests/test_orders.py") + require.NoError(t, err) + assert.Equal(t, "/Workspace/Users/someone/orders/tests/test_orders.py", got) + + _, err = rootPathWorkspacePath("", "tests") + assert.EqualError(t, err, "the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") +} + +func TestNewStartTestUpdateRequest(t *testing.T) { + tests := []struct { + selection []string + pytestArgs []string + want string + }{ + {want: `{"test_only":true}`}, + { + selection: []string{"/Workspace/tests"}, + want: `{"test_only":true,"test_options":{"test_selection":["/Workspace/tests"]}}`, + }, + { + pytestArgs: []string{"-m", "smoke,fast"}, + want: `{"test_only":true,"test_options":{"pytest_args":["-m","smoke,fast"]}}`, + }, + } + for _, tc := range tests { + body, err := json.Marshal(newStartTestUpdateRequest(tc.selection, tc.pytestArgs)) + require.NoError(t, err) + assert.JSONEq(t, tc.want, string(body)) + } +} From ad9c4beda867f1946e6c8a91aac406c398323060 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 27 Oct 2026 23:16:22 +0000 Subject: [PATCH 02/15] Test every rejected pytest option and the spaced --pytest-arg form Cover each rejected pytest option with a unit test, and add an acceptance case for `--pytest-arg VALUE`, which keeps commas and spaces. Co-authored-by: Isaac --- .../pipelines/test/selection/output.txt | 26 +++++++++++++++++++ acceptance/pipelines/test/selection/script | 4 +++ cmd/pipelines/test_test.go | 13 ++++++++++ 3 files changed, 43 insertions(+) diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index 8b3bca77bc9..46c3137cecc 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -54,6 +54,32 @@ Update: [UUID] } } +=== --pytest-arg also takes its value as the next argument +>>> [CLI] pipelines test my_pipeline --pytest-arg -k --pytest-arg orders, returns --pytest-arg -m --pytest-arg smoke +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "pytest_args": [ + "-k", + "orders, returns", + "-m", + "smoke" + ] + } + } +} + === Relative paths resolve from the working directory >>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order Recommendation: This command runs the last deployed version of the code diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script index 22348d2962f..ca96fb91835 100644 --- a/acceptance/pipelines/test/selection/script +++ b/acceptance/pipelines/test/selection/script @@ -9,6 +9,10 @@ title "Select tests by relative path and test ID, with pytest arguments" trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg=-msmoke,fast trace print_requests.py //updates +title "--pytest-arg also takes its value as the next argument" +trace $CLI pipelines test my_pipeline --pytest-arg -k --pytest-arg 'orders, returns' --pytest-arg -m --pytest-arg smoke +trace print_requests.py //updates + title "Relative paths resolve from the working directory" cd tests trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 1e12fa19094..c887cd10be9 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -35,6 +35,19 @@ func TestValidatePytestArgs(t *testing.T) { } } +// TestValidatePytestArgsRejectsUnsupportedOptions covers every pytest option that the API rejects. +func TestValidatePytestArgsRejectsUnsupportedOptions(t *testing.T) { + for _, arg := range []string{ + "--collect-only", "-x", "--exitfirst", "--maxfail=1", "--lf", "--ff", "--cache-clear", "-n", + "--pdb", "--trace", "-q", "-v", "--color=yes", "--tb=short", "-s", "--capture=no", "-p", + "--junitxml=out.xml", "--html=out.html", "--basetemp=/tmp", "-c", "--rootdir=/", "--confcutdir=/", + "--pyargs", "--timeout=10", "--no-wait", "--junit-xml=out.xml", + } { + err := validatePytestArgs([]string{arg}) + assert.EqualError(t, err, `unsupported pytest argument "`+arg+`": only -k and -m are supported`) + } +} + func TestResolveTestSelections(t *testing.T) { toWorkspacePath := func(p string) (string, error) { if p == "missing" { From a16bd5594e95f8c3d3cbb0ad8969d1b576509be9 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Fri, 30 Oct 2026 10:20:55 +0000 Subject: [PATCH 03/15] Resolve pipelines test selections like the test IDs the service reports The service names a test case relative to pytest's rootdir: the deepest folder that holds the pipeline's root_path and every test source, or, without a root_path, the folder of every test source. A test ID copied from the results now runs that test again in any project layout. A relative selection that exists locally still maps to its deployed path. Any other relative selection resolves against that folder, which the command reads from the deployed pipeline. With --pipeline-id, all relative selections resolve against it. Co-authored-by: Isaac --- .../pipelines/test/arguments/output.txt | 9 +- .../pipelines/test/pipeline-id/output.txt | 2 +- .../pipelines/test/test-ids/databricks.yml | 11 ++ .../pipelines/test/test-ids/out.test.toml | 1 + acceptance/pipelines/test/test-ids/output.txt | 46 ++++++ .../test/test-ids/pipeline/tests/test_a.py | 2 + .../pipeline/transformations/orders.py | 1 + acceptance/pipelines/test/test-ids/script | 16 ++ cmd/pipelines/test.go | 150 +++++++++++++++--- cmd/pipelines/test_test.go | 85 +++++++++- 10 files changed, 292 insertions(+), 31 deletions(-) create mode 100644 acceptance/pipelines/test/test-ids/databricks.yml create mode 100644 acceptance/pipelines/test/test-ids/out.test.toml create mode 100644 acceptance/pipelines/test/test-ids/output.txt create mode 100644 acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py create mode 100644 acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py create mode 100644 acceptance/pipelines/test/test-ids/script diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index dc32fe74d13..4c3eed3f7fd 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -9,10 +9,11 @@ Outside a project, identify the pipeline with --pipeline-id; all positional arguments are then test selections. TEST_SELECTION is a test directory, file, or pytest test ID, for example -tests/test_orders.py::test_backfill. Relative paths are converted to their -deployed workspace paths, or outside a project resolved against the pipeline's -root_path. Paths that start with / are workspace paths. Without a selection, -all of the pipeline's tests run. +tests/test_orders.py::test_backfill. A relative path that exists locally is +converted to its deployed workspace path. Any other relative path is a test ID +as the results show it: relative to the deepest folder that holds the +pipeline's root_path and its test sources. Paths that start with / are +workspace paths. Without a selection, all of the pipeline's tests run. Each --pytest-arg passes one argument to pytest, in the order given. Only -k and -m are supported, for example: diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt index 9138ee2e806..2952c195601 100644 --- a/acceptance/pipelines/test/pipeline-id/output.txt +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -21,7 +21,7 @@ Update: [UUID] === Without a root_path, test selections must be workspace paths >>> musterr [CLI] pipelines test --pipeline-id [UUID] tests -Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests +Error: cannot resolve test selection "tests": the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests >>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Shared/tests/test_shared.py::test_x Pipeline: [UUID] diff --git a/acceptance/pipelines/test/test-ids/databricks.yml b/acceptance/pipelines/test/test-ids/databricks.yml new file mode 100644 index 00000000000..423404790c4 --- /dev/null +++ b/acceptance/pipelines/test/test-ids/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-pipeline-test-ids + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-ids + root_path: ./pipeline + libraries: + - file: + path: ./pipeline/transformations/orders.py diff --git a/acceptance/pipelines/test/test-ids/out.test.toml b/acceptance/pipelines/test/test-ids/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/test-ids/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/test-ids/output.txt b/acceptance/pipelines/test/test-ids/output.txt new file mode 100644 index 00000000000..f257359806e --- /dev/null +++ b/acceptance/pipelines/test/test-ids/output.txt @@ -0,0 +1,46 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files... +Created pipelines.my_pipeline +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== A test ID resolves against the folder that test IDs are relative to + +>>> [CLI] pipelines test my_pipeline tests/test_a.py::test_one +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{"test_selection":["/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files/pipeline/tests/test_a.py::test_one"]} + +=== The same ID from the pipeline folder, where it is a local path + +>>> [CLI] pipelines test my_pipeline tests/test_a.py::test_one +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{"test_selection":["/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files/pipeline/tests/test_a.py::test_one"]} + +=== A local path from the bundle root + +>>> [CLI] pipelines test my_pipeline pipeline/tests/test_a.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{"test_selection":["/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files/pipeline/tests/test_a.py"]} diff --git a/acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py b/acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py new file mode 100644 index 00000000000..65de24b9c57 --- /dev/null +++ b/acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py @@ -0,0 +1,2 @@ +def test_one(): + pass diff --git a/acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py b/acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py new file mode 100644 index 00000000000..7d4290a117a --- /dev/null +++ b/acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py @@ -0,0 +1 @@ +x = 1 diff --git a/acceptance/pipelines/test/test-ids/script b/acceptance/pipelines/test/test-ids/script new file mode 100644 index 00000000000..3972d69e926 --- /dev/null +++ b/acceptance/pipelines/test/test-ids/script @@ -0,0 +1,16 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "A test ID resolves against the folder that test IDs are relative to\n" +trace $CLI pipelines test my_pipeline tests/test_a.py::test_one +trace print_requests.py //updates | jq -c .body.test_options + +title "The same ID from the pipeline folder, where it is a local path\n" +cd pipeline +trace $CLI pipelines test my_pipeline tests/test_a.py::test_one +cd .. +trace print_requests.py //updates | jq -c .body.test_options + +title "A local path from the bundle root\n" +trace $CLI pipelines test my_pipeline pipeline/tests/test_a.py +trace print_requests.py //updates | jq -c .body.test_options diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 7a9da8bef9d..b78c55ad057 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -10,6 +10,7 @@ import ( "path" "path/filepath" "strings" + "sync" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/resources" @@ -118,12 +119,113 @@ func bundleWorkspacePath(cwd, syncRoot, workspaceFilePath, localPath string) (st return path.Join(workspaceFilePath, filepath.ToSlash(rel)), nil } -// rootPathWorkspacePath resolves a relative path against the pipeline's root_path. -func rootPathWorkspacePath(rootPath, relPath string) (string, error) { - if rootPath == "" { - return "", errors.New("the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") +// bundleTestSelectionPath maps a relative selection to a workspace path. A path that exists locally, +// relative to the working directory, maps to its deployed path. Otherwise the selection is a test +// ID that the CLI printed, relative to the folder that testIDBase returns, unless it leaves that folder. +func bundleTestSelectionPath(cwd, syncRoot, workspaceFilePath string, testIDBase func() (string, error), localPath string) (string, error) { + rel := path.Clean(filepath.ToSlash(localPath)) + leavesBase := rel == ".." || strings.HasPrefix(rel, "../") + if _, err := os.Stat(filepath.Join(cwd, localPath)); err != nil && !leavesBase { + base, err := testIDBase() + if err != nil { + return "", err + } + if base != "" { + return testIDBasePath(base, localPath) + } + } + return bundleWorkspacePath(cwd, syncRoot, workspaceFilePath, localPath) +} + +// testIDBase returns the folder that the service names test cases relative to, or "" if the +// pipeline has neither a root_path nor test sources. pytest runs with this folder as its rootdir: +// the deepest folder that holds the root path and every test source or, without a root path, the +// folder of every test source. +func testIDBase(rootPath string, sources []string) string { + var folders [][]string + if rootPath != "" { + folders = append(folders, pathSegments(normalizeWorkspacePath(rootPath))) + } + for _, source := range sources { + segments := pathSegments(normalizeWorkspacePath(strings.TrimSuffix(source, "/**"))) + if rootPath == "" && len(segments) > 0 { + segments = segments[:len(segments)-1] + } + folders = append(folders, segments) + } + if len(folders) == 0 { + return "" + } + common := folders[0] + for _, folder := range folders[1:] { + n := 0 + for n < len(common) && n < len(folder) && common[n] == folder[n] { + n++ + } + common = common[:n] + } + return "/" + strings.Join(common, "/") +} + +// normalizeWorkspacePath adds the /Workspace prefix to an absolute path without it, as the service does. +func normalizeWorkspacePath(p string) string { + if strings.HasPrefix(p, "/") && !strings.HasPrefix(p, "/Workspace") { + return "/Workspace" + p + } + return p +} + +func pathSegments(p string) []string { + var segments []string + for segment := range strings.SplitSeq(p, "/") { + if segment != "" { + segments = append(segments, segment) + } + } + return segments +} + +// testIDBasePath resolves a relative path against the folder that test IDs are relative to. +func testIDBasePath(base, relPath string) (string, error) { + if base == "" { + return "", errors.New("the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") } - return path.Join(rootPath, filepath.ToSlash(relPath)), nil + return path.Join(base, filepath.ToSlash(relPath)), nil +} + +// pipelineTestPaths holds the fields of a Get Pipeline response that decide the folder of test IDs. +// The Go SDK has no spec.testing yet, so the response is decoded from the raw API call. +type pipelineTestPaths struct { + Spec struct { + RootPath string `json:"root_path"` + Testing struct { + Sources []struct { + Glob struct { + Include string `json:"include"` + } `json:"glob"` + } `json:"sources"` + } `json:"testing"` + } `json:"spec"` +} + +// getTestIDBase reads the deployed pipeline and returns the folder that its test IDs are relative to. +func getTestIDBase(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) (string, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return "", fmt.Errorf("failed to create API client: %w", err) + } + var response pipelineTestPaths + err = apiClient.Do(ctx, http.MethodGet, "/api/2.0/pipelines/"+pipelineID, auth.WorkspaceIDHeaders(w.Config), nil, nil, &response) + if err != nil { + return "", err + } + var sources []string + for _, source := range response.Spec.Testing.Sources { + if source.Glob.Include != "" { + sources = append(sources, source.Glob.Include) + } + } + return testIDBase(response.Spec.RootPath, sources), nil } // startTestUpdate starts a test-only update and returns its ID. @@ -166,6 +268,13 @@ func (t testTarget) printHeader(out io.Writer, updateID string) { fmt.Fprintf(out, "Update: %s\n", updateID) } +// lazyTestIDBase returns a function that reads the folder of test IDs once, when a selection needs it. +func lazyTestIDBase(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) func() (string, error) { + return sync.OnceValues(func() (string, error) { + return getTestIDBase(ctx, w, pipelineID) + }) +} + // resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) (testTarget, error) { if cmd.Flag("target").Value.String() == "" { @@ -177,19 +286,13 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - var rootPath *string + base := lazyTestIDBase(ctx, w, pipelineID) selections, err := resolveTestSelections(args, func(p string) (string, error) { - if rootPath == nil { - pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineID}) - if err != nil { - return "", err - } - rootPath = new(string) - if pipeline.Spec != nil { - *rootPath = pipeline.Spec.RootPath - } + base, err := base() + if err != nil { + return "", err } - return rootPathWorkspacePath(*rootPath, p) + return testIDBasePath(base, p) }) if err != nil { return testTarget{}, err @@ -231,13 +334,15 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e if len(args) > 1 { selectionArgs = args[1:] } + w := b.WorkspaceClient(ctx) + base := lazyTestIDBase(ctx, w, pipelineID) selections, err := resolveTestSelections(selectionArgs, func(p string) (string, error) { - return bundleWorkspacePath(cwd, b.SyncRootPath, b.Config.Workspace.FilePath, p) + return bundleTestSelectionPath(cwd, b.SyncRootPath, b.Config.Workspace.FilePath, base, p) }) if err != nil { return testTarget{}, err } - return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx)}, nil + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: w}, nil } func testCommand() *cobra.Command { @@ -253,10 +358,11 @@ Outside a project, identify the pipeline with --pipeline-id; all positional arguments are then test selections. TEST_SELECTION is a test directory, file, or pytest test ID, for example -tests/test_orders.py::test_backfill. Relative paths are converted to their -deployed workspace paths, or outside a project resolved against the pipeline's -root_path. Paths that start with / are workspace paths. Without a selection, -all of the pipeline's tests run. +tests/test_orders.py::test_backfill. A relative path that exists locally is +converted to its deployed workspace path. Any other relative path is a test ID +as the results show it: relative to the deepest folder that holds the +pipeline's root_path and its test sources. Paths that start with / are +workspace paths. Without a selection, all of the pipeline's tests run. Each --pytest-arg passes one argument to pytest, in the order given. Only -k and -m are supported, for example: diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index c887cd10be9..3c73fade96c 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -3,6 +3,7 @@ package pipelines import ( "encoding/json" "errors" + "os" "path/filepath" "testing" @@ -96,13 +97,89 @@ func TestBundleWorkspacePath(t *testing.T) { assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+syncRoot) } -func TestRootPathWorkspacePath(t *testing.T) { - got, err := rootPathWorkspacePath("/Workspace/Users/someone/orders", "tests/test_orders.py") +func TestTestIDBase(t *testing.T) { + tests := []struct { + name string + rootPath string + sources []string + want string + }{ + { + name: "deepest folder holding the root path and every test source", + rootPath: "/Workspace/Users/u/files/src/my_pipeline", + sources: []string{"/Workspace/Users/u/files/tests/**", "/Workspace/Users/u/files/checks/test_x.py"}, + want: "/Workspace/Users/u/files", + }, + { + name: "root path that holds the test sources", + rootPath: "/Users/u/sl", + sources: []string{"/Users/u/sl/tests/**"}, + want: "/Workspace/Users/u/sl", + }, + { + name: "without a root path, the folder of a file test source", + sources: []string{"/Workspace/Users/u/tests/test_x.py"}, + want: "/Workspace/Users/u/tests", + }, + { + name: "without a root path, the deepest folder holding the folders of every test source", + sources: []string{"/Workspace/Users/u/a/tests/**", "/Workspace/Users/u/b/tests/**"}, + want: "/Workspace/Users/u", + }, + { + name: "root path without test sources", + rootPath: "/Workspace/Users/u/orders", + want: "/Workspace/Users/u/orders", + }, + { + name: "neither", + want: "", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, testIDBase(tc.rootPath, tc.sources)) + }) + } +} + +func TestTestIDBasePath(t *testing.T) { + got, err := testIDBasePath("/Workspace/Users/someone/orders", "tests/test_orders.py") require.NoError(t, err) assert.Equal(t, "/Workspace/Users/someone/orders/tests/test_orders.py", got) - _, err = rootPathWorkspacePath("", "tests") - assert.EqualError(t, err, "the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") + _, err = testIDBasePath("", "tests") + assert.EqualError(t, err, "the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") +} + +func TestBundleTestSelectionPath(t *testing.T) { + syncRoot := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(syncRoot, "pipeline", "tests"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(syncRoot, "pipeline", "tests", "test_a.py"), nil, 0o644)) + filePath := "/Workspace/Users/someone/.bundle/orders/dev/files" + base := func() (string, error) { return filePath + "/pipeline", nil } + noBase := func() (string, error) { return "", nil } + + // A path that exists where the user stands is a local path. + got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, filePath, base, "tests/test_a.py") + require.NoError(t, err) + assert.Equal(t, filePath+"/pipeline/tests/test_a.py", got) + + // Otherwise it is a test ID, relative to the folder that test IDs are relative to. + got, err = bundleTestSelectionPath(syncRoot, syncRoot, filePath, base, "tests/test_a.py") + require.NoError(t, err) + assert.Equal(t, filePath+"/pipeline/tests/test_a.py", got) + + // Without that folder, it is a local path. + got, err = bundleTestSelectionPath(syncRoot, syncRoot, filePath, noBase, "tests/test_a.py") + require.NoError(t, err) + assert.Equal(t, filePath+"/tests/test_a.py", got) + + // A path that leaves that folder is a local path too. + for _, testIDBase := range []func() (string, error){base, noBase} { + _, err = bundleTestSelectionPath(syncRoot, syncRoot, filePath, testIDBase, filepath.Join("..", "tests")) + assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+syncRoot) + } } func TestNewStartTestUpdateRequest(t *testing.T) { From 31b5ec31db450cdb47c6a8ff0865670ea0b19344 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Fri, 30 Oct 2026 15:06:32 +0000 Subject: [PATCH 04/15] Map local pipelines test paths like TranslatePaths and mark the command preview A local test path mapped to workspace.file_path in every sync mode. A source-linked deployment runs the source files, so a local path now maps under the sync root, as TranslatePaths maps resource paths. An immutable deployment runs a snapshot whose path the command does not know, so a local path is an error there that points to test IDs and workspace paths, which still work. The folder of test IDs is read once, and only when a selection is a relative path. Paths in errors use forward slashes. The Short text starts with *Preview*, like other preview commands. Tests: the option list covers each rejected form once, and a folder test source without root_path has its parent as the folder of test IDs, as the service computes it. Co-authored-by: Isaac --- .../pipelines/databricks-cli-help/output.txt | 2 +- cmd/pipelines/test.go | 85 ++++++++++++------- cmd/pipelines/test_test.go | 75 +++++++++++----- 3 files changed, 111 insertions(+), 51 deletions(-) diff --git a/acceptance/pipelines/databricks-cli-help/output.txt b/acceptance/pipelines/databricks-cli-help/output.txt index 3fb01646d00..adc729f4f5c 100644 --- a/acceptance/pipelines/databricks-cli-help/output.txt +++ b/acceptance/pipelines/databricks-cli-help/output.txt @@ -30,7 +30,7 @@ Main Commands run Run a pipeline show Preview a table's columns and sample rows stop Stop a pipeline - test Run the tests of a pipeline + test *Preview* Run the tests of a pipeline Management Commands apply-environment *Public Preview* Apply the latest environment to the pipeline. diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index b78c55ad057..373ebab8be4 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -10,9 +10,9 @@ import ( "path" "path/filepath" "strings" - "sync" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/resources" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" @@ -107,34 +107,55 @@ func resolveTestSelections(selections []string, toWorkspacePath func(string) (st } // bundleWorkspacePath maps a local path, relative to the working directory, to its deployed -// path under the bundle's workspace file path. -func bundleWorkspacePath(cwd, syncRoot, workspaceFilePath, localPath string) (string, error) { +// path under filesRoot, the deployed path of the bundle's synchronized files. +func bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath string) (string, error) { if !filepath.IsAbs(localPath) { localPath = filepath.Join(cwd, localPath) } rel, err := filepath.Rel(syncRoot, localPath) if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return "", fmt.Errorf("%s is outside the synchronized bundle files in %s", localPath, syncRoot) + return "", fmt.Errorf("%s is outside the synchronized bundle files in %s", filepath.ToSlash(localPath), filepath.ToSlash(syncRoot)) + } + return path.Join(filesRoot, filepath.ToSlash(rel)), nil +} + +// bundleFilesRoot returns the deployed path of the bundle's synchronized files, as TranslatePaths +// maps resource paths, or "" for an immutable deployment, whose snapshot path the command does not know. +func bundleFilesRoot(b *bundle.Bundle) string { + switch { + case b.SyncMode() == config.SyncModeImmutable: + return "" + case b.IsSourceLinkedDeployment(): + return b.SyncRootPath + default: + return b.Config.Workspace.FilePath } - return path.Join(workspaceFilePath, filepath.ToSlash(rel)), nil } // bundleTestSelectionPath maps a relative selection to a workspace path. A path that exists locally, -// relative to the working directory, maps to its deployed path. Otherwise the selection is a test -// ID that the CLI printed, relative to the folder that testIDBase returns, unless it leaves that folder. -func bundleTestSelectionPath(cwd, syncRoot, workspaceFilePath string, testIDBase func() (string, error), localPath string) (string, error) { +// relative to the working directory, maps to its deployed path under filesRoot. Otherwise the +// selection is a test ID that the CLI printed, relative to testIDBase, unless it leaves the working +// directory. +func bundleTestSelectionPath(cwd, syncRoot, filesRoot, testIDBase, localPath string) (string, error) { rel := path.Clean(filepath.ToSlash(localPath)) - leavesBase := rel == ".." || strings.HasPrefix(rel, "../") - if _, err := os.Stat(filepath.Join(cwd, localPath)); err != nil && !leavesBase { - base, err := testIDBase() - if err != nil { - return "", err - } - if base != "" { - return testIDBasePath(base, localPath) + leavesCwd := rel == ".." || strings.HasPrefix(rel, "../") + if _, err := os.Stat(filepath.Join(cwd, localPath)); err != nil && !leavesCwd && testIDBase != "" { + return testIDBasePath(testIDBase, localPath) + } + if filesRoot == "" { + return "", errors.New("local test paths are not supported with workspace.sync_mode immutable; pass the test ID that the results show or an absolute workspace path") + } + return bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath) +} + +// needsTestIDBase reports whether a selection is a relative path, which may be a test ID. +func needsTestIDBase(selections []string) bool { + for _, selection := range selections { + if p, _, _ := strings.Cut(selection, "::"); !strings.HasPrefix(p, "/") { + return true } } - return bundleWorkspacePath(cwd, syncRoot, workspaceFilePath, localPath) + return false } // testIDBase returns the folder that the service names test cases relative to, or "" if the @@ -268,11 +289,12 @@ func (t testTarget) printHeader(out io.Writer, updateID string) { fmt.Fprintf(out, "Update: %s\n", updateID) } -// lazyTestIDBase returns a function that reads the folder of test IDs once, when a selection needs it. -func lazyTestIDBase(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) func() (string, error) { - return sync.OnceValues(func() (string, error) { - return getTestIDBase(ctx, w, pipelineID) - }) +// readTestIDBase reads the folder of test IDs from the deployed pipeline, if a selection needs it. +func readTestIDBase(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string, selections []string) (string, error) { + if !needsTestIDBase(selections) { + return "", nil + } + return getTestIDBase(ctx, w, pipelineID) } // resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. @@ -286,12 +308,11 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - base := lazyTestIDBase(ctx, w, pipelineID) + base, err := readTestIDBase(ctx, w, pipelineID, args) + if err != nil { + return testTarget{}, err + } selections, err := resolveTestSelections(args, func(p string) (string, error) { - base, err := base() - if err != nil { - return "", err - } return testIDBasePath(base, p) }) if err != nil { @@ -335,9 +356,13 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e selectionArgs = args[1:] } w := b.WorkspaceClient(ctx) - base := lazyTestIDBase(ctx, w, pipelineID) + base, err := readTestIDBase(ctx, w, pipelineID, selectionArgs) + if err != nil { + return testTarget{}, err + } + filesRoot := bundleFilesRoot(b) selections, err := resolveTestSelections(selectionArgs, func(p string) (string, error) { - return bundleTestSelectionPath(cwd, b.SyncRootPath, b.Config.Workspace.FilePath, base, p) + return bundleTestSelectionPath(cwd, b.SyncRootPath, filesRoot, base, p) }) if err != nil { return testTarget{}, err @@ -348,7 +373,7 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e func testCommand() *cobra.Command { cmd := &cobra.Command{ Use: "test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...]", - Short: "Run the tests of a pipeline", + Short: "*Preview* Run the tests of a pipeline", Long: `Run the tests of a pipeline in a test-only update. PIPELINE_KEY is the unique name of the pipeline, as defined in its YAML file. diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 3c73fade96c..fc67a1b879f 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -7,6 +7,9 @@ import ( "path/filepath" "testing" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -36,14 +39,9 @@ func TestValidatePytestArgs(t *testing.T) { } } -// TestValidatePytestArgsRejectsUnsupportedOptions covers every pytest option that the API rejects. +// TestValidatePytestArgsRejectsUnsupportedOptions covers each form of option that the API rejects. func TestValidatePytestArgsRejectsUnsupportedOptions(t *testing.T) { - for _, arg := range []string{ - "--collect-only", "-x", "--exitfirst", "--maxfail=1", "--lf", "--ff", "--cache-clear", "-n", - "--pdb", "--trace", "-q", "-v", "--color=yes", "--tb=short", "-s", "--capture=no", "-p", - "--junitxml=out.xml", "--html=out.html", "--basetemp=/tmp", "-c", "--rootdir=/", "--confcutdir=/", - "--pyargs", "--timeout=10", "--no-wait", "--junit-xml=out.xml", - } { + for _, arg := range []string{"--collect-only", "-n", "--rootdir=/", "--junit-xml=out.xml"} { err := validatePytestArgs([]string{arg}) assert.EqualError(t, err, `unsupported pytest argument "`+arg+`": only -k and -m are supported`) } @@ -126,6 +124,11 @@ func TestTestIDBase(t *testing.T) { sources: []string{"/Workspace/Users/u/a/tests/**", "/Workspace/Users/u/b/tests/**"}, want: "/Workspace/Users/u", }, + { + name: "without a root path, the parent of a folder test source", + sources: []string{"/Workspace/Users/u/tests/**"}, + want: "/Workspace/Users/u", + }, { name: "root path without test sources", rootPath: "/Workspace/Users/u/orders", @@ -152,34 +155,66 @@ func TestTestIDBasePath(t *testing.T) { assert.EqualError(t, err, "the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") } +func TestBundleFilesRoot(t *testing.T) { + filePath := "/Workspace/Users/someone/.bundle/orders/dev/files" + syncRoot := "/Workspace/Users/someone/orders" + for _, tc := range []struct { + mode config.SyncMode + want string + }{ + {mode: config.SyncModeOverwrite, want: filePath}, + // Source-linked deployments run the source files, as TranslatePaths maps them. + {mode: config.SyncModeInPlace, want: syncRoot}, + // Immutable deployments run a snapshot, whose path the command does not know. + {mode: config.SyncModeImmutable, want: ""}, + } { + b := &bundle.Bundle{SyncRootPath: syncRoot} + b.Config.Workspace.FilePath = filePath + b.Config.Workspace.SyncMode = tc.mode + assert.Equal(t, tc.want, bundleFilesRoot(b), tc.mode) + } +} + func TestBundleTestSelectionPath(t *testing.T) { syncRoot := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(syncRoot, "pipeline", "tests"), 0o755)) require.NoError(t, os.WriteFile(filepath.Join(syncRoot, "pipeline", "tests", "test_a.py"), nil, 0o644)) - filePath := "/Workspace/Users/someone/.bundle/orders/dev/files" - base := func() (string, error) { return filePath + "/pipeline", nil } - noBase := func() (string, error) { return "", nil } + filesRoot := "/Workspace/Users/someone/.bundle/orders/dev/files" + base := filesRoot + "/pipeline" // A path that exists where the user stands is a local path. - got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, filePath, base, "tests/test_a.py") + got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, filesRoot, base, "tests/test_a.py") require.NoError(t, err) - assert.Equal(t, filePath+"/pipeline/tests/test_a.py", got) + assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) // Otherwise it is a test ID, relative to the folder that test IDs are relative to. - got, err = bundleTestSelectionPath(syncRoot, syncRoot, filePath, base, "tests/test_a.py") + got, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, base, "tests/test_a.py") require.NoError(t, err) - assert.Equal(t, filePath+"/pipeline/tests/test_a.py", got) + assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) // Without that folder, it is a local path. - got, err = bundleTestSelectionPath(syncRoot, syncRoot, filePath, noBase, "tests/test_a.py") + got, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "", "tests/test_a.py") require.NoError(t, err) - assert.Equal(t, filePath+"/tests/test_a.py", got) + assert.Equal(t, filesRoot+"/tests/test_a.py", got) - // A path that leaves that folder is a local path too. - for _, testIDBase := range []func() (string, error){base, noBase} { - _, err = bundleTestSelectionPath(syncRoot, syncRoot, filePath, testIDBase, filepath.Join("..", "tests")) - assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+syncRoot) + // A path that leaves that folder is a local path too. The error shows forward slashes. + for _, testIDBase := range []string{base, ""} { + _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, testIDBase, filepath.Join("..", "tests")) + assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+filepath.ToSlash(syncRoot)) } + + // When the deployed path of local files is unknown, a local path is an error, but a test ID still works. + _, err = bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, "", base, "tests/test_a.py") + assert.EqualError(t, err, "local test paths are not supported with workspace.sync_mode immutable; pass the test ID that the results show or an absolute workspace path") + got, err = bundleTestSelectionPath(syncRoot, syncRoot, "", base, "tests/test_a.py") + require.NoError(t, err) + assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) +} + +func TestNeedsTestIDBase(t *testing.T) { + assert.False(t, needsTestIDBase(nil)) + assert.False(t, needsTestIDBase([]string{"/Workspace/tests", "/Workspace/a.py::test_x"})) + assert.True(t, needsTestIDBase([]string{"/Workspace/tests", "tests/test_a.py::test_x"})) } func TestNewStartTestUpdateRequest(t *testing.T) { From f7898c2b3e3add82ff37a6b5e2ba6c50aa60e48b Mon Sep 17 00:00:00 2001 From: Bhukya Date: Fri, 30 Oct 2026 18:59:04 +0000 Subject: [PATCH 05/15] Link the changelog entry to its PR Co-authored-by: Isaac --- .nextchanges/cli/pipelines-test.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/cli/pipelines-test.md b/.nextchanges/cli/pipelines-test.md index f7ac2beee9b..4d548dd6581 100644 --- a/.nextchanges/cli/pipelines-test.md +++ b/.nextchanges/cli/pipelines-test.md @@ -1 +1 @@ -* Add the `databricks pipelines test` command, which runs a pipeline's tests in a test-only update. +* Add the `databricks pipelines test` command, which runs a pipeline's tests in a test-only update. ([#7271](https://github.com/databricks/cli/pull/7271)) From 683cedee5e84521c9049949695b276ebb7fbe766 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 15:52:32 +0000 Subject: [PATCH 06/15] Reject a pipelines test -k or -m value that starts with - --- cmd/pipelines/test.go | 4 ++++ cmd/pipelines/test_test.go | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 373ebab8be4..b3118b32daf 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -51,6 +51,7 @@ func newStartTestUpdateRequest(selection, pytestArgs []string) startTestUpdateRe const pytestArgHint = "pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'" // validatePytestArgs accepts only -k and -m, each with a value in the next argument or attached to it. +// A value in the next argument cannot start with "-", because pytest reads it as an option. func validatePytestArgs(args []string) error { for i := 0; i < len(args); i++ { arg := args[i] @@ -59,6 +60,9 @@ func validatePytestArgs(args []string) error { if i == len(args)-1 { return fmt.Errorf("pytest argument %s requires a value: pass it in the next --pytest-arg", arg) } + if next := args[i+1]; strings.HasPrefix(next, "-") { + return fmt.Errorf("pytest argument %s requires a value, not %q: pass it in the next --pytest-arg", arg, next) + } i++ case strings.HasPrefix(arg, "-k") || strings.HasPrefix(arg, "-m"): case !strings.HasPrefix(arg, "-"): diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index fc67a1b879f..8b951b2c46a 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -22,7 +22,8 @@ func TestValidatePytestArgs(t *testing.T) { {args: nil}, {args: []string{"-k", "orders and not slow", "-m", "smoke,fast"}}, {args: []string{"-korders", "-msmoke"}}, - {args: []string{"-k", "-m"}}, + {args: []string{"-k", "-m"}, wantErr: `pytest argument -k requires a value, not "-m": pass it in the next --pytest-arg`}, + {args: []string{"-m", "--maxfail=1"}, wantErr: `pytest argument -m requires a value, not "--maxfail=1": pass it in the next --pytest-arg`}, {args: []string{"-k"}, wantErr: "pytest argument -k requires a value: pass it in the next --pytest-arg"}, {args: []string{"-m", "smoke", "-m"}, wantErr: "pytest argument -m requires a value: pass it in the next --pytest-arg"}, {args: []string{"-x"}, wantErr: `unsupported pytest argument "-x": only -k and -m are supported`}, From 368ddfed7f626e06b7fd0e57b9130a96c2ff32fd Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 16:00:47 +0000 Subject: [PATCH 07/15] Recommend pipelines deploy only after pipelines test finds the pipeline --- acceptance/pipelines/test/selection/output.txt | 4 ---- cmd/pipelines/test.go | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index 46c3137cecc..6e188b5d08a 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -105,10 +105,6 @@ Update: [UUID] === Test selections require the pipeline KEY first >>> musterr [CLI] pipelines test tests/test_orders.py -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - Error: resource with key "tests/test_orders.py" not found; the first argument must be a pipeline KEY when test selections are given === Test selections must be in the synchronized files diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index b3118b32daf..061e70c44ce 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -341,7 +341,6 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e return testTarget{}, err } ctx := cmd.Context() - suggestPipelineDeploy(ctx, cmd) if _, err := resources.Lookup(b, key, isPipeline); err != nil { return testTarget{}, fmt.Errorf("%w; the first argument must be a pipeline KEY when test selections are given", err) @@ -350,6 +349,7 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e if err != nil { return testTarget{}, err } + suggestPipelineDeploy(ctx, cmd) cwd, err := os.Getwd() if err != nil { From fd40c0ccf3eb267729e33b9e9e0df077c42c8df7 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 16:19:18 +0000 Subject: [PATCH 08/15] Resolve pipelines test selections as local paths or against root_path --- .../pipelines/test/arguments/output.txt | 10 +- .../pipelines/test/pipeline-id/output.txt | 2 +- .../pipelines/test/selection/output.txt | 8 + acceptance/pipelines/test/selection/script | 3 + .../pipelines/test/test-ids/databricks.yml | 11 -- .../pipelines/test/test-ids/out.test.toml | 1 - acceptance/pipelines/test/test-ids/output.txt | 46 ----- .../test/test-ids/pipeline/tests/test_a.py | 2 - .../pipeline/transformations/orders.py | 1 - acceptance/pipelines/test/test-ids/script | 16 -- cmd/pipelines/test.go | 166 +++++------------- cmd/pipelines/test_test.go | 101 +++-------- 12 files changed, 84 insertions(+), 283 deletions(-) delete mode 100644 acceptance/pipelines/test/test-ids/databricks.yml delete mode 100644 acceptance/pipelines/test/test-ids/out.test.toml delete mode 100644 acceptance/pipelines/test/test-ids/output.txt delete mode 100644 acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py delete mode 100644 acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py delete mode 100644 acceptance/pipelines/test/test-ids/script diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 4c3eed3f7fd..953f88b1e31 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -9,11 +9,11 @@ Outside a project, identify the pipeline with --pipeline-id; all positional arguments are then test selections. TEST_SELECTION is a test directory, file, or pytest test ID, for example -tests/test_orders.py::test_backfill. A relative path that exists locally is -converted to its deployed workspace path. Any other relative path is a test ID -as the results show it: relative to the deepest folder that holds the -pipeline's root_path and its test sources. Paths that start with / are -workspace paths. Without a selection, all of the pipeline's tests run. +tests/test_orders.py::test_backfill. In a project, a relative path is a local +path, relative to the working directory, and is converted to its deployed +workspace path. With --pipeline-id, a relative path is relative to the +pipeline's root_path. Paths that start with / are workspace paths. Without a +selection, all of the pipeline's tests run. Each --pytest-arg passes one argument to pytest, in the order given. Only -k and -m are supported, for example: diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt index 2952c195601..9138ee2e806 100644 --- a/acceptance/pipelines/test/pipeline-id/output.txt +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -21,7 +21,7 @@ Update: [UUID] === Without a root_path, test selections must be workspace paths >>> musterr [CLI] pipelines test --pipeline-id [UUID] tests -Error: cannot resolve test selection "tests": the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests +Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests >>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Shared/tests/test_shared.py::test_x Pipeline: [UUID] diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index 6e188b5d08a..61f940aee57 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -114,3 +114,11 @@ Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. Error: cannot resolve test selection "../test_outside.py": [TEST_TMP_DIR]_PARENT/test_outside.py is outside the synchronized bundle files in [TEST_TMP_DIR] + +=== Relative test selections must exist locally +>>> musterr [CLI] pipelines test my_pipeline tests/test_ordrs.py::test_backfill +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot resolve test selection "tests/test_ordrs.py::test_backfill": [TEST_TMP_DIR]/tests/test_ordrs.py does not exist; pass a local path or an absolute workspace path diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script index ca96fb91835..2fbe58cb4ed 100644 --- a/acceptance/pipelines/test/selection/script +++ b/acceptance/pipelines/test/selection/script @@ -24,3 +24,6 @@ trace musterr $CLI pipelines test tests/test_orders.py title "Test selections must be in the synchronized files" trace musterr $CLI pipelines test my_pipeline ../test_outside.py + +title "Relative test selections must exist locally" +trace musterr $CLI pipelines test my_pipeline tests/test_ordrs.py::test_backfill diff --git a/acceptance/pipelines/test/test-ids/databricks.yml b/acceptance/pipelines/test/test-ids/databricks.yml deleted file mode 100644 index 423404790c4..00000000000 --- a/acceptance/pipelines/test/test-ids/databricks.yml +++ /dev/null @@ -1,11 +0,0 @@ -bundle: - name: test-pipeline-test-ids - -resources: - pipelines: - my_pipeline: - name: test-pipeline-test-ids - root_path: ./pipeline - libraries: - - file: - path: ./pipeline/transformations/orders.py diff --git a/acceptance/pipelines/test/test-ids/out.test.toml b/acceptance/pipelines/test/test-ids/out.test.toml deleted file mode 100644 index 18b1a88417e..00000000000 --- a/acceptance/pipelines/test/test-ids/out.test.toml +++ /dev/null @@ -1 +0,0 @@ -Cloud = false diff --git a/acceptance/pipelines/test/test-ids/output.txt b/acceptance/pipelines/test/test-ids/output.txt deleted file mode 100644 index f257359806e..00000000000 --- a/acceptance/pipelines/test/test-ids/output.txt +++ /dev/null @@ -1,46 +0,0 @@ - ->>> [CLI] pipelines deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files... -Created pipelines.my_pipeline -Files: 3 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] - -=== A test ID resolves against the folder that test IDs are relative to - ->>> [CLI] pipelines test my_pipeline tests/test_a.py::test_one -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Pipeline: my_pipeline ([UUID]) -Update: [UUID] - ->>> print_requests.py //updates -{"test_selection":["/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files/pipeline/tests/test_a.py::test_one"]} - -=== The same ID from the pipeline folder, where it is a local path - ->>> [CLI] pipelines test my_pipeline tests/test_a.py::test_one -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Pipeline: my_pipeline ([UUID]) -Update: [UUID] - ->>> print_requests.py //updates -{"test_selection":["/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files/pipeline/tests/test_a.py::test_one"]} - -=== A local path from the bundle root - ->>> [CLI] pipelines test my_pipeline pipeline/tests/test_a.py -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Pipeline: my_pipeline ([UUID]) -Update: [UUID] - ->>> print_requests.py //updates -{"test_selection":["/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files/pipeline/tests/test_a.py"]} diff --git a/acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py b/acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py deleted file mode 100644 index 65de24b9c57..00000000000 --- a/acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_one(): - pass diff --git a/acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py b/acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py deleted file mode 100644 index 7d4290a117a..00000000000 --- a/acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py +++ /dev/null @@ -1 +0,0 @@ -x = 1 diff --git a/acceptance/pipelines/test/test-ids/script b/acceptance/pipelines/test/test-ids/script deleted file mode 100644 index 3972d69e926..00000000000 --- a/acceptance/pipelines/test/test-ids/script +++ /dev/null @@ -1,16 +0,0 @@ -trace $CLI pipelines deploy -rm "$OUT_REQUESTS" - -title "A test ID resolves against the folder that test IDs are relative to\n" -trace $CLI pipelines test my_pipeline tests/test_a.py::test_one -trace print_requests.py //updates | jq -c .body.test_options - -title "The same ID from the pipeline folder, where it is a local path\n" -cd pipeline -trace $CLI pipelines test my_pipeline tests/test_a.py::test_one -cd .. -trace print_requests.py //updates | jq -c .body.test_options - -title "A local path from the bundle root\n" -trace $CLI pipelines test my_pipeline pipeline/tests/test_a.py -trace print_requests.py //updates | jq -c .body.test_options diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 061e70c44ce..32542c459e6 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "io/fs" "net/http" "os" "path" @@ -136,24 +137,29 @@ func bundleFilesRoot(b *bundle.Bundle) string { } } -// bundleTestSelectionPath maps a relative selection to a workspace path. A path that exists locally, -// relative to the working directory, maps to its deployed path under filesRoot. Otherwise the -// selection is a test ID that the CLI printed, relative to testIDBase, unless it leaves the working -// directory. -func bundleTestSelectionPath(cwd, syncRoot, filesRoot, testIDBase, localPath string) (string, error) { - rel := path.Clean(filepath.ToSlash(localPath)) - leavesCwd := rel == ".." || strings.HasPrefix(rel, "../") - if _, err := os.Stat(filepath.Join(cwd, localPath)); err != nil && !leavesCwd && testIDBase != "" { - return testIDBasePath(testIDBase, localPath) - } +// bundleTestSelectionPath maps a local path, relative to the working directory, to its deployed path +// under filesRoot. The path must exist locally, so that a typo fails before compute starts. +func bundleTestSelectionPath(cwd, syncRoot, filesRoot, localPath string) (string, error) { if filesRoot == "" { - return "", errors.New("local test paths are not supported with workspace.sync_mode immutable; pass the test ID that the results show or an absolute workspace path") + return "", errors.New("local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path") + } + workspacePath, err := bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath) + if err != nil { + return "", err + } + if !filepath.IsAbs(localPath) { + localPath = filepath.Join(cwd, localPath) + } + if _, err := os.Stat(localPath); errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("%s does not exist; pass a local path or an absolute workspace path", filepath.ToSlash(localPath)) + } else if err != nil { + return "", err } - return bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath) + return workspacePath, nil } -// needsTestIDBase reports whether a selection is a relative path, which may be a test ID. -func needsTestIDBase(selections []string) bool { +// hasRelativeSelection reports whether a selection is a relative path. +func hasRelativeSelection(selections []string) bool { for _, selection := range selections { if p, _, _ := strings.Cut(selection, "::"); !strings.HasPrefix(p, "/") { return true @@ -162,95 +168,24 @@ func needsTestIDBase(selections []string) bool { return false } -// testIDBase returns the folder that the service names test cases relative to, or "" if the -// pipeline has neither a root_path nor test sources. pytest runs with this folder as its rootdir: -// the deepest folder that holds the root path and every test source or, without a root path, the -// folder of every test source. -func testIDBase(rootPath string, sources []string) string { - var folders [][]string - if rootPath != "" { - folders = append(folders, pathSegments(normalizeWorkspacePath(rootPath))) +// rootPathSelection resolves a relative path against the pipeline's root_path. +func rootPathSelection(rootPath, relPath string) (string, error) { + if rootPath == "" { + return "", errors.New("the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") } - for _, source := range sources { - segments := pathSegments(normalizeWorkspacePath(strings.TrimSuffix(source, "/**"))) - if rootPath == "" && len(segments) > 0 { - segments = segments[:len(segments)-1] - } - folders = append(folders, segments) - } - if len(folders) == 0 { - return "" - } - common := folders[0] - for _, folder := range folders[1:] { - n := 0 - for n < len(common) && n < len(folder) && common[n] == folder[n] { - n++ - } - common = common[:n] - } - return "/" + strings.Join(common, "/") -} - -// normalizeWorkspacePath adds the /Workspace prefix to an absolute path without it, as the service does. -func normalizeWorkspacePath(p string) string { - if strings.HasPrefix(p, "/") && !strings.HasPrefix(p, "/Workspace") { - return "/Workspace" + p - } - return p + return path.Join(rootPath, filepath.ToSlash(relPath)), nil } -func pathSegments(p string) []string { - var segments []string - for segment := range strings.SplitSeq(p, "/") { - if segment != "" { - segments = append(segments, segment) - } - } - return segments -} - -// testIDBasePath resolves a relative path against the folder that test IDs are relative to. -func testIDBasePath(base, relPath string) (string, error) { - if base == "" { - return "", errors.New("the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") - } - return path.Join(base, filepath.ToSlash(relPath)), nil -} - -// pipelineTestPaths holds the fields of a Get Pipeline response that decide the folder of test IDs. -// The Go SDK has no spec.testing yet, so the response is decoded from the raw API call. -type pipelineTestPaths struct { - Spec struct { - RootPath string `json:"root_path"` - Testing struct { - Sources []struct { - Glob struct { - Include string `json:"include"` - } `json:"glob"` - } `json:"sources"` - } `json:"testing"` - } `json:"spec"` -} - -// getTestIDBase reads the deployed pipeline and returns the folder that its test IDs are relative to. -func getTestIDBase(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) (string, error) { - apiClient, err := client.New(w.Config) - if err != nil { - return "", fmt.Errorf("failed to create API client: %w", err) - } - var response pipelineTestPaths - err = apiClient.Do(ctx, http.MethodGet, "/api/2.0/pipelines/"+pipelineID, auth.WorkspaceIDHeaders(w.Config), nil, nil, &response) +// getRootPath reads the root_path of the deployed pipeline. +func getRootPath(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) (string, error) { + pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineID}) if err != nil { return "", err } - var sources []string - for _, source := range response.Spec.Testing.Sources { - if source.Glob.Include != "" { - sources = append(sources, source.Glob.Include) - } + if pipeline.Spec == nil { + return "", nil } - return testIDBase(response.Spec.RootPath, sources), nil + return pipeline.Spec.RootPath, nil } // startTestUpdate starts a test-only update and returns its ID. @@ -293,14 +228,6 @@ func (t testTarget) printHeader(out io.Writer, updateID string) { fmt.Fprintf(out, "Update: %s\n", updateID) } -// readTestIDBase reads the folder of test IDs from the deployed pipeline, if a selection needs it. -func readTestIDBase(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string, selections []string) (string, error) { - if !needsTestIDBase(selections) { - return "", nil - } - return getTestIDBase(ctx, w, pipelineID) -} - // resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) (testTarget, error) { if cmd.Flag("target").Value.String() == "" { @@ -312,12 +239,16 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - base, err := readTestIDBase(ctx, w, pipelineID, args) - if err != nil { - return testTarget{}, err + var rootPath string + if hasRelativeSelection(args) { + var err error + rootPath, err = getRootPath(ctx, w, pipelineID) + if err != nil { + return testTarget{}, err + } } selections, err := resolveTestSelections(args, func(p string) (string, error) { - return testIDBasePath(base, p) + return rootPathSelection(rootPath, p) }) if err != nil { return testTarget{}, err @@ -359,19 +290,14 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e if len(args) > 1 { selectionArgs = args[1:] } - w := b.WorkspaceClient(ctx) - base, err := readTestIDBase(ctx, w, pipelineID, selectionArgs) - if err != nil { - return testTarget{}, err - } filesRoot := bundleFilesRoot(b) selections, err := resolveTestSelections(selectionArgs, func(p string) (string, error) { - return bundleTestSelectionPath(cwd, b.SyncRootPath, filesRoot, base, p) + return bundleTestSelectionPath(cwd, b.SyncRootPath, filesRoot, p) }) if err != nil { return testTarget{}, err } - return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: w}, nil + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx)}, nil } func testCommand() *cobra.Command { @@ -387,11 +313,11 @@ Outside a project, identify the pipeline with --pipeline-id; all positional arguments are then test selections. TEST_SELECTION is a test directory, file, or pytest test ID, for example -tests/test_orders.py::test_backfill. A relative path that exists locally is -converted to its deployed workspace path. Any other relative path is a test ID -as the results show it: relative to the deepest folder that holds the -pipeline's root_path and its test sources. Paths that start with / are -workspace paths. Without a selection, all of the pipeline's tests run. +tests/test_orders.py::test_backfill. In a project, a relative path is a local +path, relative to the working directory, and is converted to its deployed +workspace path. With --pipeline-id, a relative path is relative to the +pipeline's root_path. Paths that start with / are workspace paths. Without a +selection, all of the pipeline's tests run. Each --pytest-arg passes one argument to pytest, in the order given. Only -k and -m are supported, for example: diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 8b951b2c46a..ffc845ca551 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -96,64 +96,13 @@ func TestBundleWorkspacePath(t *testing.T) { assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+syncRoot) } -func TestTestIDBase(t *testing.T) { - tests := []struct { - name string - rootPath string - sources []string - want string - }{ - { - name: "deepest folder holding the root path and every test source", - rootPath: "/Workspace/Users/u/files/src/my_pipeline", - sources: []string{"/Workspace/Users/u/files/tests/**", "/Workspace/Users/u/files/checks/test_x.py"}, - want: "/Workspace/Users/u/files", - }, - { - name: "root path that holds the test sources", - rootPath: "/Users/u/sl", - sources: []string{"/Users/u/sl/tests/**"}, - want: "/Workspace/Users/u/sl", - }, - { - name: "without a root path, the folder of a file test source", - sources: []string{"/Workspace/Users/u/tests/test_x.py"}, - want: "/Workspace/Users/u/tests", - }, - { - name: "without a root path, the deepest folder holding the folders of every test source", - sources: []string{"/Workspace/Users/u/a/tests/**", "/Workspace/Users/u/b/tests/**"}, - want: "/Workspace/Users/u", - }, - { - name: "without a root path, the parent of a folder test source", - sources: []string{"/Workspace/Users/u/tests/**"}, - want: "/Workspace/Users/u", - }, - { - name: "root path without test sources", - rootPath: "/Workspace/Users/u/orders", - want: "/Workspace/Users/u/orders", - }, - { - name: "neither", - want: "", - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.want, testIDBase(tc.rootPath, tc.sources)) - }) - } -} - -func TestTestIDBasePath(t *testing.T) { - got, err := testIDBasePath("/Workspace/Users/someone/orders", "tests/test_orders.py") +func TestRootPathSelection(t *testing.T) { + got, err := rootPathSelection("/Workspace/Users/someone/orders", "tests/test_orders.py") require.NoError(t, err) assert.Equal(t, "/Workspace/Users/someone/orders/tests/test_orders.py", got) - _, err = testIDBasePath("", "tests") - assert.EqualError(t, err, "the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") + _, err = rootPathSelection("", "tests") + assert.EqualError(t, err, "the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") } func TestBundleFilesRoot(t *testing.T) { @@ -181,41 +130,33 @@ func TestBundleTestSelectionPath(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Join(syncRoot, "pipeline", "tests"), 0o755)) require.NoError(t, os.WriteFile(filepath.Join(syncRoot, "pipeline", "tests", "test_a.py"), nil, 0o644)) filesRoot := "/Workspace/Users/someone/.bundle/orders/dev/files" - base := filesRoot + "/pipeline" - // A path that exists where the user stands is a local path. - got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, filesRoot, base, "tests/test_a.py") + // A local path maps to its deployed path, relative to where the user stands. + got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, filesRoot, "tests/test_a.py") require.NoError(t, err) assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) - // Otherwise it is a test ID, relative to the folder that test IDs are relative to. - got, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, base, "tests/test_a.py") + got, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "pipeline/tests") require.NoError(t, err) - assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) + assert.Equal(t, filesRoot+"/pipeline/tests", got) - // Without that folder, it is a local path. - got, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "", "tests/test_a.py") - require.NoError(t, err) - assert.Equal(t, filesRoot+"/tests/test_a.py", got) + // A path that does not exist locally is an error, so that a typo fails before compute starts. + _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "tests/test_a.py") + assert.EqualError(t, err, filepath.ToSlash(filepath.Join(syncRoot, "tests", "test_a.py"))+" does not exist; pass a local path or an absolute workspace path") - // A path that leaves that folder is a local path too. The error shows forward slashes. - for _, testIDBase := range []string{base, ""} { - _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, testIDBase, filepath.Join("..", "tests")) - assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+filepath.ToSlash(syncRoot)) - } + // A path outside the synchronized files is an error. The error shows forward slashes. + _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, filepath.Join("..", "tests")) + assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+filepath.ToSlash(syncRoot)) - // When the deployed path of local files is unknown, a local path is an error, but a test ID still works. - _, err = bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, "", base, "tests/test_a.py") - assert.EqualError(t, err, "local test paths are not supported with workspace.sync_mode immutable; pass the test ID that the results show or an absolute workspace path") - got, err = bundleTestSelectionPath(syncRoot, syncRoot, "", base, "tests/test_a.py") - require.NoError(t, err) - assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) + // When the deployed path of local files is unknown, a local path is an error. + _, err = bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, "", "tests/test_a.py") + assert.EqualError(t, err, "local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path") } -func TestNeedsTestIDBase(t *testing.T) { - assert.False(t, needsTestIDBase(nil)) - assert.False(t, needsTestIDBase([]string{"/Workspace/tests", "/Workspace/a.py::test_x"})) - assert.True(t, needsTestIDBase([]string{"/Workspace/tests", "tests/test_a.py::test_x"})) +func TestHasRelativeSelection(t *testing.T) { + assert.False(t, hasRelativeSelection(nil)) + assert.False(t, hasRelativeSelection([]string{"/Workspace/tests", "/Workspace/a.py::test_x"})) + assert.True(t, hasRelativeSelection([]string{"/Workspace/tests", "tests/test_a.py::test_x"})) } func TestNewStartTestUpdateRequest(t *testing.T) { From b020fc599504b477580f24ebfbdaa3059224c60b Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 16:37:46 +0000 Subject: [PATCH 09/15] Cover pipelines test selections with workspace.sync_mode immutable --- .../pipelines/test/immutable/databricks.yml | 13 +++++ .../pipelines/test/immutable/out.test.toml | 1 + .../pipelines/test/immutable/output.txt | 53 +++++++++++++++++++ .../pipelines/test/immutable/pipeline_file.py | 6 +++ acceptance/pipelines/test/immutable/script | 10 ++++ .../test/immutable/tests/test_orders.py | 2 + 6 files changed, 85 insertions(+) create mode 100644 acceptance/pipelines/test/immutable/databricks.yml create mode 100644 acceptance/pipelines/test/immutable/out.test.toml create mode 100644 acceptance/pipelines/test/immutable/output.txt create mode 100644 acceptance/pipelines/test/immutable/pipeline_file.py create mode 100644 acceptance/pipelines/test/immutable/script create mode 100644 acceptance/pipelines/test/immutable/tests/test_orders.py diff --git a/acceptance/pipelines/test/immutable/databricks.yml b/acceptance/pipelines/test/immutable/databricks.yml new file mode 100644 index 00000000000..483234d0dd4 --- /dev/null +++ b/acceptance/pipelines/test/immutable/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-pipeline-test-immutable + +workspace: + sync_mode: immutable + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-immutable + libraries: + - file: + path: pipeline_file.py diff --git a/acceptance/pipelines/test/immutable/out.test.toml b/acceptance/pipelines/test/immutable/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/immutable/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/immutable/output.txt b/acceptance/pipelines/test/immutable/output.txt new file mode 100644 index 00000000000..a162096897f --- /dev/null +++ b/acceptance/pipelines/test/immutable/output.txt @@ -0,0 +1,53 @@ + +>>> [CLI] pipelines deploy +Created internal_immutable_snapshots.immutable +Created pipelines.my_pipeline +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== Local test paths are not supported with workspace.sync_mode immutable +>>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py::test_backfill +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot resolve test selection "tests/test_orders.py::test_backfill": local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path + +=== Absolute workspace paths and running all tests still work +>>> [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> [CLI] pipelines test my_pipeline +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill" + ] + } + } +} +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true + } +} diff --git a/acceptance/pipelines/test/immutable/pipeline_file.py b/acceptance/pipelines/test/immutable/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/immutable/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/immutable/script b/acceptance/pipelines/test/immutable/script new file mode 100644 index 00000000000..972ac6e3e45 --- /dev/null +++ b/acceptance/pipelines/test/immutable/script @@ -0,0 +1,10 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "Local test paths are not supported with workspace.sync_mode immutable" +trace musterr $CLI pipelines test my_pipeline tests/test_orders.py::test_backfill + +title "Absolute workspace paths and running all tests still work" +trace $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill +trace $CLI pipelines test my_pipeline +trace print_requests.py //updates diff --git a/acceptance/pipelines/test/immutable/tests/test_orders.py b/acceptance/pipelines/test/immutable/tests/test_orders.py new file mode 100644 index 00000000000..ec31eaa1b01 --- /dev/null +++ b/acceptance/pipelines/test/immutable/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_backfill(): + assert True From 8a916d9a3776b242c50ac3173de743870ae1e85e Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 17:14:52 +0000 Subject: [PATCH 10/15] Require pipelines test -k and -m values in the next --pytest-arg, as the API does --- acceptance/pipelines/test/arguments/output.txt | 10 +++++++++- acceptance/pipelines/test/arguments/script | 4 ++++ acceptance/pipelines/test/selection/output.txt | 5 +++-- acceptance/pipelines/test/selection/script | 2 +- cmd/pipelines/test.go | 16 ++++++++++++---- cmd/pipelines/test_test.go | 5 ++++- 6 files changed, 33 insertions(+), 9 deletions(-) diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 953f88b1e31..73804fcebfc 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -16,7 +16,8 @@ pipeline's root_path. Paths that start with / are workspace paths. Without a selection, all of the pipeline's tests run. Each --pytest-arg passes one argument to pytest, in the order given. Only -k -and -m are supported, for example: +and -m are supported, each followed by its value in the next --pytest-arg, for +example: databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' @@ -57,6 +58,13 @@ Error: pytest argument -k requires a value: pass it in the next --pytest-arg >>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=tests/test_orders.py Error: unsupported pytest argument "tests/test_orders.py": pass test paths and test IDs as TEST_SELECTION arguments +=== -k and -m take their value in the next --pytest-arg +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-korders +Error: pytest argument "-korders" must be split: pass --pytest-arg=-k --pytest-arg=orders + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-k --pytest-arg=-m +Error: pytest argument -k requires a value, not "-m": pass it in the next --pytest-arg + === JSON output is not supported >>> musterr [CLI] pipelines test --pipeline-id 1234 -o json Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script index dfe93d5c4b6..c4fd234f51c 100644 --- a/acceptance/pipelines/test/arguments/script +++ b/acceptance/pipelines/test/arguments/script @@ -10,5 +10,9 @@ trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-k trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=tests/test_orders.py +title "-k and -m take their value in the next --pytest-arg" +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-korders +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-k --pytest-arg=-m + title "JSON output is not supported" trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index 61f940aee57..ea657be82b5 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -25,7 +25,7 @@ Update: [UUID] } === Select tests by relative path and test ID, with pytest arguments ->>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg=-msmoke,fast +>>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg=-m --pytest-arg=smoke,fast Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -48,7 +48,8 @@ Update: [UUID] "pytest_args": [ "-k", "orders and not slow", - "-msmoke,fast" + "-m", + "smoke,fast" ] } } diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script index 2fbe58cb4ed..264927f6685 100644 --- a/acceptance/pipelines/test/selection/script +++ b/acceptance/pipelines/test/selection/script @@ -6,7 +6,7 @@ trace $CLI pipelines test trace print_requests.py //updates title "Select tests by relative path and test ID, with pytest arguments" -trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg=-msmoke,fast +trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg=-m --pytest-arg=smoke,fast trace print_requests.py //updates title "--pytest-arg also takes its value as the next argument" diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 32542c459e6..0e75a73c66a 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -22,6 +22,7 @@ import ( "github.com/databricks/cli/libs/cmdgroup" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/shellquote" databricks "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -51,8 +52,8 @@ func newStartTestUpdateRequest(selection, pytestArgs []string) startTestUpdateRe const pytestArgHint = "pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'" -// validatePytestArgs accepts only -k and -m, each with a value in the next argument or attached to it. -// A value in the next argument cannot start with "-", because pytest reads it as an option. +// validatePytestArgs accepts only -k and -m, each followed by its value in the next argument, as the API +// requires. The value cannot be empty or start with "-", because pytest reads it as an option. func validatePytestArgs(args []string) error { for i := 0; i < len(args); i++ { arg := args[i] @@ -61,11 +62,17 @@ func validatePytestArgs(args []string) error { if i == len(args)-1 { return fmt.Errorf("pytest argument %s requires a value: pass it in the next --pytest-arg", arg) } - if next := args[i+1]; strings.HasPrefix(next, "-") { + next := args[i+1] + if strings.TrimSpace(next) == "" { + return fmt.Errorf("pytest argument %s requires a non-empty value", arg) + } + if strings.HasPrefix(next, "-") { return fmt.Errorf("pytest argument %s requires a value, not %q: pass it in the next --pytest-arg", arg, next) } i++ case strings.HasPrefix(arg, "-k") || strings.HasPrefix(arg, "-m"): + option, value := arg[:2], strings.TrimPrefix(arg[2:], "=") + return fmt.Errorf("pytest argument %q must be split: pass --pytest-arg=%s --pytest-arg=%s", arg, option, shellquote.BashArg(value)) case !strings.HasPrefix(arg, "-"): return fmt.Errorf("unsupported pytest argument %q: pass test paths and test IDs as TEST_SELECTION arguments", arg) default: @@ -320,7 +327,8 @@ pipeline's root_path. Paths that start with / are workspace paths. Without a selection, all of the pipeline's tests run. Each --pytest-arg passes one argument to pytest, in the order given. Only -k -and -m are supported, for example: +and -m are supported, each followed by its value in the next --pytest-arg, for +example: databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow'`, } diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index ffc845ca551..a6ee609623a 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -21,7 +21,10 @@ func TestValidatePytestArgs(t *testing.T) { }{ {args: nil}, {args: []string{"-k", "orders and not slow", "-m", "smoke,fast"}}, - {args: []string{"-korders", "-msmoke"}}, + {args: []string{"-korders"}, wantErr: `pytest argument "-korders" must be split: pass --pytest-arg=-k --pytest-arg=orders`}, + {args: []string{"-msmoke,fast"}, wantErr: `pytest argument "-msmoke,fast" must be split: pass --pytest-arg=-m --pytest-arg='smoke,fast'`}, + {args: []string{"-k=orders and not slow"}, wantErr: `pytest argument "-k=orders and not slow" must be split: pass --pytest-arg=-k --pytest-arg='orders and not slow'`}, + {args: []string{"-k", " "}, wantErr: "pytest argument -k requires a non-empty value"}, {args: []string{"-k", "-m"}, wantErr: `pytest argument -k requires a value, not "-m": pass it in the next --pytest-arg`}, {args: []string{"-m", "--maxfail=1"}, wantErr: `pytest argument -m requires a value, not "--maxfail=1": pass it in the next --pytest-arg`}, {args: []string{"-k"}, wantErr: "pytest argument -k requires a value: pass it in the next --pytest-arg"}, From 0a42ad0b1aec042242272a19f10eb03affa6149e Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 2 Nov 2026 19:46:27 +0000 Subject: [PATCH 11/15] Link the pipelines test changelog entries to #7297 --- .nextchanges/cli/pipelines-test.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/cli/pipelines-test.md b/.nextchanges/cli/pipelines-test.md index 4d548dd6581..675b38ccc11 100644 --- a/.nextchanges/cli/pipelines-test.md +++ b/.nextchanges/cli/pipelines-test.md @@ -1 +1 @@ -* Add the `databricks pipelines test` command, which runs a pipeline's tests in a test-only update. ([#7271](https://github.com/databricks/cli/pull/7271)) +* Add the `databricks pipelines test` command, which runs a pipeline's tests in a test-only update. ([#7297](https://github.com/databricks/cli/pull/7297)) From b938039d269fe28af1c55b683d72671a365459b3 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 2 Nov 2026 22:24:56 +0000 Subject: [PATCH 12/15] Check the pipelines test sync root with filepath.IsLocal, as TranslatePaths does --- cmd/pipelines/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 0e75a73c66a..174e3e229c7 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -125,7 +125,7 @@ func bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath string) (string, er localPath = filepath.Join(cwd, localPath) } rel, err := filepath.Rel(syncRoot, localPath) - if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + if err != nil || !filepath.IsLocal(rel) { return "", fmt.Errorf("%s is outside the synchronized bundle files in %s", filepath.ToSlash(localPath), filepath.ToSlash(syncRoot)) } return path.Join(filesRoot, filepath.ToSlash(rel)), nil From 826b42ebcdd06287e8918107b309ad5cb9f2162d Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 2 Nov 2026 22:25:00 +0000 Subject: [PATCH 13/15] Test each pipelines test behavior once and use /Workspace/Users paths Error messages and edge cases stay in unit tests; wiring, request bodies and command output stay in acceptance tests. Examples avoid /Workspace/Shared. --- .../pipelines/test/arguments/output.txt | 18 +-- acceptance/pipelines/test/arguments/script | 9 +- .../pipelines/test/pipeline-id/output.txt | 8 +- acceptance/pipelines/test/pipeline-id/script | 4 +- .../pipelines/test/selection/output.txt | 48 +------- acceptance/pipelines/test/selection/script | 14 +-- cmd/pipelines/test_test.go | 113 ++---------------- 7 files changed, 24 insertions(+), 190 deletions(-) diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 73804fcebfc..2413305fded 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -42,29 +42,13 @@ Global Flags: >>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders Error: -k is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' ->>> musterr [CLI] pipelines test --pipeline-id 1234 -msmoke -Error: -m is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' - >>> musterr [CLI] pipelines test --pipeline-id 1234 -- -k orders Error: the -- separator is not supported: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' -=== Only -k and -m are supported +=== Invalid --pytest-arg values fail before any request >>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x Error: unsupported pytest argument "-x": only -k and -m are supported ->>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-k -Error: pytest argument -k requires a value: pass it in the next --pytest-arg - ->>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=tests/test_orders.py -Error: unsupported pytest argument "tests/test_orders.py": pass test paths and test IDs as TEST_SELECTION arguments - -=== -k and -m take their value in the next --pytest-arg ->>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-korders -Error: pytest argument "-korders" must be split: pass --pytest-arg=-k --pytest-arg=orders - ->>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-k --pytest-arg=-m -Error: pytest argument -k requires a value, not "-m": pass it in the next --pytest-arg - === JSON output is not supported >>> musterr [CLI] pipelines test --pipeline-id 1234 -o json Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script index c4fd234f51c..20584316005 100644 --- a/acceptance/pipelines/test/arguments/script +++ b/acceptance/pipelines/test/arguments/script @@ -2,17 +2,10 @@ trace $CLI pipelines test --help title "pytest arguments must be passed with --pytest-arg" trace musterr $CLI pipelines test --pipeline-id 1234 -k orders -trace musterr $CLI pipelines test --pipeline-id 1234 -msmoke trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders -title "Only -k and -m are supported" +title "Invalid --pytest-arg values fail before any request" trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x -trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-k -trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=tests/test_orders.py - -title "-k and -m take their value in the next --pytest-arg" -trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-korders -trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-k --pytest-arg=-m title "JSON output is not supported" trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt index 9138ee2e806..8c72ffd34f7 100644 --- a/acceptance/pipelines/test/pipeline-id/output.txt +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -1,6 +1,6 @@ === Relative test selections resolve against the pipeline root_path ->>> [CLI] pipelines test --pipeline-id [UUID] tests/test_orders.py::test_backfill /Workspace/Shared/tests +>>> [CLI] pipelines test --pipeline-id [UUID] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests Pipeline: [UUID] Update: [UUID] @@ -13,7 +13,7 @@ Update: [UUID] "test_options": { "test_selection": [ "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill", - "/Workspace/Shared/tests" + "/Workspace/Users/someone@example.com/other/tests" ] } } @@ -23,7 +23,7 @@ Update: [UUID] >>> musterr [CLI] pipelines test --pipeline-id [UUID] tests Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests ->>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Shared/tests/test_shared.py::test_x +>>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Users/someone@example.com/tests/test_other.py::test_x Pipeline: [UUID] Update: [UUID] @@ -35,7 +35,7 @@ Update: [UUID] "test_only": true, "test_options": { "test_selection": [ - "/Workspace/Shared/tests/test_shared.py::test_x" + "/Workspace/Users/someone@example.com/tests/test_other.py::test_x" ] } } diff --git a/acceptance/pipelines/test/pipeline-id/script b/acceptance/pipelines/test/pipeline-id/script index 2ef512b1210..5f07186c723 100644 --- a/acceptance/pipelines/test/pipeline-id/script +++ b/acceptance/pipelines/test/pipeline-id/script @@ -3,12 +3,12 @@ no_root_pipeline_id=$($CLI pipelines create --json '{"name": "no-root"}' | jq -r rm "$OUT_REQUESTS" title "Relative test selections resolve against the pipeline root_path" -trace $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Shared/tests +trace $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests trace print_requests.py //updates title "Without a root_path, test selections must be workspace paths" trace musterr $CLI pipelines test --pipeline-id $no_root_pipeline_id tests -trace $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Shared/tests/test_shared.py::test_x +trace $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/tests/test_other.py::test_x trace print_requests.py //updates title "Outside a project, --pipeline-id is required" diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index ea657be82b5..f5a290665f9 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -24,8 +24,8 @@ Update: [UUID] } } -=== Select tests by relative path and test ID, with pytest arguments ->>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg=-m --pytest-arg=smoke,fast +=== Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms +>>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -43,7 +43,7 @@ Update: [UUID] "test_selection": [ "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests", "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_backfill", - "/Workspace/Shared/tests/test_shared.py" + "/Workspace/Users/someone@example.com/tests/test_other.py" ], "pytest_args": [ "-k", @@ -55,32 +55,6 @@ Update: [UUID] } } -=== --pytest-arg also takes its value as the next argument ->>> [CLI] pipelines test my_pipeline --pytest-arg -k --pytest-arg orders, returns --pytest-arg -m --pytest-arg smoke -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Pipeline: my_pipeline ([UUID]) -Update: [UUID] - ->>> print_requests.py //updates -{ - "method": "POST", - "path": "/api/2.0/pipelines/[UUID]/updates", - "body": { - "test_only": true, - "test_options": { - "pytest_args": [ - "-k", - "orders, returns", - "-m", - "smoke" - ] - } - } -} - === Relative paths resolve from the working directory >>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order Recommendation: This command runs the last deployed version of the code @@ -107,19 +81,3 @@ Update: [UUID] === Test selections require the pipeline KEY first >>> musterr [CLI] pipelines test tests/test_orders.py Error: resource with key "tests/test_orders.py" not found; the first argument must be a pipeline KEY when test selections are given - -=== Test selections must be in the synchronized files ->>> musterr [CLI] pipelines test my_pipeline ../test_outside.py -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Error: cannot resolve test selection "../test_outside.py": [TEST_TMP_DIR]_PARENT/test_outside.py is outside the synchronized bundle files in [TEST_TMP_DIR] - -=== Relative test selections must exist locally ->>> musterr [CLI] pipelines test my_pipeline tests/test_ordrs.py::test_backfill -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Error: cannot resolve test selection "tests/test_ordrs.py::test_backfill": [TEST_TMP_DIR]/tests/test_ordrs.py does not exist; pass a local path or an absolute workspace path diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script index 264927f6685..1134c783f15 100644 --- a/acceptance/pipelines/test/selection/script +++ b/acceptance/pipelines/test/selection/script @@ -5,12 +5,8 @@ title "Run all tests of the only pipeline" trace $CLI pipelines test trace print_requests.py //updates -title "Select tests by relative path and test ID, with pytest arguments" -trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg=-m --pytest-arg=smoke,fast -trace print_requests.py //updates - -title "--pytest-arg also takes its value as the next argument" -trace $CLI pipelines test my_pipeline --pytest-arg -k --pytest-arg 'orders, returns' --pytest-arg -m --pytest-arg smoke +title "Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms" +trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg -m --pytest-arg smoke,fast trace print_requests.py //updates title "Relative paths resolve from the working directory" @@ -21,9 +17,3 @@ trace print_requests.py //updates title "Test selections require the pipeline KEY first" trace musterr $CLI pipelines test tests/test_orders.py - -title "Test selections must be in the synchronized files" -trace musterr $CLI pipelines test my_pipeline ../test_outside.py - -title "Relative test selections must exist locally" -trace musterr $CLI pipelines test my_pipeline tests/test_ordrs.py::test_backfill diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index a6ee609623a..dc4de92ed0f 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -1,7 +1,6 @@ package pipelines import ( - "encoding/json" "errors" "os" "path/filepath" @@ -22,15 +21,12 @@ func TestValidatePytestArgs(t *testing.T) { {args: nil}, {args: []string{"-k", "orders and not slow", "-m", "smoke,fast"}}, {args: []string{"-korders"}, wantErr: `pytest argument "-korders" must be split: pass --pytest-arg=-k --pytest-arg=orders`}, - {args: []string{"-msmoke,fast"}, wantErr: `pytest argument "-msmoke,fast" must be split: pass --pytest-arg=-m --pytest-arg='smoke,fast'`}, {args: []string{"-k=orders and not slow"}, wantErr: `pytest argument "-k=orders and not slow" must be split: pass --pytest-arg=-k --pytest-arg='orders and not slow'`}, {args: []string{"-k", " "}, wantErr: "pytest argument -k requires a non-empty value"}, {args: []string{"-k", "-m"}, wantErr: `pytest argument -k requires a value, not "-m": pass it in the next --pytest-arg`}, - {args: []string{"-m", "--maxfail=1"}, wantErr: `pytest argument -m requires a value, not "--maxfail=1": pass it in the next --pytest-arg`}, {args: []string{"-k"}, wantErr: "pytest argument -k requires a value: pass it in the next --pytest-arg"}, - {args: []string{"-m", "smoke", "-m"}, wantErr: "pytest argument -m requires a value: pass it in the next --pytest-arg"}, {args: []string{"-x"}, wantErr: `unsupported pytest argument "-x": only -k and -m are supported`}, - {args: []string{"--maxfail=1"}, wantErr: `unsupported pytest argument "--maxfail=1": only -k and -m are supported`}, + {args: []string{"--collect-only"}, wantErr: `unsupported pytest argument "--collect-only": only -k and -m are supported`}, {args: []string{"tests/test_orders.py"}, wantErr: `unsupported pytest argument "tests/test_orders.py": pass test paths and test IDs as TEST_SELECTION arguments`}, } for _, tc := range tests { @@ -43,14 +39,6 @@ func TestValidatePytestArgs(t *testing.T) { } } -// TestValidatePytestArgsRejectsUnsupportedOptions covers each form of option that the API rejects. -func TestValidatePytestArgsRejectsUnsupportedOptions(t *testing.T) { - for _, arg := range []string{"--collect-only", "-n", "--rootdir=/", "--junit-xml=out.xml"} { - err := validatePytestArgs([]string{arg}) - assert.EqualError(t, err, `unsupported pytest argument "`+arg+`": only -k and -m are supported`) - } -} - func TestResolveTestSelections(t *testing.T) { toWorkspacePath := func(p string) (string, error) { if p == "missing" { @@ -78,70 +66,24 @@ func TestResolveTestSelections(t *testing.T) { assert.EqualError(t, err, `cannot resolve test selection "missing::test_x": not found`) } -func TestBundleWorkspacePath(t *testing.T) { - syncRoot := t.TempDir() - bundleRoot := filepath.Join(syncRoot, "orders") - filePath := "/Workspace/Users/someone/.bundle/orders/dev/files" - - got, err := bundleWorkspacePath(bundleRoot, syncRoot, filePath, filepath.Join("tests", "test_orders.py")) - require.NoError(t, err) - assert.Equal(t, filePath+"/orders/tests/test_orders.py", got) - - got, err = bundleWorkspacePath(filepath.Join(bundleRoot, "tests"), syncRoot, filePath, filepath.Join("..", "conftest.py")) - require.NoError(t, err) - assert.Equal(t, filePath+"/orders/conftest.py", got) - - got, err = bundleWorkspacePath(syncRoot, syncRoot, filePath, ".") - require.NoError(t, err) - assert.Equal(t, filePath, got) - - _, err = bundleWorkspacePath(syncRoot, syncRoot, filePath, filepath.Join("..", "tests")) - assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+syncRoot) -} - -func TestRootPathSelection(t *testing.T) { - got, err := rootPathSelection("/Workspace/Users/someone/orders", "tests/test_orders.py") - require.NoError(t, err) - assert.Equal(t, "/Workspace/Users/someone/orders/tests/test_orders.py", got) - - _, err = rootPathSelection("", "tests") - assert.EqualError(t, err, "the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") -} - -func TestBundleFilesRoot(t *testing.T) { - filePath := "/Workspace/Users/someone/.bundle/orders/dev/files" - syncRoot := "/Workspace/Users/someone/orders" - for _, tc := range []struct { - mode config.SyncMode - want string - }{ - {mode: config.SyncModeOverwrite, want: filePath}, - // Source-linked deployments run the source files, as TranslatePaths maps them. - {mode: config.SyncModeInPlace, want: syncRoot}, - // Immutable deployments run a snapshot, whose path the command does not know. - {mode: config.SyncModeImmutable, want: ""}, - } { - b := &bundle.Bundle{SyncRootPath: syncRoot} - b.Config.Workspace.FilePath = filePath - b.Config.Workspace.SyncMode = tc.mode - assert.Equal(t, tc.want, bundleFilesRoot(b), tc.mode) - } +// Source-linked deployments run the source files, as TranslatePaths maps them. +func TestBundleFilesRootSourceLinked(t *testing.T) { + b := &bundle.Bundle{SyncRootPath: "/Workspace/Users/someone/orders"} + b.Config.Workspace.FilePath = "/Workspace/Users/someone/.bundle/orders/dev/files" + b.Config.Workspace.SyncMode = config.SyncModeInPlace + assert.Equal(t, "/Workspace/Users/someone/orders", bundleFilesRoot(b)) } func TestBundleTestSelectionPath(t *testing.T) { syncRoot := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(syncRoot, "pipeline", "tests"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(syncRoot, "pipeline", "tests", "test_a.py"), nil, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(syncRoot, "conftest.py"), nil, 0o644)) filesRoot := "/Workspace/Users/someone/.bundle/orders/dev/files" - // A local path maps to its deployed path, relative to where the user stands. - got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, filesRoot, "tests/test_a.py") - require.NoError(t, err) - assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) - - got, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "pipeline/tests") + // A path may leave the working directory as long as it stays in the synchronized files. + got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline", "tests"), syncRoot, filesRoot, filepath.Join("..", "..", "conftest.py")) require.NoError(t, err) - assert.Equal(t, filesRoot+"/pipeline/tests", got) + assert.Equal(t, filesRoot+"/conftest.py", got) // A path that does not exist locally is an error, so that a typo fails before compute starts. _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "tests/test_a.py") @@ -150,37 +92,4 @@ func TestBundleTestSelectionPath(t *testing.T) { // A path outside the synchronized files is an error. The error shows forward slashes. _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, filepath.Join("..", "tests")) assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+filepath.ToSlash(syncRoot)) - - // When the deployed path of local files is unknown, a local path is an error. - _, err = bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, "", "tests/test_a.py") - assert.EqualError(t, err, "local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path") -} - -func TestHasRelativeSelection(t *testing.T) { - assert.False(t, hasRelativeSelection(nil)) - assert.False(t, hasRelativeSelection([]string{"/Workspace/tests", "/Workspace/a.py::test_x"})) - assert.True(t, hasRelativeSelection([]string{"/Workspace/tests", "tests/test_a.py::test_x"})) -} - -func TestNewStartTestUpdateRequest(t *testing.T) { - tests := []struct { - selection []string - pytestArgs []string - want string - }{ - {want: `{"test_only":true}`}, - { - selection: []string{"/Workspace/tests"}, - want: `{"test_only":true,"test_options":{"test_selection":["/Workspace/tests"]}}`, - }, - { - pytestArgs: []string{"-m", "smoke,fast"}, - want: `{"test_only":true,"test_options":{"pytest_args":["-m","smoke,fast"]}}`, - }, - } - for _, tc := range tests { - body, err := json.Marshal(newStartTestUpdateRequest(tc.selection, tc.pytestArgs)) - require.NoError(t, err) - assert.JSONEq(t, tc.want, string(body)) - } } From abb95372d44478f214228d7aa12d0babae7651ed Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 2 Nov 2026 22:29:19 +0000 Subject: [PATCH 14/15] Test how pipelines test shows a submit error, such as an active update --- .../pipelines/test/submit-error/out.test.toml | 1 + acceptance/pipelines/test/submit-error/output.txt | 13 +++++++++++++ acceptance/pipelines/test/submit-error/script | 3 +++ acceptance/pipelines/test/submit-error/test.toml | 9 +++++++++ 4 files changed, 26 insertions(+) create mode 100644 acceptance/pipelines/test/submit-error/out.test.toml create mode 100644 acceptance/pipelines/test/submit-error/output.txt create mode 100644 acceptance/pipelines/test/submit-error/script create mode 100644 acceptance/pipelines/test/submit-error/test.toml diff --git a/acceptance/pipelines/test/submit-error/out.test.toml b/acceptance/pipelines/test/submit-error/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/submit-error/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt new file mode 100644 index 00000000000..524700892b2 --- /dev/null +++ b/acceptance/pipelines/test/submit-error/output.txt @@ -0,0 +1,13 @@ + +=== A submit error from the service is shown as is, and no update ID is printed +>>> musterr [CLI] pipelines test --pipeline-id 1234 +Error: An active update '[UUID]' already exists for pipeline '1234'. + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/1234/updates", + "body": { + "test_only": true + } +} diff --git a/acceptance/pipelines/test/submit-error/script b/acceptance/pipelines/test/submit-error/script new file mode 100644 index 00000000000..fe472e04fb5 --- /dev/null +++ b/acceptance/pipelines/test/submit-error/script @@ -0,0 +1,3 @@ +title "A submit error from the service is shown as is, and no update ID is printed" +trace musterr $CLI pipelines test --pipeline-id 1234 +trace print_requests.py //updates diff --git a/acceptance/pipelines/test/submit-error/test.toml b/acceptance/pipelines/test/submit-error/test.toml new file mode 100644 index 00000000000..166ee3c82b6 --- /dev/null +++ b/acceptance/pipelines/test/submit-error/test.toml @@ -0,0 +1,9 @@ +[[Server]] +Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates" +Response.Body = ''' +{ + "error_code": "RESOURCE_CONFLICT", + "message": "An active update '00000000-0000-0000-0000-000000000001' already exists for pipeline '1234'." +} +''' +Response.StatusCode = 409 From 5db2dc999f68b20649d1c149981a22a265d41cb2 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 3 Nov 2026 00:41:26 +0000 Subject: [PATCH 15/15] Gate databricks pipelines test behind DATABRICKS_CLI_PIPELINES_TEST while it is experimental --- acceptance/pipelines/test/arguments/output.txt | 6 ++++++ acceptance/pipelines/test/arguments/script | 3 +++ acceptance/pipelines/test/test.toml | 2 ++ cmd/pipelines/test.go | 9 +++++++++ 4 files changed, 20 insertions(+) diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 2413305fded..ed845172e52 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -2,6 +2,8 @@ >>> [CLI] pipelines test --help Run the tests of a pipeline in a test-only update. +This command is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it. + PIPELINE_KEY is the unique name of the pipeline, as defined in its YAML file. It is required when test selections are given. If there is only one pipeline in the project and no arguments are given, it is selected automatically. @@ -38,6 +40,10 @@ Global Flags: -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in project config. Example: --var="foo=bar" +=== The command is experimental and off by default +>>> musterr [CLI] pipelines test --pipeline-id 1234 +Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it + === pytest arguments must be passed with --pytest-arg >>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders Error: -k is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script index 20584316005..d027c13dcb8 100644 --- a/acceptance/pipelines/test/arguments/script +++ b/acceptance/pipelines/test/arguments/script @@ -1,5 +1,8 @@ trace $CLI pipelines test --help +title "The command is experimental and off by default" +(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234) + title "pytest arguments must be passed with --pytest-arg" trace musterr $CLI pipelines test --pipeline-id 1234 -k orders trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders diff --git a/acceptance/pipelines/test/test.toml b/acceptance/pipelines/test/test.toml index 9fc0358f8ac..b20092d3b88 100644 --- a/acceptance/pipelines/test/test.toml +++ b/acceptance/pipelines/test/test.toml @@ -1,3 +1,5 @@ RecordRequests = true Ignore = ['.databricks'] Env.MSYS_NO_PATHCONV = "1" +# The command is experimental; the gate case in arguments turns it off. +Env.DATABRICKS_CLI_PIPELINES_TEST = "true" diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 174e3e229c7..96295a1d1b2 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -20,6 +20,7 @@ import ( "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdgroup" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/shellquote" @@ -50,6 +51,9 @@ func newStartTestUpdateRequest(selection, pytestArgs []string) startTestUpdateRe return req } +// pipelinesTestEnvVar enables the experimental pipelines test command. +const pipelinesTestEnvVar = "DATABRICKS_CLI_PIPELINES_TEST" + const pytestArgHint = "pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'" // validatePytestArgs accepts only -k and -m, each followed by its value in the next argument, as the API @@ -313,6 +317,8 @@ func testCommand() *cobra.Command { Short: "*Preview* Run the tests of a pipeline", Long: `Run the tests of a pipeline in a test-only update. +This command is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it. + PIPELINE_KEY is the unique name of the pipeline, as defined in its YAML file. It is required when test selections are given. If there is only one pipeline in the project and no arguments are given, it is selected automatically. @@ -346,6 +352,9 @@ example: cmd.SetFlagErrorFunc(testFlagErrorFunc) cmd.RunE = func(cmd *cobra.Command, args []string) error { + if enabled, _ := env.GetBool(cmd.Context(), pipelinesTestEnvVar); !enabled { + return fmt.Errorf("pipelines test is experimental. Set %s=true to enable it", pipelinesTestEnvVar) + } if cmd.ArgsLenAtDash() >= 0 { return errors.New("the -- separator is not supported: " + pytestArgHint) }