From 06d9cfa7fd6ea0d26d1d9ea5f47498ee126c1b76 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 19:05:59 +0000 Subject: [PATCH 1/6] Add the testing property of pipelines to bundles --- .../bundles/editable-project-folder.md | 1 + .../bundles/pipelines-testing-plan.md | 1 + .nextchanges/bundles/pipelines-testing.md | 1 + .../editable-folder/databricks.yml | 33 +++ .../editable-folder/out.test.toml | 2 + .../environments/editable-folder/output.txt | 13 ++ .../editable-folder/project/pyproject.toml | 6 + .../environments/editable-folder/script | 1 + .../environments/editable-folder/test.toml | 1 + .../bundle/refschema/output/pipelines.txt | 8 + .../testing-dependencies/databricks.yml | 23 ++ .../orders/requirements.txt | 0 .../testing-dependencies/out.test.toml | 2 + .../pipelines/testing-dependencies/output.txt | 39 ++++ .../pipelines/testing-dependencies/script | 1 + .../pipelines/testing-dependencies/test.toml | 2 + .../pipelines/testing/databricks.yml | 19 ++ .../dist/test_helpers-0.1-py3-none-any.whl | 0 .../resources/pipelines/testing/out.test.toml | 2 + .../resources/pipelines/testing/output.txt | 215 ++++++++++++++++++ .../bundle/resources/pipelines/testing/script | 34 +++ .../resources/pipelines/testing/test.toml | 6 + .../pipelines/testing/tests/requirements.txt | 1 + .../pipelines/testing/tests/test_orders.py | 2 + .../testing/transformations/orders.py | 1 + .../pipelines/test/immutable/output.txt | 32 +-- .../pipelines/test/live/databricks.yml.tmpl | 19 ++ acceptance/pipelines/test/live/out.test.toml | 2 + acceptance/pipelines/test/live/output.txt | 22 ++ acceptance/pipelines/test/live/script | 20 ++ acceptance/pipelines/test/live/test.toml | 6 + .../pipelines/test/live/tests/test_orders.py | 2 + .../test/live/transformations/orders.py | 6 + .../pipelines/test/many-cases/databricks.yml | 5 +- .../pipelines/test/pipeline-id/output.txt | 14 +- acceptance/pipelines/test/pipeline-id/script | 16 +- .../pipelines/test/results/databricks.yml | 4 + .../pipelines/test/run-control/databricks.yml | 4 + .../pipelines/test/selection/databricks.yml | 4 + .../pipelines/test/selection/output.txt | 21 +- acceptance/pipelines/test/selection/script | 10 +- .../mutator/paths/pipeline_paths_visitor.go | 60 +++-- .../paths/pipeline_paths_visitor_test.go | 7 + bundle/config/mutator/translate_paths.go | 24 +- bundle/config/mutator/translate_paths_test.go | 44 ++++ bundle/config/resources/pipeline.go | 28 +++ .../validate/pipeline_testing_dependencies.go | 123 ++++++++++ .../pipeline_testing_dependencies_test.go | 64 ++++++ bundle/direct/dresources/pipeline.go | 93 +++++++- bundle/direct/dresources/pipeline_test.go | 61 +++++ bundle/internal/schema/annotations.yml | 20 ++ bundle/libraries/expand_glob_references.go | 13 ++ bundle/libraries/remote_path.go | 1 + bundle/phases/initialize.go | 4 + bundle/schema/jsonschema.json | 78 +++++++ libs/testserver/fake_workspace.go | 2 + libs/testserver/pipelines.go | 23 +- libs/testserver/pipelines_testing.go | 97 +++++++- libs/testserver/pipelines_testing_test.go | 51 ++++- .../databricks/bundles/pipelines/__init__.py | 24 ++ .../bundles/pipelines/_models/pipeline.py | 18 ++ .../_models/pipeline_test_environment.py | 42 ++++ .../pipelines/_models/pipeline_test_source.py | 46 ++++ .../pipelines/_models/pipeline_testing.py | 60 +++++ 64 files changed, 1475 insertions(+), 109 deletions(-) create mode 100644 .nextchanges/bundles/editable-project-folder.md create mode 100644 .nextchanges/bundles/pipelines-testing-plan.md create mode 100644 .nextchanges/bundles/pipelines-testing.md create mode 100644 acceptance/bundle/environments/editable-folder/databricks.yml create mode 100644 acceptance/bundle/environments/editable-folder/out.test.toml create mode 100644 acceptance/bundle/environments/editable-folder/output.txt create mode 100644 acceptance/bundle/environments/editable-folder/project/pyproject.toml create mode 100644 acceptance/bundle/environments/editable-folder/script create mode 100644 acceptance/bundle/environments/editable-folder/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-dependencies/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-dependencies/orders/requirements.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-dependencies/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-dependencies/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-dependencies/script create mode 100644 acceptance/bundle/resources/pipelines/testing-dependencies/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing/script create mode 100644 acceptance/bundle/resources/pipelines/testing/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/tests/requirements.txt create mode 100644 acceptance/bundle/resources/pipelines/testing/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing/transformations/orders.py create mode 100644 acceptance/pipelines/test/live/databricks.yml.tmpl create mode 100644 acceptance/pipelines/test/live/out.test.toml create mode 100644 acceptance/pipelines/test/live/output.txt create mode 100644 acceptance/pipelines/test/live/script create mode 100644 acceptance/pipelines/test/live/test.toml create mode 100644 acceptance/pipelines/test/live/tests/test_orders.py create mode 100644 acceptance/pipelines/test/live/transformations/orders.py create mode 100644 bundle/config/validate/pipeline_testing_dependencies.go create mode 100644 bundle/config/validate/pipeline_testing_dependencies_test.go create mode 100644 python/databricks/bundles/pipelines/_models/pipeline_test_environment.py create mode 100644 python/databricks/bundles/pipelines/_models/pipeline_test_source.py create mode 100644 python/databricks/bundles/pipelines/_models/pipeline_testing.py diff --git a/.nextchanges/bundles/editable-project-folder.md b/.nextchanges/bundles/editable-project-folder.md new file mode 100644 index 00000000000..23d74409d7a --- /dev/null +++ b/.nextchanges/bundles/editable-project-folder.md @@ -0,0 +1 @@ +* Fix `-e ./folder` in environment dependencies, which failed before deploy. A local project folder after `-e`, with optional extras such as `-e ./project[test]`, now maps to its deployed folder. ([#7274](https://github.com/databricks/cli/pull/7274)) diff --git a/.nextchanges/bundles/pipelines-testing-plan.md b/.nextchanges/bundles/pipelines-testing-plan.md new file mode 100644 index 00000000000..e109905d2bc --- /dev/null +++ b/.nextchanges/bundles/pipelines-testing-plan.md @@ -0,0 +1 @@ +* `bundle plan` now shows a change to a pipeline's `testing` that was made outside the bundle, and `bundle deploy` undoes it. ([#7274](https://github.com/databricks/cli/pull/7274)) diff --git a/.nextchanges/bundles/pipelines-testing.md b/.nextchanges/bundles/pipelines-testing.md new file mode 100644 index 00000000000..956e09a0b93 --- /dev/null +++ b/.nextchanges/bundles/pipelines-testing.md @@ -0,0 +1 @@ +* Add the `testing` property of pipelines, which sets the test sources and test-only dependencies that `databricks pipelines test` runs with. ([#7274](https://github.com/databricks/cli/pull/7274)) diff --git a/acceptance/bundle/environments/editable-folder/databricks.yml b/acceptance/bundle/environments/editable-folder/databricks.yml new file mode 100644 index 00000000000..f16db8d0d95 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/databricks.yml @@ -0,0 +1,33 @@ +bundle: + name: editable-folder + +resources: + jobs: + my_job: + name: my job + tasks: + - task_key: main + python_wheel_task: + package_name: orders + entry_point: main + environment_key: default + environments: + - environment_key: default + spec: + client: "1" + dependencies: + - -e ./project + + pipelines: + my_pipeline: + name: my pipeline + environment: + dependencies: + - -e ./project + testing: + sources: + - glob: + include: /Workspace/Shared/orders/tests/** + environment: + dependencies: + - -e ./project[test] diff --git a/acceptance/bundle/environments/editable-folder/out.test.toml b/acceptance/bundle/environments/editable-folder/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/environments/editable-folder/output.txt b/acceptance/bundle/environments/editable-folder/output.txt new file mode 100644 index 00000000000..5c032cead61 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] bundle validate -o json +{ + "job": [ + "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project" + ], + "pipeline": [ + "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project" + ], + "testing": [ + "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project[test]" + ] +} diff --git a/acceptance/bundle/environments/editable-folder/project/pyproject.toml b/acceptance/bundle/environments/editable-folder/project/pyproject.toml new file mode 100644 index 00000000000..468fddd1ca7 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/project/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "orders" +version = "0.1.0" + +[project.optional-dependencies] +test = ["faker"] diff --git a/acceptance/bundle/environments/editable-folder/script b/acceptance/bundle/environments/editable-folder/script new file mode 100644 index 00000000000..24314e31c00 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/script @@ -0,0 +1 @@ +trace $CLI bundle validate -o json | jq '{job: .resources.jobs.my_job.environments[0].spec.dependencies, pipeline: .resources.pipelines.my_pipeline.environment.dependencies, testing: .resources.pipelines.my_pipeline.testing.environment.dependencies}' diff --git a/acceptance/bundle/environments/editable-folder/test.toml b/acceptance/bundle/environments/editable-folder/test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/bundle/refschema/output/pipelines.txt b/acceptance/bundle/refschema/output/pipelines.txt index c8bc57f512c..9c2b4e56b5c 100644 --- a/acceptance/bundle/refschema/output/pipelines.txt +++ b/acceptance/bundle/refschema/output/pipelines.txt @@ -867,6 +867,14 @@ resources.pipelines.*.storage string ALL resources.pipelines.*.tags map[string]string ALL resources.pipelines.*.tags.* string ALL resources.pipelines.*.target string ALL +resources.pipelines.*.testing *resources.PipelineTesting ALL +resources.pipelines.*.testing.environment *resources.PipelineTestEnvironment ALL +resources.pipelines.*.testing.environment.dependencies []string ALL +resources.pipelines.*.testing.environment.dependencies[*] string ALL +resources.pipelines.*.testing.sources []resources.PipelineTestSource ALL +resources.pipelines.*.testing.sources[*] resources.PipelineTestSource ALL +resources.pipelines.*.testing.sources[*].glob *pipelines.PathPattern ALL +resources.pipelines.*.testing.sources[*].glob.include string ALL resources.pipelines.*.trigger *pipelines.PipelineTrigger ALL resources.pipelines.*.trigger.cron *pipelines.CronTrigger ALL resources.pipelines.*.trigger.cron.quartz_cron_schedule string ALL diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/databricks.yml b/acceptance/bundle/resources/pipelines/testing-dependencies/databricks.yml new file mode 100644 index 00000000000..6bc337c57d3 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: pipeline-testing-dependencies + +resources: + pipelines: + orders: + name: orders + testing: + sources: + - glob: + include: /Workspace/Shared/orders/tests/** + environment: + dependencies: + # Accepted, as by the API. + - --editable /Workspace/Shared/orders[test] + - --requirement=/Workspace/Shared/orders/requirements.txt + - -r ./orders/requirements.txt + # Rejected, as by the API. + - faker==33.0 + - git+https://github.com/example/helpers + - /Workspace/Shared/orders/requirements.txt + - --requirement ./orders/requirements.txt + - /Workspace/Shared/orders -i https://example.com/simple diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/orders/requirements.txt b/acceptance/bundle/resources/pipelines/testing-dependencies/orders/requirements.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/out.test.toml b/acceptance/bundle/resources/pipelines/testing-dependencies/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/output.txt b/acceptance/bundle/resources/pipelines/testing-dependencies/output.txt new file mode 100644 index 00000000000..434913a1bca --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/output.txt @@ -0,0 +1,39 @@ + +>>> musterr [CLI] bundle validate +Error: testing.environment.dependencies must point at files, not "faker==33.0" + at resources.pipelines.orders.testing.environment.dependencies[3] + in databricks.yml:19:15 + +A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported. + +Error: testing.environment.dependencies must point at files, not "git+https://github.com/example/helpers" + at resources.pipelines.orders.testing.environment.dependencies[4] + in databricks.yml:20:15 + +A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported. + +Error: testing.environment.dependencies needs -r before the requirements file "/Workspace/Shared/orders/requirements.txt" + at resources.pipelines.orders.testing.environment.dependencies[5] + in databricks.yml:21:15 + +Without -r, pip treats the file as a package to install. For example: -r ./tests/requirements.txt. + +Error: testing.environment.dependencies must point at files, not "--requirement ./orders/requirements.txt" + at resources.pipelines.orders.testing.environment.dependencies[6] + in databricks.yml:22:15 + +A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported. + +Error: testing.environment.dependencies must point at files, not "/Workspace/Shared/orders -i https://example.com/simple" + at resources.pipelines.orders.testing.environment.dependencies[7] + in databricks.yml:23:15 + +A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported. + +Name: pipeline-testing-dependencies +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dependencies/default + +Found 5 errors diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/script b/acceptance/bundle/resources/pipelines/testing-dependencies/script new file mode 100644 index 00000000000..44dbe24759c --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/script @@ -0,0 +1 @@ +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml b/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml new file mode 100644 index 00000000000..3be7711de0f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml @@ -0,0 +1,2 @@ +# Runs only against the local test server. +Cloud = false diff --git a/acceptance/bundle/resources/pipelines/testing/databricks.yml b/acceptance/bundle/resources/pipelines/testing/databricks.yml new file mode 100644 index 00000000000..a751a032169 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: pipeline-testing + +resources: + pipelines: + orders: + name: orders + libraries: + - glob: + include: ./transformations/** + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - --editable ${workspace.file_path}[test] + - -r ./tests/requirements.txt + - ./dist/*.whl diff --git a/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing/out.test.toml b/acceptance/bundle/resources/pipelines/testing/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt new file mode 100644 index 00000000000..e9296bd2e55 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -0,0 +1,215 @@ + +=== Relative test paths resolve to the deployed workspace paths +>>> [CLI] bundle validate -o json +{ + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "./dist/*.whl" + ] + }, + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ] +} + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Created pipelines.orders +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/transformations/**" + } + } + ], + "name": "orders", + "testing": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ], + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + } + } +} + +=== No changes after the deploy +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== A remote edit that drops testing shows in the plan +>>> [CLI] bundle plan -o json +{ + "action": "update", + "old": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ], + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + }, + "new": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ], + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + } +} + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Changing testing updates the pipeline +>>> update_file.py databricks.yml ./tests/** ./tests/unit/** + +>>> [CLI] bundle plan +update pipelines.orders + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[ORDERS_ID]", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/transformations/**" + } + } + ], + "name": "orders", + "testing": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/unit/**" + } + } + ], + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + } + } +} + +=== Removing testing sends an update without it +>>> [CLI] bundle plan +update pipelines.orders + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[ORDERS_ID]", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/transformations/**" + } + } + ], + "name": "orders" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.orders + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.orders + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script new file mode 100644 index 00000000000..c30595708fe --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -0,0 +1,34 @@ +title "Relative test paths resolve to the deployed workspace paths" +trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing + +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp + +title "No changes after the deploy" +trace $CLI bundle plan +rm "$OUT_REQUESTS" + +title "A remote edit that drops testing shows in the plan" +PIPELINE_ID=$(read_id.py orders) +$CLI api get "/api/2.0/pipelines/$PIPELINE_ID" | jq '.spec | del(.testing)' > edit.json +$CLI api put "/api/2.0/pipelines/$PIPELINE_ID" --json @edit.json &> LOG.edit +rm edit.json +trace $CLI bundle plan -o json | jq '.plan["resources.pipelines.orders"].changes.testing' +trace $CLI bundle deploy +trace $CLI bundle plan +rm "$OUT_REQUESTS" + +title "Changing testing updates the pipeline" +trace update_file.py databricks.yml './tests/**' './tests/unit/**' +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp + +title "Removing testing sends an update without it" +update_file.py databricks.yml "$(sed -n '/^ testing:/,$p' databricks.yml)" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp + +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/pipelines/testing/test.toml b/acceptance/bundle/resources/pipelines/testing/test.toml new file mode 100644 index 00000000000..45649fd8fbd --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/test.toml @@ -0,0 +1,6 @@ +# Runs only against the local test server. +Cloud = false +RecordRequests = true +Ignore = [".databricks"] +# The api get and put paths start with "/"; keep Git Bash on Windows from rewriting them. +Env.MSYS_NO_PATHCONV = "1" diff --git a/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt new file mode 100644 index 00000000000..9d7f6e827a1 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt @@ -0,0 +1 @@ +faker diff --git a/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py new file mode 100644 index 00000000000..3a22621c3bf --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py @@ -0,0 +1 @@ +from pyspark import pipelines as dp diff --git a/acceptance/pipelines/test/immutable/output.txt b/acceptance/pipelines/test/immutable/output.txt index 486b91e4a1e..855d885c12e 100644 --- a/acceptance/pipelines/test/immutable/output.txt +++ b/acceptance/pipelines/test/immutable/output.txt @@ -20,34 +20,6 @@ Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. -Pipeline: my_pipeline ([UUID]) -Update: [UUID] +Error: The pipeline has no test sources. Set `testing.sources` to the test files or folders to run. ->>> [CLI] pipelines test my_pipeline --no-wait -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Pipeline: my_pipeline ([UUID]) -Update: [UUID] - ->>> print_requests.py //updates -{ - "method": "POST", - "path": "/api/2.0/pipelines/[UUID]/updates", - "body": { - "test_only": true, - "test_options": { - "test_selection": [ - "/Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill" - ] - } - } -} -{ - "method": "POST", - "path": "/api/2.0/pipelines/[UUID]/updates", - "body": { - "test_only": true - } -} +Exit code: 1 diff --git a/acceptance/pipelines/test/live/databricks.yml.tmpl b/acceptance/pipelines/test/live/databricks.yml.tmpl new file mode 100644 index 00000000000..6dc3f0ec9de --- /dev/null +++ b/acceptance/pipelines/test/live/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: test-pipelines-test-live-$UNIQUE_NAME + +resources: + pipelines: + orders: + # The Pipelines API rejects duplicate names, so the name is unique per run. + name: test-pipelines-test-live-$UNIQUE_NAME + catalog: main + schema: default + serverless: true + root_path: . + libraries: + - file: + path: ./transformations/orders.py + testing: + sources: + - glob: + include: ./tests/** diff --git a/acceptance/pipelines/test/live/out.test.toml b/acceptance/pipelines/test/live/out.test.toml new file mode 100644 index 00000000000..314df1c2386 --- /dev/null +++ b/acceptance/pipelines/test/live/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +CloudSlow = true diff --git a/acceptance/pipelines/test/live/output.txt b/acceptance/pipelines/test/live/output.txt new file mode 100644 index 00000000000..69eeceb76fb --- /dev/null +++ b/acceptance/pipelines/test/live/output.txt @@ -0,0 +1,22 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipelines-test-live-[UNIQUE_NAME]/default/files... +Created pipelines.orders +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Run the tests and wait for the results +1 test: 1 passed, 0 failed, 0 skipped, 0 errors +>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.orders + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.orders + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipelines-test-live-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/pipelines/test/live/script b/acceptance/pipelines/test/live/script new file mode 100644 index 00000000000..4f2b883cd60 --- /dev/null +++ b/acceptance/pipelines/test/live/script @@ -0,0 +1,20 @@ +if [ -n "$CLOUD_ENV" ] && [ -z "${TEST_PIPELINES_TESTING_ENABLED:-}" ]; then + echo "SKIP_TEST set TEST_PIPELINES_TESTING_ENABLED for a workspace with pipeline testing enabled" + exit 0 +fi + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Run the tests and wait for the results\n" +# Durations differ between runs, so only the summary line and the report are compared. The script +# stops if the command fails. +$CLI pipelines test --junit-xml junit.xml &> LOG.test +grep -E '^[0-9]+ tests?:' LOG.test +grep -o ']*name="[^"]*"' junit.xml diff --git a/acceptance/pipelines/test/live/test.toml b/acceptance/pipelines/test/live/test.toml new file mode 100644 index 00000000000..8a1d4b5164c --- /dev/null +++ b/acceptance/pipelines/test/live/test.toml @@ -0,0 +1,6 @@ +# Also runs against a real workspace, when that workspace has pipeline testing enabled. +# The script skips the cloud run unless TEST_PIPELINES_TESTING_ENABLED is set. +Cloud = true +CloudSlow = true +RecordRequests = false +Ignore = [".databricks", "databricks.yml", "junit.xml"] diff --git a/acceptance/pipelines/test/live/tests/test_orders.py b/acceptance/pipelines/test/live/tests/test_orders.py new file mode 100644 index 00000000000..527956ffe74 --- /dev/null +++ b/acceptance/pipelines/test/live/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_order_total(): + assert sum([1, 2, 3]) == 6 diff --git a/acceptance/pipelines/test/live/transformations/orders.py b/acceptance/pipelines/test/live/transformations/orders.py new file mode 100644 index 00000000000..c0ec524e2ee --- /dev/null +++ b/acceptance/pipelines/test/live/transformations/orders.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.materialized_view +def orders(): + return spark.range(3) diff --git a/acceptance/pipelines/test/many-cases/databricks.yml b/acceptance/pipelines/test/many-cases/databricks.yml index 56b2facefa4..b9a6dd0f214 100644 --- a/acceptance/pipelines/test/many-cases/databricks.yml +++ b/acceptance/pipelines/test/many-cases/databricks.yml @@ -5,7 +5,10 @@ resources: pipelines: my_pipeline: name: test-pipeline-test-many-cases - root_path: . libraries: - file: path: pipeline_file.py + testing: + sources: + - glob: + include: ./tests/** diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt index 1d33c4efa8e..244da6f5fea 100644 --- a/acceptance/pipelines/test/pipeline-id/output.txt +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -1,6 +1,6 @@ === Relative test selections resolve against the pipeline root_path ->>> [CLI] pipelines test --pipeline-id [UUID] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests --no-wait +>>> [CLI] pipelines test --pipeline-id [UUID] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/orders/tests --no-wait Pipeline: [UUID] Update: [UUID] @@ -13,13 +13,13 @@ Update: [UUID] "test_options": { "test_selection": [ "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill", - "/Workspace/Users/someone@example.com/other/tests" + "/Workspace/Users/someone@example.com/orders/tests" ] } } } -=== Test case IDs are shown relative to the pipeline root_path +=== Test case IDs are shown relative to root_path >>> [CLI] pipelines test --pipeline-id [UUID] tests Pipeline: [UUID] Update: [UUID] @@ -34,7 +34,7 @@ tests/test_orders.py::test_valid_order PASSED (1.00s) Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests === With --no-wait and only workspace paths, the pipeline is not read ->>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Users/someone@example.com/tests/test_other.py::test_x --no-wait +>>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order --no-wait Pipeline: [UUID] Update: [UUID] @@ -46,7 +46,7 @@ Update: [UUID] "test_only": true, "test_options": { "test_selection": [ - "/Workspace/Users/someone@example.com/tests/test_other.py::test_x" + "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order" ] } } @@ -62,6 +62,10 @@ Running tests... 1 test: 1 passed, 0 failed, 0 skipped, 0 errors +=== Without test sources, the service rejects the run +>>> musterr [CLI] pipelines test --pipeline-id [UUID] +Error: The pipeline has no test sources. Set `testing.sources` to the test files or folders to run. + === Outside a project, --pipeline-id is required >>> musterr [CLI] pipelines test tests Error: outside a project, identify the pipeline with --pipeline-id diff --git a/acceptance/pipelines/test/pipeline-id/script b/acceptance/pipelines/test/pipeline-id/script index 3c18fd0e8fb..cd7af026930 100644 --- a/acceptance/pipelines/test/pipeline-id/script +++ b/acceptance/pipelines/test/pipeline-id/script @@ -1,16 +1,19 @@ -pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) -no_root_pipeline_id=$($CLI pipelines create --json '{"name": "no-root"}' | jq -r .pipeline_id) +# testing is not in the Go SDK yet, so "pipelines create" would drop it. +orders_tests='{"sources": [{"glob": {"include": "/Workspace/Users/someone@example.com/orders/tests/**"}}]}' +pipeline_id=$($CLI api post /api/2.0/pipelines --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders", "testing": '"$orders_tests"'}' | jq -r .pipeline_id) +no_root_pipeline_id=$($CLI api post /api/2.0/pipelines --json '{"name": "no-root", "testing": '"$orders_tests"'}' | jq -r .pipeline_id) +no_tests_pipeline_id=$($CLI api post /api/2.0/pipelines --json '{"name": "no-tests", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) $CLI workspace import-dir orders /Workspace/Users/someone@example.com/orders &> LOG.import rm "$OUT_REQUESTS" title "Relative test selections resolve against the pipeline root_path" -trace $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests --no-wait +trace $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/orders/tests --no-wait trace print_requests.py //updates # Like the service, the test server rejects a new update while one is active. $CLI api post /api/2.0/pipelines/$pipeline_id/stop &>> LOG.stop rm "$OUT_REQUESTS" -title "Test case IDs are shown relative to the pipeline root_path" +title "Test case IDs are shown relative to root_path" trace $CLI pipelines test --pipeline-id $pipeline_id tests rm "$OUT_REQUESTS" @@ -19,7 +22,7 @@ trace musterr $CLI pipelines test --pipeline-id $no_root_pipeline_id tests rm "$OUT_REQUESTS" title "With --no-wait and only workspace paths, the pipeline is not read" -trace $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/tests/test_other.py::test_x --no-wait +trace $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order --no-wait trace print_requests.py --get //pipelines $CLI api post /api/2.0/pipelines/$no_root_pipeline_id/stop &>> LOG.stop rm "$OUT_REQUESTS" @@ -28,5 +31,8 @@ title "Without a root_path, test case IDs are absolute workspace paths" trace $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/orders/tests/test_orders.py rm "$OUT_REQUESTS" +title "Without test sources, the service rejects the run" +trace musterr $CLI pipelines test --pipeline-id $no_tests_pipeline_id + title "Outside a project, --pipeline-id is required" trace musterr $CLI pipelines test tests diff --git a/acceptance/pipelines/test/results/databricks.yml b/acceptance/pipelines/test/results/databricks.yml index ab5a7d66a1a..4bfcd4fa46e 100644 --- a/acceptance/pipelines/test/results/databricks.yml +++ b/acceptance/pipelines/test/results/databricks.yml @@ -9,3 +9,7 @@ resources: libraries: - file: path: pipeline_file.py + testing: + sources: + - glob: + include: ./tests/** diff --git a/acceptance/pipelines/test/run-control/databricks.yml b/acceptance/pipelines/test/run-control/databricks.yml index 3c55712d9e8..66723f3d9d8 100644 --- a/acceptance/pipelines/test/run-control/databricks.yml +++ b/acceptance/pipelines/test/run-control/databricks.yml @@ -9,3 +9,7 @@ resources: libraries: - file: path: pipeline_file.py + testing: + sources: + - glob: + include: ./tests/** diff --git a/acceptance/pipelines/test/selection/databricks.yml b/acceptance/pipelines/test/selection/databricks.yml index 0cf849a7919..ea0662b1043 100644 --- a/acceptance/pipelines/test/selection/databricks.yml +++ b/acceptance/pipelines/test/selection/databricks.yml @@ -8,3 +8,7 @@ resources: libraries: - file: path: pipeline_file.py + testing: + sources: + - glob: + include: ./tests/** diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index a8e75414bda..aac5c8df289 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -24,8 +24,8 @@ Update: [UUID] } } -=== Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms ->>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast --no-wait +=== Select tests by relative path, test ID, and workspace path, with pytest arguments in both --pytest-arg forms +>>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_valid_order --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -43,7 +43,7 @@ Update: [UUID] "test_selection": [ "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests", "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_backfill", - "/Workspace/Users/someone@example.com/tests/test_other.py" + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_valid_order" ], "pytest_args": [ "-k", @@ -55,6 +55,21 @@ Update: [UUID] } } +=== The API rejects selections outside testing.sources +>>> musterr [CLI] pipelines test my_pipeline pipeline_file.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: The test selection '/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/pipeline_file.py' is outside the pipeline's test sources. Select a file or folder inside testing.sources. + +>>> musterr [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_other.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: The test selection '/Workspace/Users/someone@example.com/tests/test_other.py' is outside the pipeline's test sources. Select a file or folder inside testing.sources. + === Relative paths resolve from the working directory >>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order --no-wait Recommendation: This command runs the last deployed version of the code diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script index d39180763cf..edb640dda64 100644 --- a/acceptance/pipelines/test/selection/script +++ b/acceptance/pipelines/test/selection/script @@ -8,12 +8,18 @@ trace print_requests.py //updates $CLI pipelines stop my_pipeline &>> LOG.stop rm "$OUT_REQUESTS" -title "Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms" -trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg -m --pytest-arg smoke,fast --no-wait +title "Select tests by relative path, test ID, and workspace path, with pytest arguments in both --pytest-arg forms" +file_path=$($CLI bundle validate -o json | jq -r .workspace.file_path) +trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill $file_path/tests/test_orders.py::test_valid_order --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg -m --pytest-arg smoke,fast --no-wait trace print_requests.py //updates $CLI pipelines stop my_pipeline &>> LOG.stop rm "$OUT_REQUESTS" +title "The API rejects selections outside testing.sources" +trace musterr $CLI pipelines test my_pipeline pipeline_file.py +trace musterr $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_other.py +rm "$OUT_REQUESTS" + title "Relative paths resolve from the working directory" cd tests trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order --no-wait diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index b5af086dd34..88978e93ae3 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -44,48 +44,42 @@ func pipelineRewritePatterns() []pipelineRewritePattern { mode: TranslateModeDirectory, skipRewrite: noSkipRewrite, }, + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("sources"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")), + mode: TranslateModeGlob, + skipRewrite: noSkipRewrite, + }, } return allPatterns } func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { - pipelineEnvironmentsPatterns := []pipelineRewritePattern{ - { - pattern: dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - mode: TranslateModeLocalRelativeWithPrefix, - skipRewrite: func(s string) bool { - return !libraries.IsLibraryLocal(s) + var patterns []pipelineRewritePattern + // Test dependencies take the same forms as the pipeline's environment dependencies. + for _, dependencies := range []dyn.Pattern{ + base.Append(dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + } { + patterns = append(patterns, + pipelineRewritePattern{ + pattern: dependencies, + mode: TranslateModeLocalRelativeWithPrefix, + skipRewrite: func(s string) bool { + return !libraries.IsLibraryLocal(s) + }, }, - }, - } - - pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ - { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - TranslateModeEnvironmentPipFlag, - func(s string) bool { - _, _, ok := libraries.IsLocalPathInPipFlag(s) - return !ok + pipelineRewritePattern{ + pattern: dependencies, + mode: TranslateModeEnvironmentPipFlag, + skipRewrite: func(s string) bool { + _, _, ok := libraries.IsLocalPathInPipFlag(s) + return !ok + }, }, - }, + ) } - - return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) + return patterns } func VisitPipelinePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go index eb9748bba37..85c29065c5b 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go @@ -40,6 +40,13 @@ func TestVisitPipelinePaths(t *testing.T) { }, }, }, + // The acceptance test bundle/resources/pipelines/testing covers the testing paths + // that are rewritten. A package name is not a path. + Testing: &resources.PipelineTesting{ + Environment: &resources.PipelineTestEnvironment{ + Dependencies: []string{"faker"}, + }, + }, }, }, }, diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 956d010ec95..cdf9a95f902 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -180,9 +180,7 @@ func (t *translateContext) rewritePath( case paths.TranslateModeLocalRelativeWithPrefix: interp, err = t.translateLocalRelativeWithPrefixPath(ctx, input, localPath, localRelPath) case paths.TranslateModeEnvironmentPipFlag: - interp, err = t.translateFilePath(ctx, input, localPath, localRelPath) - // Add the flag prefix to the path to indicate it's a local file used in pip flags for environment dependencies. - interp = flag + " " + interp + interp, err = t.translatePipFlagPath(ctx, flag, input, localPath, localRelPath) default: return "", fmt.Errorf("unsupported translate mode: %d", opts.Mode) } @@ -251,6 +249,26 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local return path.Join(t.remoteRoot, localRelPath), nil } +// translatePipFlagPath translates the local path after a pip flag of an environment dependency, +// and keeps the flag in front of it. After -e, the path can also be a project folder, with +// optional extras such as ./project[test]. +func (t *translateContext) translatePipFlagPath(ctx context.Context, flag, literal, localFullPath, localRelPath string) (string, error) { + if flag == "-e" { + folder, extras := localRelPath, "" + if i := strings.LastIndex(localRelPath, "["); i > 0 && strings.HasSuffix(localRelPath, "]") { + folder, extras = localRelPath[:i], localRelPath[i:] + } + if info, err := t.b.SyncRoot.Stat(folder); err == nil && info.IsDir() { + return flag + " " + path.Join(t.remoteRoot, folder) + extras, nil + } + } + interp, err := t.translateFilePath(ctx, literal, localFullPath, localRelPath) + if err != nil { + return "", err + } + return flag + " " + interp, nil +} + func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { if t.skipLocalFileValidation { return path.Join(t.remoteRoot, localRelPath), nil diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 8aaf9ce9c71..b96d1ade278 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -816,6 +816,50 @@ func TestTranslatePathJobEnvironments(t *testing.T) { assert.Equal(t, "https://foo@bar.test/packages/pypi/simple", b.Config.Resources.Jobs["job"].Environments[0].Spec.Dependencies[6]) } +func TestTranslatePathEditableProjectFolder(t *testing.T) { + dir := t.TempDir() + touchEmptyFile(t, filepath.Join(dir, "project", "pyproject.toml")) + touchEmptyFile(t, filepath.Join(dir, "requirements.txt")) + + translate := func(dependencies ...string) (*bundle.Bundle, diag.Diagnostics) { + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Workspace: config.Workspace{ + FilePath: "/bundle", + }, + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Environment: &pipelines.PipelinesEnvironment{Dependencies: dependencies}, + }, + }, + }, + }, + }, + } + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}}) + return b, bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) + } + + b, diags := translate("-e ./project", "-e ./project[test]", "-e .", "-e .[test]", "-r ./requirements.txt") + require.NoError(t, diags.Error()) + assert.Equal(t, []string{ + "-e /bundle/project", + "-e /bundle/project[test]", + "-e /bundle", + "-e /bundle[test]", + "-r /bundle/requirements.txt", + }, b.Config.Resources.Pipelines["pipeline"].Environment.Dependencies) + + // -r takes a file, so a folder after it is still an error. + _, diags = translate("-r ./project") + assert.ErrorContains(t, diags.Error(), "project") +} + func TestTranslatePathWithComplexVariables(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index fd1fc61ce86..1b8b8e0ef43 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -22,6 +22,34 @@ type Pipeline struct { // Nil (unset) uses the server default, which cascades. Set to false to retain datasets // when the pipeline is deleted. Delete-time only: never sent on create/update. CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` + + // Testing is PipelineSpec.testing, which the Go SDK does not have yet. Remove it, and the + // types below, once CreatePipeline has it. + Testing *PipelineTesting `json:"testing,omitempty"` +} + +// PipelineTesting is the test configuration of a pipeline. Only test-only updates use it. +type PipelineTesting struct { + // Sources are the workspace files and folders that hold the pipeline's tests. They must + // not overlap the pipeline's libraries. + Sources []PipelineTestSource `json:"sources,omitempty"` + + // Environment lists the packages that only the tests need. + Environment *PipelineTestEnvironment `json:"environment,omitempty"` +} + +// PipelineTestSource is a workspace location that holds pipeline tests. +type PipelineTestSource struct { + // Glob is a test file, or a folder followed by /**. + Glob *pipelines.PathPattern `json:"glob,omitempty"` +} + +// PipelineTestEnvironment lists the packages that a pipeline's tests need in addition to the +// pipeline's environment. +type PipelineTestEnvironment struct { + // Dependencies are files that declare the packages: a project folder, a requirements + // file, or a wheel. Package names and URLs are rejected by the API. + Dependencies []string `json:"dependencies,omitempty"` } func (p *Pipeline) UnmarshalJSON(b []byte) error { diff --git a/bundle/config/validate/pipeline_testing_dependencies.go b/bundle/config/validate/pipeline_testing_dependencies.go new file mode 100644 index 00000000000..047c55e4f8d --- /dev/null +++ b/bundle/config/validate/pipeline_testing_dependencies.go @@ -0,0 +1,123 @@ +package validate + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" +) + +// pipelineTestingDependencyOptions are the pip options that may come before a test dependency's path. +var pipelineTestingDependencyOptions = []string{"--requirement", "--editable", "-r", "-e"} + +const ( + pipelineTestingDependencyNotFile = "not a file" + pipelineTestingDependencyNeedsRequirementOption = "requirements file without -r" +) + +type pipelineTestingDependencies struct{ bundle.RO } + +// PipelineTestingDependencies rejects the entries of testing.environment.dependencies that the API +// rejects on deploy, such as package names and URLs. It runs after paths are translated. +func PipelineTestingDependencies() bundle.ReadOnlyMutator { + return &pipelineTestingDependencies{} +} + +func (v *pipelineTestingDependencies) Name() string { + return "validate:pipeline_testing_dependencies" +} + +func (v *pipelineTestingDependencies) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + pattern := dyn.NewPattern( + dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), + dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex(), + ) + _, err := dyn.MapByPattern(b.Config.Value(), pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + dependency, ok := v.AsString() + if !ok { + return v, nil + } + summary := fmt.Sprintf("testing.environment.dependencies must point at files, not %q", dependency) + detail := "A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported." + switch pipelineTestingDependencyProblem(dependency) { + case "": + return v, nil + case pipelineTestingDependencyNeedsRequirementOption: + summary = fmt.Sprintf("testing.environment.dependencies needs -r before the requirements file %q", dependency) + detail = "Without -r, pip treats the file as a package to install. For example: -r ./tests/requirements.txt." + } + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: summary, + Detail: detail, + Locations: v.Locations(), + Paths: dyn.ToStructPaths(p), + }) + return v, nil + }) + if err != nil { + return diag.FromErr(err) + } + return diags +} + +// pipelineTestingDependencyProblem returns why dependency is a common mistake that the API rejects, +// or "" if it is not: a package name, a URL, another pip option, "..", a path outside /Workspace and +// /Volumes, or a requirements file without -r. Anything else is left to the API. dependency is a +// value after path translation, so a relative path is a local library that deploy uploads. +func pipelineTestingDependencyProblem(dependency string) string { + // A reference resolved at deploy time, such as a volume path, is checked by the API. + if dynvar.ContainsVariableReference(dependency) { + return "" + } + value := strings.TrimSpace(dependency) + option, filePath := cutPipelineTestingDependencyOption(value) + switch { + case strings.Contains(value, "://"): + return pipelineTestingDependencyNotFile + case strings.HasPrefix(filePath, "-") || strings.Contains(filePath, " -"): + return pipelineTestingDependencyNotFile + case strings.HasPrefix(filePath, "/"): + if !strings.HasPrefix(filePath, "/Workspace/") && !strings.HasPrefix(filePath, "/Volumes/") { + return pipelineTestingDependencyNotFile + } + if slices.Contains(strings.Split(filePath, "/"), "..") { + return pipelineTestingDependencyNotFile + } + // Path translation rewrites a path in the bundle after "-r " or "-e ", so a relative path that + // remains after an option was not rewritten. + case option != "" || !libraries.IsLibraryLocal(filePath): + return pipelineTestingDependencyNotFile + } + if strings.HasSuffix(strings.ToLower(filePath), ".txt") && option != "-r" && option != "--requirement" { + return pipelineTestingDependencyNeedsRequirementOption + } + return "" +} + +// cutPipelineTestingDependencyOption splits a leading pip option, such as "-r", from the path after it. +func cutPipelineTestingDependencyOption(value string) (string, string) { + for _, option := range pipelineTestingDependencyOptions { + rest, ok := strings.CutPrefix(value, option) + if !ok { + continue + } + switch { + case strings.HasPrefix(rest, "=") && strings.HasPrefix(option, "--"): + return option, strings.TrimSpace(rest[1:]) + case strings.HasPrefix(rest, " ") || strings.HasPrefix(rest, "\t"): + return option, strings.TrimSpace(rest) + case strings.HasPrefix(rest, "/") && !strings.HasPrefix(option, "--"): + return option, rest + } + } + return "", value +} diff --git a/bundle/config/validate/pipeline_testing_dependencies_test.go b/bundle/config/validate/pipeline_testing_dependencies_test.go new file mode 100644 index 00000000000..10a8954815d --- /dev/null +++ b/bundle/config/validate/pipeline_testing_dependencies_test.go @@ -0,0 +1,64 @@ +package validate + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPipelineTestingDependencyProblem(t *testing.T) { + accepted := []string{ + "--editable /Workspace/Users/someone@example.com/orders[test]", + "-e /Workspace/orders", + "-e/Workspace/orders", + "--editable=/Workspace/orders", + "-r /Workspace/orders/tests/requirements.txt", + "--requirement=/Workspace/orders/requirements.txt", + "--requirement /Volumes/main/default/reqs/requirements.TXT", + "/Volumes/main/default/wheels/helpers-0.1-py3-none-any.whl", + "/Workspace/Users/someone@example.com/my orders/project", + "./dist/*.whl", + "dist/helpers-0.1-py3-none-any.whl", + "${resources.volumes.wheels.volume_path}/helpers-0.1-py3-none-any.whl", + // Not a common mistake, so the API decides. + "/Workspace/orders\n/Workspace/shared", + } + for _, dependency := range accepted { + assert.Empty(t, pipelineTestingDependencyProblem(dependency), dependency) + } + + notFiles := []string{ + // Package names. + "faker", + "faker==33.0", + "requests>=2", + // URLs. + "git+https://github.com/example/helpers", + "https://example.com/helpers-0.1-py3-none-any.whl", + "--editable git+https://github.com/example/helpers", + // Other pip options. + "-i https://example.com/simple", + "/Workspace/orders -i example", + "--no-deps /Workspace/orders", + // Paths outside /Workspace and /Volumes. + "/dbfs/helpers-0.1-py3-none-any.whl", + "/Users/someone@example.com/helpers-0.1-py3-none-any.whl", + // "..". + "/Workspace/orders/../shared", + // A path in the bundle that path translation did not rewrite. + "--editable ./project", + "--requirement ./tests/requirements.txt", + } + for _, dependency := range notFiles { + assert.Equal(t, pipelineTestingDependencyNotFile, pipelineTestingDependencyProblem(dependency), dependency) + } + + needsRequirementOption := []string{ + "/Workspace/orders/requirements.txt", + "--editable /Workspace/orders/requirements.txt", + "./tests/requirements.txt", + } + for _, dependency := range needsRequirementOption { + assert.Equal(t, pipelineTestingDependencyNeedsRequirementOption, pipelineTestingDependencyProblem(dependency), dependency) + } +} diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 6f367d4e498..2f5517556bb 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -2,13 +2,18 @@ package dresources import ( "context" + "encoding/json" + "fmt" + "net/http" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/pipelines" ) @@ -22,6 +27,9 @@ type PipelineState struct { // STs, Views). Nil means the server default (cascade) applies. Read from persisted state at // delete time; never sent on create/update. CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` + + // Testing is PipelineSpec.testing, which the Go SDK does not have yet. + Testing *resources.PipelineTesting `json:"testing,omitempty"` } func (s *PipelineState) UnmarshalJSON(b []byte) error { @@ -52,6 +60,9 @@ type PipelineRemote struct { PipelineId string `json:"pipeline_id,omitempty"` RunAsUserName string `json:"run_as_user_name,omitempty"` State pipelines.PipelineState `json:"state,omitempty"` + + // Testing is spec.testing of the Get response, read by DoRead because the Go SDK does not have it yet. + Testing *resources.PipelineTesting `json:"testing,omitempty"` } // Custom marshaler needed because embedded CreatePipeline has its own MarshalJSON @@ -66,18 +77,27 @@ func (s PipelineRemote) MarshalJSON() ([]byte, error) { type ResourcePipeline struct { client *databricks.WorkspaceClient + + // apiClient sends the requests that the Go SDK cannot express yet because it lacks the testing + // field. It is built once, so that those requests share one rate limiter. + apiClient *client.DatabricksClient + apiClientErr error } -func (*ResourcePipeline) New(client *databricks.WorkspaceClient) *ResourcePipeline { - return &ResourcePipeline{ - client: client, +func (*ResourcePipeline) New(w *databricks.WorkspaceClient) *ResourcePipeline { + r := &ResourcePipeline{client: w, apiClient: nil, apiClientErr: nil} + // The adapter also builds a resource without a client to read its configuration. + if w != nil { + r.apiClient, r.apiClientErr = client.New(w.Config) } + return r } func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *PipelineState { return &PipelineState{ CreatePipeline: input.CreatePipeline, CascadeOnDestroy: input.CascadeOnDestroy, + Testing: input.Testing, } } @@ -86,15 +106,35 @@ func (*ResourcePipeline) RemapState(remote *PipelineRemote) *PipelineState { CreatePipeline: remote.CreatePipeline, // cascade_on_destroy is input-only and absent from PipelineRemote, so it stays nil here. CascadeOnDestroy: nil, + Testing: remote.Testing, } } +// pipelineTestingResponse holds spec.testing of a Get Pipeline response. +type pipelineTestingResponse struct { + Spec struct { + Testing *resources.PipelineTesting `json:"testing"` + } `json:"spec"` +} + +// DoRead sends the same request as GetByPipelineId, but raw, so that it can also read spec.testing, +// which GetPipelineResponse does not have yet. Switch back to GetByPipelineId once it does. func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemote, error) { - resp, err := r.client.Pipelines.GetByPipelineId(ctx, id) - if err != nil { + var raw json.RawMessage + if err := r.doRaw(ctx, http.MethodGet, "/api/2.0/pipelines/"+id, nil, &raw); err != nil { return nil, err } - return makePipelineRemote(resp), nil + var resp pipelines.GetPipelineResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, err + } + var testing pipelineTestingResponse + if err := json.Unmarshal(raw, &testing); err != nil { + return nil, err + } + remote := makePipelineRemote(&resp) + remote.Testing = testing.Spec.Testing + return remote, nil } func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { @@ -155,10 +195,20 @@ func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { PipelineId: p.PipelineId, RunAsUserName: p.RunAsUserName, State: p.State, + // GetPipelineResponse has no spec.testing; DoRead sets it from the raw response. + Testing: nil, } } func (r *ResourcePipeline) DoCreate(ctx context.Context, config *PipelineState) (string, *PipelineRemote, error) { + if config.Testing != nil { + var response pipelines.CreatePipelineResponse + err := r.doWithTesting(ctx, http.MethodPost, "/api/2.0/pipelines", config.CreatePipeline, config.Testing, &response) + if err != nil { + return "", nil, err + } + return response.PipelineId, nil, nil + } response, err := r.client.Pipelines.Create(ctx, config.CreatePipeline) if err != nil { return "", nil, err @@ -212,6 +262,9 @@ func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *Pipe ForceSendFields: utils.FilterFields[pipelines.EditPipeline](config.ForceSendFields), } + if config.Testing != nil { + return nil, r.doWithTesting(ctx, http.MethodPut, "/api/2.0/pipelines/"+id, request, config.Testing, nil) + } return nil, r.client.Pipelines.Update(ctx, request) } @@ -242,6 +295,34 @@ func (*ResourcePipeline) OverrideChangeDesc(_ context.Context, path *structpath. return nil } +// doWithTesting sends a Create or Edit request with the testing field, which the Go SDK does not +// have yet. The body is the SDK's JSON for the request plus testing. Remove this once +// CreatePipeline and EditPipeline have the field; pipelines without testing already use the SDK. +func (r *ResourcePipeline) doWithTesting(ctx context.Context, method, path string, request any, testing *resources.PipelineTesting, response any) error { + data, err := json.Marshal(request) + if err != nil { + return err + } + // RawMessage values keep the SDK's encoding, including large integers, unchanged. + var body map[string]json.RawMessage + if err := json.Unmarshal(data, &body); err != nil { + return err + } + body["testing"], err = json.Marshal(testing) + if err != nil { + return err + } + return r.doRaw(ctx, method, path, body, response) +} + +// doRaw sends a request that the Go SDK cannot express yet because it lacks the testing field. +func (r *ResourcePipeline) doRaw(ctx context.Context, method, path string, request, response any) error { + if r.apiClientErr != nil { + return fmt.Errorf("failed to create API client: %w", r.apiClientErr) + } + return r.apiClient.Do(ctx, method, path, auth.WorkspaceIDHeaders(r.client.Config), nil, request, response) +} + // Note, terraform provider either // a) reads back state at least once and fails create if state is "failed" // b) repeatededly reads state until state is "running" (if spec.Contionous is set). diff --git a/bundle/direct/dresources/pipeline_test.go b/bundle/direct/dresources/pipeline_test.go index 3629255a600..418555d0578 100644 --- a/bundle/direct/dresources/pipeline_test.go +++ b/bundle/direct/dresources/pipeline_test.go @@ -4,8 +4,12 @@ import ( "reflect" "testing" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -53,3 +57,60 @@ func TestPipelineOverrideChangeDescCascadeOnDestroy(t *testing.T) { assert.Equal(t, deployplan.Skip, change.Action) }) } + +func TestPipelineRawRequests(t *testing.T) { + server := testserver.New(t) + var bodies, workspaceIDs []string + server.Handle("POST", "/api/2.0/pipelines", func(req testserver.Request) any { + bodies = append(bodies, string(req.Body)) + workspaceIDs = append(workspaceIDs, req.Headers.Get(auth.WorkspaceIDHeader)) + return pipelines.CreatePipelineResponse{PipelineId: "p1"} + }) + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req testserver.Request) any { + workspaceIDs = append(workspaceIDs, req.Headers.Get(auth.WorkspaceIDHeader)) + return map[string]any{"pipeline_id": "p1", "spec": map[string]any{"name": "orders"}} + }) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token", WorkspaceID: "123"}) + require.NoError(t, err) + r := (&ResourcePipeline{}).New(w) + + testing := &resources.PipelineTesting{Sources: []resources.PipelineTestSource{{Glob: &pipelines.PathPattern{Include: "/Workspace/orders/tests/**"}}}} + _, _, err = r.DoCreate(t.Context(), &PipelineState{Name: "orders", Testing: testing}) + require.NoError(t, err) + _, err = r.DoRead(t.Context(), "p1") + require.NoError(t, err) + // Without testing, the request is the SDK's. + _, _, err = r.DoCreate(t.Context(), &PipelineState{Name: "orders"}) + require.NoError(t, err) + + // The raw requests carry the workspace ID, as the SDK's do. + assert.Equal(t, []string{"123", "123", "123"}, workspaceIDs) + assert.JSONEq(t, `{"name": "orders"}`, bodies[1]) +} + +func TestPipelineReadReturnsTesting(t *testing.T) { + server := testserver.New(t) + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req testserver.Request) any { + return map[string]any{ + "pipeline_id": req.Vars["pipeline_id"], + "state": "IDLE", + "spec": map[string]any{ + "name": "orders", + "testing": map[string]any{"sources": []any{map[string]any{"glob": map[string]any{"include": "/Workspace/orders/tests/**"}}}}, + }, + } + }) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + r := (&ResourcePipeline{}).New(w) + + remote, err := r.DoRead(t.Context(), "p1") + require.NoError(t, err) + assert.Equal(t, "p1", remote.PipelineId) + assert.Equal(t, "orders", remote.Name) + assert.Equal(t, pipelines.PipelineStateIdle, remote.State) + require.NotNil(t, remote.Testing) + require.Len(t, remote.Testing.Sources, 1) + assert.Equal(t, "/Workspace/orders/tests/**", remote.Testing.Sources[0].Glob.Include) + assert.Same(t, remote.Testing, r.RemapState(remote).Testing) +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index d4885599805..28ab10430c1 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1592,6 +1592,26 @@ resources: "user_name": "description": |- The name of the user granted the permission level. + "testing": + "description": |- + The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. + "x-databricks-launch-stage": |- + PRIVATE_PREVIEW + "$fields": + "environment": + "description": |- + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + "$fields": + "dependencies": + "description": |- + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and an extra, for example `--editable ${workspace.file_path}[test]`. Package names and URLs are rejected. + "sources": + "description": |- + The workspace files and folders that hold the pipeline's tests. They must not overlap the pipeline's `libraries`. + "$fields": + "glob": + "description": |- + A test file, or a folder followed by `/**`. "trigger": "deprecation_message": |- Use continuous instead diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index df089f880a7..91f5ab261b1 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -198,6 +198,15 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) +var pipelineTestingEnvDepsPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("pipelines"), + dyn.AnyKey(), + dyn.Key("testing"), + dyn.Key("environment"), + dyn.Key("dependencies"), +) + var clusterLibrariesPattern = dyn.NewPattern( dyn.Key("resources"), dyn.Key("clusters"), @@ -223,6 +232,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { pattern: pipelineEnvDepsPattern, fn: expandEnvironmentDeps, }, + { + pattern: pipelineTestingEnvDepsPattern, + fn: expandEnvironmentDeps, + }, { pattern: clusterLibrariesPattern, fn: expandLibraries, diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index 70e577a1a2a..359744328b4 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -143,6 +143,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), // The AI Runtime task's code_source_path is a local archive (typically an // artifact-built .tar.gz) that must be uploaded and referenced by its remote // path, exactly like a wheel or jar library. diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index ef74a786ca1..627d3eadb2d 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -222,6 +222,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { mutator.TranslatePaths(), + // Reject the pipelines' testing.environment.dependencies that the API rejects. Runs after + // TranslatePaths, so that it checks the paths that deploy sends. + validate.PipelineTestingDependencies(), + // Reads (typed): b.Config.Experimental.PythonWheelWrapper, b.Config.Workspace.SyncMode (checks Python wheel wrapper and deployment mode settings) // Reads (dynamic): resources.jobs.*.tasks (checks for tasks with local libraries and incompatible DBR versions) // Provides warnings when Python wheel tasks are used with DBR < 13.3 or when wheel wrapper is incompatible with source-linked deployment diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 959273d2586..f07b44dcbaa 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2309,6 +2309,12 @@ "deprecationMessage": "This field is deprecated", "deprecated": true }, + "testing": { + "description": "[Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTesting", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, "trigger": { "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger", @@ -2365,6 +2371,64 @@ } ] }, + "resources.PipelineTestEnvironment": { + "oneOf": [ + { + "type": "object", + "properties": { + "dependencies": { + "description": "A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and an extra, for example `--editable ${workspace.file_path}[test]`. Package names and URLs are rejected.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelineTestSource": { + "oneOf": [ + { + "type": "object", + "properties": { + "glob": { + "description": "A test file, or a folder followed by `/**`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PathPattern" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelineTesting": { + "oneOf": [ + { + "type": "object", + "properties": { + "environment": { + "description": "The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTestEnvironment" + }, + "sources": { + "description": "The workspace files and folders that hold the pipeline's tests. They must not overlap the pipeline's `libraries`.", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.PipelineTestSource" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresBranch": { "oneOf": [ { @@ -20875,6 +20939,20 @@ } ] }, + "resources.PipelineTestSource": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTestSource" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.SecretScopePermission": { "oneOf": [ { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 1e2fe6dc536..b1a28872475 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -218,6 +218,7 @@ type FakeWorkspace struct { Pipelines map[string]pipelines.GetPipelineResponse PipelineUpdates map[string]*fakePipelineUpdate pipelineEvents []fakePipelineEvent + pipelineTesting map[string]json.RawMessage pipelineEventTime time.Time Monitors map[string]catalog.MonitorInfo Apps map[string]apps.App @@ -520,6 +521,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Pipelines: map[string]pipelines.GetPipelineResponse{}, PipelineUpdates: map[string]*fakePipelineUpdate{}, pipelineEventTime: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + pipelineTesting: map[string]json.RawMessage{}, Monitors: map[string]catalog.MonitorInfo{}, Apps: map[string]apps.App{}, Catalogs: map[string]catalog.CatalogInfo{}, diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 48e0744acfb..d5488f17bc7 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -18,6 +18,13 @@ func (s *FakeWorkspace) PipelineGet(pipelineId string) Response { Body: map[string]string{"message": fmt.Sprintf("The specified pipeline %s was not found.", pipelineId)}, } } + if testing, ok := s.pipelineTesting[pipelineId]; ok { + body, err := withPipelineTesting(value, testing) + if err != nil { + return Response{StatusCode: 500, Body: fmt.Sprintf("internal error: %s", err)} + } + return Response{Body: body} + } return Response{ Body: value, } @@ -80,6 +87,7 @@ func (s *FakeWorkspace) PipelineCreate(req Request) Response { setSpecDefaults(&spec, pipelineId) s.Pipelines[pipelineId] = r + s.setPipelineTesting(pipelineId, req.Body) return Response{ Body: pipelines.CreatePipelineResponse{ @@ -139,6 +147,7 @@ func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response } setSpecDefaults(&spec, pipelineId) s.Pipelines[pipelineId] = item + s.setPipelineTesting(pipelineId, req.Body) return Response{} } @@ -146,8 +155,7 @@ func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response func (s *FakeWorkspace) PipelineStartUpdate(pipelineId string, body []byte) Response { defer s.LockUnlock()() - pipeline, exists := s.Pipelines[pipelineId] - if !exists { + if _, exists := s.Pipelines[pipelineId]; !exists { return Response{ StatusCode: 404, Body: map[string]string{"message": fmt.Sprintf("The specified pipeline %s was not found.", pipelineId)}, @@ -200,11 +208,14 @@ func (s *FakeWorkspace) PipelineStartUpdate(pipelineId string, body []byte) Resp if test.TestOptions != nil { selection = test.TestOptions.TestSelection } - var rootPath string - if pipeline.Spec != nil { - rootPath = pipeline.Spec.RootPath + targets, message := pipelineTestTargets(s.pipelineTestSources(pipelineId), selection) + if message != "" { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, + } } - update = s.startPipelineTestRun(pipelineId, updateId, s.collectPipelineTests(rootPath, selection)) + update = s.startPipelineTestRun(pipelineId, updateId, s.collectPipelineTests(targets)) } s.PipelineUpdates[updateId] = update diff --git a/libs/testserver/pipelines_testing.go b/libs/testserver/pipelines_testing.go index 23b8e498863..0cff5fe5890 100644 --- a/libs/testserver/pipelines_testing.go +++ b/libs/testserver/pipelines_testing.go @@ -130,17 +130,98 @@ func pipelineTestOptionsError(pytestArgs, selection []string) string { return "" } -// collectPipelineTests stands in for pytest collection: it finds the top-level "def test_*" -// functions in the workspace test_*.py files under each selection, or under rootPath without one. -// Like origin.test_case_id, a test ID is the absolute workspace path of its file followed by the -// pytest test ID, such as /Workspace/Users/someone/orders/tests/test_orders.py::test_backfill. -func (s *FakeWorkspace) collectPipelineTests(rootPath string, selection []string) []string { - if len(selection) == 0 && rootPath != "" { - selection = []string{rootPath} +// pipelineTestingRequest holds the testing field of a Create or Edit request, which is not in the +// Go SDK yet. +type pipelineTestingRequest struct { + Testing json.RawMessage `json:"testing"` +} + +// pipelineTesting is the part of the testing field that test-only updates read. +type pipelineTesting struct { + Sources []struct { + Glob *struct { + Include string `json:"include"` + } `json:"glob"` + } `json:"sources"` +} + +// setPipelineTesting records the testing field of a Create or Edit body, so that Get returns it. +// Like the rest of the spec, an Edit without testing removes it. +func (s *FakeWorkspace) setPipelineTesting(pipelineID string, body []byte) { + var request pipelineTestingRequest + _ = json.Unmarshal(body, &request) + if len(request.Testing) == 0 || string(request.Testing) == "null" { + delete(s.pipelineTesting, pipelineID) + return } + s.pipelineTesting[pipelineID] = request.Testing +} - var ids []string +// pipelineTestSources returns the glob includes of the pipeline's testing.sources. +func (s *FakeWorkspace) pipelineTestSources(pipelineID string) []string { + var testing pipelineTesting + _ = json.Unmarshal(s.pipelineTesting[pipelineID], &testing) + var sources []string + for _, source := range testing.Sources { + if source.Glob != nil { + sources = append(sources, source.Glob.Include) + } + } + return sources +} + +// withPipelineTesting adds spec.testing to a Get Pipeline response, which the Go SDK type cannot +// hold. The JSON is merged as raw values, so numbers and the other fields are kept as they are. +func withPipelineTesting(value pipelines.GetPipelineResponse, testing json.RawMessage) (map[string]json.RawMessage, error) { + data, err := json.Marshal(value) + if err != nil { + return nil, err + } + var body map[string]json.RawMessage + if err := json.Unmarshal(data, &body); err != nil { + return nil, err + } + var spec map[string]json.RawMessage + if err := json.Unmarshal(body["spec"], &spec); err != nil { + return nil, err + } + spec["testing"] = testing + body["spec"], err = json.Marshal(spec) + return body, err +} + +// pipelineTestTargets returns the paths to collect tests from: the selection, which must be +// within the test sources, or the test sources themselves. A non-empty message rejects the request. +func pipelineTestTargets(sources, selection []string) ([]string, string) { + if len(sources) == 0 { + return nil, "The pipeline has no test sources. Set `testing.sources` to the test files or folders to run." + } + if len(selection) == 0 { + var targets []string + for _, source := range sources { + targets = append(targets, strings.TrimSuffix(source, "/**")) + } + return targets, "" + } for _, target := range selection { + targetPath, _, _ := strings.Cut(target, "::") + if !slices.ContainsFunc(sources, func(source string) bool { + folder, recursive := strings.CutSuffix(source, "/**") + return targetPath == folder || (recursive && strings.HasPrefix(targetPath, folder+"/")) + }) { + return nil, fmt.Sprintf("The test selection '%s' is outside the pipeline's test sources. Select a file or folder inside testing.sources.", target) + } + } + return selection, "" +} + +// collectPipelineTests stands in for pytest collection: it finds the top-level "def test_*" +// functions in the workspace test_*.py files under each target. Like origin.test_case_id, a test ID +// is the absolute workspace path of its file followed by the pytest test ID, such as +// /Workspace/Users/someone/orders/tests/test_orders.py::test_backfill. +func (s *FakeWorkspace) collectPipelineTests(targets []string) []string { + var ids []string + for _, target := range targets { targetPath, testName, _ := strings.Cut(target, "::") var files []string if _, ok := s.files[targetPath]; ok { diff --git a/libs/testserver/pipelines_testing_test.go b/libs/testserver/pipelines_testing_test.go index c58bc074ba2..a73b8310350 100644 --- a/libs/testserver/pipelines_testing_test.go +++ b/libs/testserver/pipelines_testing_test.go @@ -1,6 +1,7 @@ package testserver import ( + "encoding/json" "net/url" "testing" @@ -12,7 +13,7 @@ import ( const testRootPath = "/Workspace/Users/someone/orders" func createTestPipelineWithTests(t *testing.T, workspace *FakeWorkspace) string { - response := workspace.PipelineCreate(Request{Body: []byte(`{"name": "orders", "root_path": "` + testRootPath + `"}`)}) + response := workspace.PipelineCreate(Request{Body: []byte(`{"name": "orders", "testing": {"sources": [{"glob": {"include": "` + testRootPath + `/tests/**"}}]}}`)}) require.Equal(t, 0, response.StatusCode) workspace.files[testRootPath+"/tests/test_orders.py"] = FileEntry{Data: []byte("def test_ok():\n pass\n\ndef test_fails():\n pass\n\n def test_nested():\n pass\n")} workspace.files[testRootPath+"/tests/test_slow.py"] = FileEntry{Data: []byte("def test_hang():\n pass\n")} @@ -57,14 +58,50 @@ func TestCollectPipelineTests(t *testing.T) { // Like origin.test_case_id, an ID is the absolute workspace path of its file and the pytest test ID. tests := testRootPath + "/tests/" assert.Equal(t, []string{tests + "test_orders.py::test_ok", tests + "test_orders.py::test_fails", tests + "test_slow.py::test_hang"}, - workspace.collectPipelineTests(testRootPath, nil)) + workspace.collectPipelineTests([]string{testRootPath + "/tests"})) assert.Equal(t, []string{tests + "test_orders.py::test_fails"}, - workspace.collectPipelineTests(testRootPath, []string{testRootPath + "/tests/test_orders.py::test_fails", testRootPath + "/tests/test_orders.py::test_fails"})) + workspace.collectPipelineTests([]string{testRootPath + "/tests/test_orders.py::test_fails", testRootPath + "/tests/test_orders.py::test_fails"})) assert.Equal(t, []string{tests + "helpers.py::test_helper"}, - workspace.collectPipelineTests(testRootPath, []string{testRootPath + "/tests/helpers.py"})) - assert.Equal(t, []string{testRootPath + "/tests/test_slow.py::test_hang"}, - workspace.collectPipelineTests("", []string{testRootPath + "/tests/test_slow.py"})) - assert.Empty(t, workspace.collectPipelineTests("", nil)) + workspace.collectPipelineTests([]string{testRootPath + "/tests/helpers.py"})) + assert.Empty(t, workspace.collectPipelineTests(nil)) +} + +func TestPipelineTestTargets(t *testing.T) { + sources := []string{"/Workspace/orders/tests/**", "/Workspace/orders/checks/test_x.py"} + + targets, message := pipelineTestTargets(sources, nil) + assert.Empty(t, message) + assert.Equal(t, []string{"/Workspace/orders/tests", "/Workspace/orders/checks/test_x.py"}, targets) + + selection := []string{"/Workspace/orders/tests/unit/test_a.py::test_a", "/Workspace/orders/tests", "/Workspace/orders/checks/test_x.py::test_x"} + targets, message = pipelineTestTargets(sources, selection) + assert.Empty(t, message) + assert.Equal(t, selection, targets) + + _, message = pipelineTestTargets(sources, []string{"/Workspace/orders/transformations/orders.py"}) + // The messages are the service's. + assert.Equal(t, "The test selection '/Workspace/orders/transformations/orders.py' is outside the pipeline's test sources. Select a file or folder inside testing.sources.", message) + _, message = pipelineTestTargets(sources, []string{"/Workspace/orders/tests2/test_a.py"}) + assert.NotEmpty(t, message) + _, message = pipelineTestTargets(nil, nil) + assert.Equal(t, "The pipeline has no test sources. Set `testing.sources` to the test files or folders to run.", message) +} + +func TestPipelineEditReplacesTestSources(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + assert.Equal(t, []string{testRootPath + "/tests/**"}, workspace.pipelineTestSources(pipelineID)) + body, err := json.Marshal(workspace.PipelineGet(pipelineID).Body) + require.NoError(t, err) + assert.Contains(t, string(body), `"testing":{"sources":[{"glob":{"include":"`+testRootPath+`/tests/**"}}]}`) + + response := workspace.PipelineUpdate(Request{Body: []byte(`{"name": "orders"}`)}, pipelineID) + require.Equal(t, 0, response.StatusCode) + assert.Empty(t, workspace.pipelineTestSources(pipelineID)) + body, err = json.Marshal(workspace.PipelineGet(pipelineID).Body) + require.NoError(t, err) + assert.NotContains(t, string(body), `"testing"`) + assert.Equal(t, 400, workspace.PipelineStartUpdate(pipelineID, []byte(`{"test_only": true}`)).StatusCode) } func TestPipelineTestUpdate(t *testing.T) { diff --git a/python/databricks/bundles/pipelines/__init__.py b/python/databricks/bundles/pipelines/__init__.py index 412cbfa59b3..d9bf369bdb3 100644 --- a/python/databricks/bundles/pipelines/__init__.py +++ b/python/databricks/bundles/pipelines/__init__.py @@ -207,6 +207,15 @@ "PipelinePermissionLevel", "PipelinePermissionLevelParam", "PipelinePermissionParam", + "PipelineTestEnvironment", + "PipelineTestEnvironmentDict", + "PipelineTestEnvironmentParam", + "PipelineTestSource", + "PipelineTestSourceDict", + "PipelineTestSourceParam", + "PipelineTesting", + "PipelineTestingDict", + "PipelineTestingParam", "PipelinesEnvironment", "PipelinesEnvironmentDict", "PipelinesEnvironmentParam", @@ -645,6 +654,21 @@ PipelinePermissionLevel, PipelinePermissionLevelParam, ) +from databricks.bundles.pipelines._models.pipeline_test_environment import ( + PipelineTestEnvironment, + PipelineTestEnvironmentDict, + PipelineTestEnvironmentParam, +) +from databricks.bundles.pipelines._models.pipeline_test_source import ( + PipelineTestSource, + PipelineTestSourceDict, + PipelineTestSourceParam, +) +from databricks.bundles.pipelines._models.pipeline_testing import ( + PipelineTesting, + PipelineTestingDict, + PipelineTestingParam, +) from databricks.bundles.pipelines._models.pipelines_environment import ( PipelinesEnvironment, PipelinesEnvironmentDict, diff --git a/python/databricks/bundles/pipelines/_models/pipeline.py b/python/databricks/bundles/pipelines/_models/pipeline.py index 070e813ef34..e0010a55e8f 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/python/databricks/bundles/pipelines/_models/pipeline.py @@ -47,6 +47,10 @@ PipelinePermission, PipelinePermissionParam, ) +from databricks.bundles.pipelines._models.pipeline_testing import ( + PipelineTesting, + PipelineTestingParam, +) from databricks.bundles.pipelines._models.pipelines_environment import ( PipelinesEnvironment, PipelinesEnvironmentParam, @@ -260,6 +264,13 @@ class Pipeline(Resource): [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. """ + testing: VariableOrOptional[PipelineTesting] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. + """ + usage_policy_id: VariableOrOptional[str] = None """ :meta private: [EXPERIMENTAL] @@ -473,6 +484,13 @@ class PipelineDict(TypedDict, total=False): [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. """ + testing: VariableOrOptional[PipelineTestingParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. + """ + usage_policy_id: VariableOrOptional[str] """ :meta private: [EXPERIMENTAL] diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py new file mode 100644 index 00000000000..3386c1d69b1 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py @@ -0,0 +1,42 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTestEnvironment: + """ + :meta private: [EXPERIMENTAL] + """ + + dependencies: VariableOrList[str] = field(default_factory=list) + """ + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and an extra, for example `--editable ${workspace.file_path}[test]`. Package names and URLs are rejected. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestEnvironmentDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestEnvironmentDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestEnvironmentDict(TypedDict, total=False): + """""" + + dependencies: VariableOrList[str] + """ + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and an extra, for example `--editable ${workspace.file_path}[test]`. Package names and URLs are rejected. + """ + + +PipelineTestEnvironmentParam = PipelineTestEnvironmentDict | PipelineTestEnvironment diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_source.py b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py new file mode 100644 index 00000000000..b893d161d69 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py @@ -0,0 +1,46 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.pipelines._models.path_pattern import ( + PathPattern, + PathPatternParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTestSource: + """ + :meta private: [EXPERIMENTAL] + """ + + glob: VariableOrOptional[PathPattern] = None + """ + A test file, or a folder followed by `/**`. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestSourceDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestSourceDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestSourceDict(TypedDict, total=False): + """""" + + glob: VariableOrOptional[PathPatternParam] + """ + A test file, or a folder followed by `/**`. + """ + + +PipelineTestSourceParam = PipelineTestSourceDict | PipelineTestSource diff --git a/python/databricks/bundles/pipelines/_models/pipeline_testing.py b/python/databricks/bundles/pipelines/_models/pipeline_testing.py new file mode 100644 index 00000000000..c8f20835c95 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_testing.py @@ -0,0 +1,60 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.pipelines._models.pipeline_test_environment import ( + PipelineTestEnvironment, + PipelineTestEnvironmentParam, +) +from databricks.bundles.pipelines._models.pipeline_test_source import ( + PipelineTestSource, + PipelineTestSourceParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTesting: + """ + :meta private: [EXPERIMENTAL] + """ + + environment: VariableOrOptional[PipelineTestEnvironment] = None + """ + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + """ + + sources: VariableOrList[PipelineTestSource] = field(default_factory=list) + """ + The workspace files and folders that hold the pipeline's tests. They must not overlap the pipeline's `libraries`. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestingDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestingDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestingDict(TypedDict, total=False): + """""" + + environment: VariableOrOptional[PipelineTestEnvironmentParam] + """ + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + """ + + sources: VariableOrList[PipelineTestSourceParam] + """ + The workspace files and folders that hold the pipeline's tests. They must not overlap the pipeline's `libraries`. + """ + + +PipelineTestingParam = PipelineTestingDict | PipelineTesting From 9651632d126c9d242e9ac7e09b6f4436067cb38e Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 19:07:59 +0000 Subject: [PATCH 2/6] Check pipeline test dependencies for wheels with the same name --- .../databricks.yml | 21 +++++++++++++++++++ .../testing-same-name-libraries/out.test.toml | 2 ++ .../testing-same-name-libraries/output.txt | 12 +++++++++++ .../test_helpers-0.1-py3-none-any.whl | 0 .../testing-same-name-libraries/script | 1 + .../testing-same-name-libraries/test.toml | 2 ++ .../tests/test_helpers-0.1-py3-none-any.whl | 0 .../tests/test_orders.py | 2 ++ .../transformations/orders.py | 1 + bundle/libraries/same_name_libraries.go | 1 + 10 files changed, 42 insertions(+) create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/script create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml new file mode 100644 index 00000000000..381095e1cd3 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: pipeline-testing-same-name-libraries + +resources: + pipelines: + orders: + name: orders + libraries: + - file: + path: ./transformations/orders.py + environment: + dependencies: + - ./pipeline/test_helpers-0.1-py3-none-any.whl + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + # A different wheel with the same file name would be uploaded to the same path. + - ./tests/test_helpers-0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt new file mode 100644 index 00000000000..b9f00092a5b --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt @@ -0,0 +1,12 @@ + +>>> errcode [CLI] bundle deploy +Error: Duplicate local library names: test_helpers-0.1-py3-none-any.whl + at resources.pipelines.orders.environment.dependencies[0] + resources.pipelines.orders.testing.environment.dependencies[0] + in databricks.yml:13:13 + databricks.yml:21:15 + +Local library names must be unique but found libraries with the same name: pipeline/test_helpers-0.1-py3-none-any.whl, tests/test_helpers-0.1-py3-none-any.whl + + +Exit code: 1 diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script new file mode 100644 index 00000000000..e4b9ddaaa91 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script @@ -0,0 +1 @@ +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml new file mode 100644 index 00000000000..3be7711de0f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml @@ -0,0 +1,2 @@ +# Runs only against the local test server. +Cloud = false diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py new file mode 100644 index 00000000000..7d4290a117a --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py @@ -0,0 +1 @@ +x = 1 diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 722d42c7f64..e4e1c62c614 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -22,6 +22,7 @@ var patterns = []dyn.Pattern{ clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), } type libData struct { From 8996c38ae0478954b3ea8b199191bd8f4c00e31b Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 19:08:48 +0000 Subject: [PATCH 3/6] Install a test dependency in the live pipelines test smoke test --- acceptance/pipelines/test/live/databricks.yml.tmpl | 3 +++ .../pipelines/test/live/helpers/order_helpers.py | 2 ++ acceptance/pipelines/test/live/helpers/pyproject.toml | 10 ++++++++++ acceptance/pipelines/test/live/output.txt | 2 +- acceptance/pipelines/test/live/tests/test_orders.py | 6 +++++- 5 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 acceptance/pipelines/test/live/helpers/order_helpers.py create mode 100644 acceptance/pipelines/test/live/helpers/pyproject.toml diff --git a/acceptance/pipelines/test/live/databricks.yml.tmpl b/acceptance/pipelines/test/live/databricks.yml.tmpl index 6dc3f0ec9de..d9bc81ca0ca 100644 --- a/acceptance/pipelines/test/live/databricks.yml.tmpl +++ b/acceptance/pipelines/test/live/databricks.yml.tmpl @@ -17,3 +17,6 @@ resources: sources: - glob: include: ./tests/** + environment: + dependencies: + - -e ./helpers diff --git a/acceptance/pipelines/test/live/helpers/order_helpers.py b/acceptance/pipelines/test/live/helpers/order_helpers.py new file mode 100644 index 00000000000..8c963d8c00f --- /dev/null +++ b/acceptance/pipelines/test/live/helpers/order_helpers.py @@ -0,0 +1,2 @@ +def order_total(amounts): + return sum(amounts) diff --git a/acceptance/pipelines/test/live/helpers/pyproject.toml b/acceptance/pipelines/test/live/helpers/pyproject.toml new file mode 100644 index 00000000000..8062e6015d7 --- /dev/null +++ b/acceptance/pipelines/test/live/helpers/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "order-helpers" +version = "0.1.0" + +[tool.setuptools] +py-modules = ["order_helpers"] diff --git a/acceptance/pipelines/test/live/output.txt b/acceptance/pipelines/test/live/output.txt index 69eeceb76fb..fe4f3bf3d1f 100644 --- a/acceptance/pipelines/test/live/output.txt +++ b/acceptance/pipelines/test/live/output.txt @@ -2,7 +2,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipelines-test-live-[UNIQUE_NAME]/default/files... Created pipelines.orders -Files: 4 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Run the tests and wait for the results diff --git a/acceptance/pipelines/test/live/tests/test_orders.py b/acceptance/pipelines/test/live/tests/test_orders.py index 527956ffe74..89d502a0f45 100644 --- a/acceptance/pipelines/test/live/tests/test_orders.py +++ b/acceptance/pipelines/test/live/tests/test_orders.py @@ -1,2 +1,6 @@ +# order_helpers is not on the import path; only its installation as a test dependency provides it. +from order_helpers import order_total + + def test_order_total(): - assert sum([1, 2, 3]) == 6 + assert order_total([1, 2, 3]) == 6 From f53140d924b141d2ffa6b5ca85386dbd7b036e36 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 2 Nov 2026 19:46:48 +0000 Subject: [PATCH 4/6] Link the pipelines test changelog entries to #7300 --- .nextchanges/bundles/editable-project-folder.md | 2 +- .nextchanges/bundles/pipelines-testing-plan.md | 2 +- .nextchanges/bundles/pipelines-testing.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.nextchanges/bundles/editable-project-folder.md b/.nextchanges/bundles/editable-project-folder.md index 23d74409d7a..315f643662b 100644 --- a/.nextchanges/bundles/editable-project-folder.md +++ b/.nextchanges/bundles/editable-project-folder.md @@ -1 +1 @@ -* Fix `-e ./folder` in environment dependencies, which failed before deploy. A local project folder after `-e`, with optional extras such as `-e ./project[test]`, now maps to its deployed folder. ([#7274](https://github.com/databricks/cli/pull/7274)) +* Fix `-e ./folder` in environment dependencies, which failed before deploy. A local project folder after `-e`, with optional extras such as `-e ./project[test]`, now maps to its deployed folder. ([#7300](https://github.com/databricks/cli/pull/7300)) diff --git a/.nextchanges/bundles/pipelines-testing-plan.md b/.nextchanges/bundles/pipelines-testing-plan.md index e109905d2bc..501c4a3752d 100644 --- a/.nextchanges/bundles/pipelines-testing-plan.md +++ b/.nextchanges/bundles/pipelines-testing-plan.md @@ -1 +1 @@ -* `bundle plan` now shows a change to a pipeline's `testing` that was made outside the bundle, and `bundle deploy` undoes it. ([#7274](https://github.com/databricks/cli/pull/7274)) +* `bundle plan` now shows a change to a pipeline's `testing` that was made outside the bundle, and `bundle deploy` undoes it. ([#7300](https://github.com/databricks/cli/pull/7300)) diff --git a/.nextchanges/bundles/pipelines-testing.md b/.nextchanges/bundles/pipelines-testing.md index 956e09a0b93..54b3280c82e 100644 --- a/.nextchanges/bundles/pipelines-testing.md +++ b/.nextchanges/bundles/pipelines-testing.md @@ -1 +1 @@ -* Add the `testing` property of pipelines, which sets the test sources and test-only dependencies that `databricks pipelines test` runs with. ([#7274](https://github.com/databricks/cli/pull/7274)) +* Add the `testing` property of pipelines, which sets the test sources and test-only dependencies that `databricks pipelines test` runs with. ([#7300](https://github.com/databricks/cli/pull/7300)) From 742ef870d6ec9ae0472e688a524cf444e1090cb0 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 3 Nov 2026 00:51:27 +0000 Subject: [PATCH 5/6] Gate the testing property of pipelines behind DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING --- .../environments/editable-folder/test.toml | 1 + .../pipelines/testing-dependencies/test.toml | 1 + .../pipelines/testing-disabled/databricks.yml | 11 +++++ .../pipelines/testing-disabled/out.test.toml | 2 + .../pipelines/testing-disabled/output.txt | 10 ++++ .../pipelines/testing-disabled/script | 1 + .../pipelines/testing-disabled/test.toml | 2 + .../testing-disabled/tests/test_orders.py | 2 + .../testing-same-name-libraries/test.toml | 1 + .../resources/pipelines/testing/test.toml | 1 + acceptance/pipelines/test/test.toml | 1 + bundle/config/mutator/translate_paths.go | 7 +-- bundle/config/mutator/translate_paths_test.go | 16 ++++-- .../validate/pipeline_testing_enabled.go | 49 +++++++++++++++++++ bundle/direct/dresources/pipeline.go | 8 +++ bundle/direct/dresources/pipeline_test.go | 10 ++++ bundle/env/experimental_pipeline_testing.go | 17 +++++++ bundle/phases/initialize.go | 1 + 18 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/testing-disabled/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-disabled/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-disabled/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-disabled/script create mode 100644 acceptance/bundle/resources/pipelines/testing-disabled/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-disabled/tests/test_orders.py create mode 100644 bundle/config/validate/pipeline_testing_enabled.go create mode 100644 bundle/env/experimental_pipeline_testing.go diff --git a/acceptance/bundle/environments/editable-folder/test.toml b/acceptance/bundle/environments/editable-folder/test.toml index 18b1a88417e..fdf8b9f0a35 100644 --- a/acceptance/bundle/environments/editable-folder/test.toml +++ b/acceptance/bundle/environments/editable-folder/test.toml @@ -1 +1,2 @@ Cloud = false +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml b/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml index 3be7711de0f..b6b581497c5 100644 --- a/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml @@ -1,2 +1,3 @@ # Runs only against the local test server. Cloud = false +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" diff --git a/acceptance/bundle/resources/pipelines/testing-disabled/databricks.yml b/acceptance/bundle/resources/pipelines/testing-disabled/databricks.yml new file mode 100644 index 00000000000..57c6246a47b --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-disabled/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: pipeline-testing-disabled + +resources: + pipelines: + orders: + name: orders + testing: + sources: + - glob: + include: ./tests/** diff --git a/acceptance/bundle/resources/pipelines/testing-disabled/out.test.toml b/acceptance/bundle/resources/pipelines/testing-disabled/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-disabled/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-disabled/output.txt b/acceptance/bundle/resources/pipelines/testing-disabled/output.txt new file mode 100644 index 00000000000..97796bc8bc8 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-disabled/output.txt @@ -0,0 +1,10 @@ + +>>> musterr [CLI] bundle validate +Error: the testing property of pipelines is experimental. Set DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true to enable it + at resources.pipelines.orders.testing + in databricks.yml:9:9 + +Name: pipeline-testing-disabled +Target: default + +Found 1 error diff --git a/acceptance/bundle/resources/pipelines/testing-disabled/script b/acceptance/bundle/resources/pipelines/testing-disabled/script new file mode 100644 index 00000000000..44dbe24759c --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-disabled/script @@ -0,0 +1 @@ +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/pipelines/testing-disabled/test.toml b/acceptance/bundle/resources/pipelines/testing-disabled/test.toml new file mode 100644 index 00000000000..4e81fd86218 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-disabled/test.toml @@ -0,0 +1,2 @@ +# Runs only against the local test server. The experimental gate is off. +Cloud = false diff --git a/acceptance/bundle/resources/pipelines/testing-disabled/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-disabled/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-disabled/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml index 3be7711de0f..b6b581497c5 100644 --- a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml @@ -1,2 +1,3 @@ # Runs only against the local test server. Cloud = false +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" diff --git a/acceptance/bundle/resources/pipelines/testing/test.toml b/acceptance/bundle/resources/pipelines/testing/test.toml index 45649fd8fbd..cb8f7d69d31 100644 --- a/acceptance/bundle/resources/pipelines/testing/test.toml +++ b/acceptance/bundle/resources/pipelines/testing/test.toml @@ -4,3 +4,4 @@ RecordRequests = true Ignore = [".databricks"] # The api get and put paths start with "/"; keep Git Bash on Windows from rewriting them. Env.MSYS_NO_PATHCONV = "1" +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" diff --git a/acceptance/pipelines/test/test.toml b/acceptance/pipelines/test/test.toml index b20092d3b88..775ae20aca9 100644 --- a/acceptance/pipelines/test/test.toml +++ b/acceptance/pipelines/test/test.toml @@ -3,3 +3,4 @@ Ignore = ['.databricks'] Env.MSYS_NO_PATHCONV = "1" # The command is experimental; the gate case in arguments turns it off. Env.DATABRICKS_CLI_PIPELINES_TEST = "true" +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index cdf9a95f902..a287567e9b7 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/notebook" @@ -250,10 +251,10 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local } // translatePipFlagPath translates the local path after a pip flag of an environment dependency, -// and keeps the flag in front of it. After -e, the path can also be a project folder, with -// optional extras such as ./project[test]. +// and keeps the flag in front of it. With pipeline testing enabled, the path after -e can also be +// a project folder, with optional extras such as ./project[test]. func (t *translateContext) translatePipFlagPath(ctx context.Context, flag, literal, localFullPath, localRelPath string) (string, error) { - if flag == "-e" { + if flag == "-e" && env.ExperimentalPipelineTesting(ctx) { folder, extras := localRelPath, "" if i := strings.LastIndex(localRelPath, "["); i > 0 && strings.HasSuffix(localRelPath, "]") { folder, extras = localRelPath[:i], localRelPath[i:] diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index b96d1ade278..a319e021b6d 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -8,6 +8,9 @@ import ( "runtime" "testing" + bundleenv "github.com/databricks/cli/bundle/env" + libsenv "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" @@ -821,7 +824,7 @@ func TestTranslatePathEditableProjectFolder(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "project", "pyproject.toml")) touchEmptyFile(t, filepath.Join(dir, "requirements.txt")) - translate := func(dependencies ...string) (*bundle.Bundle, diag.Diagnostics) { + translate := func(ctx context.Context, dependencies ...string) (*bundle.Bundle, diag.Diagnostics) { b := &bundle.Bundle{ SyncRootPath: dir, BundleRootPath: dir, @@ -842,10 +845,15 @@ func TestTranslatePathEditableProjectFolder(t *testing.T) { }, } bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}}) - return b, bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) + return b, bundle.ApplySeq(ctx, b, mutator.NormalizePaths(), mutator.TranslatePaths()) } - b, diags := translate("-e ./project", "-e ./project[test]", "-e .", "-e .[test]", "-r ./requirements.txt") + // Without pipeline testing, a folder after -e stays an error, as it was before the feature. + _, diags := translate(t.Context(), "-e ./project") + assert.ErrorContains(t, diags.Error(), "is a directory") + + ctx := libsenv.Set(t.Context(), bundleenv.ExperimentalPipelineTestingVariable, "true") + b, diags := translate(ctx, "-e ./project", "-e ./project[test]", "-e .", "-e .[test]", "-r ./requirements.txt") require.NoError(t, diags.Error()) assert.Equal(t, []string{ "-e /bundle/project", @@ -856,7 +864,7 @@ func TestTranslatePathEditableProjectFolder(t *testing.T) { }, b.Config.Resources.Pipelines["pipeline"].Environment.Dependencies) // -r takes a file, so a folder after it is still an error. - _, diags = translate("-r ./project") + _, diags = translate(ctx, "-r ./project") assert.ErrorContains(t, diags.Error(), "project") } diff --git a/bundle/config/validate/pipeline_testing_enabled.go b/bundle/config/validate/pipeline_testing_enabled.go new file mode 100644 index 00000000000..85fdb85ed4b --- /dev/null +++ b/bundle/config/validate/pipeline_testing_enabled.go @@ -0,0 +1,49 @@ +package validate + +import ( + "context" + "fmt" + "maps" + "slices" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type pipelineTestingEnabled struct{ bundle.RO } + +// PipelineTestingEnabled rejects the testing property of pipelines while the experimental feature is off. +func PipelineTestingEnabled() bundle.ReadOnlyMutator { + return &pipelineTestingEnabled{} +} + +func (v *pipelineTestingEnabled) Name() string { + return "validate:pipeline_testing_enabled" +} + +func (v *pipelineTestingEnabled) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if env.ExperimentalPipelineTesting(ctx) { + return nil + } + var diags diag.Diagnostics + pipelines := b.Config.Resources.Pipelines + for _, key := range slices.Sorted(maps.Keys(pipelines)) { + if pipelines[key] == nil || pipelines[key].Testing == nil { + continue + } + path := dyn.NewPath(dyn.Key("resources"), dyn.Key("pipelines"), dyn.Key(key), dyn.Key("testing")) + value, err := dyn.GetByPath(b.Config.Value(), path) + if err != nil { + return diag.FromErr(err) + } + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("the testing property of pipelines is experimental. Set %s=true to enable it", env.ExperimentalPipelineTestingVariable), + Locations: value.Locations(), + Paths: dyn.ToStructPaths(path), + }) + } + return diags +} diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 2f5517556bb..c6060f5ee63 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" @@ -120,6 +121,13 @@ type pipelineTestingResponse struct { // DoRead sends the same request as GetByPipelineId, but raw, so that it can also read spec.testing, // which GetPipelineResponse does not have yet. Switch back to GetByPipelineId once it does. func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemote, error) { + if !env.ExperimentalPipelineTesting(ctx) { + resp, err := r.client.Pipelines.GetByPipelineId(ctx, id) + if err != nil { + return nil, err + } + return makePipelineRemote(resp), nil + } var raw json.RawMessage if err := r.doRaw(ctx, http.MethodGet, "/api/2.0/pipelines/"+id, nil, &raw); err != nil { return nil, err diff --git a/bundle/direct/dresources/pipeline_test.go b/bundle/direct/dresources/pipeline_test.go index 418555d0578..34f41ad005d 100644 --- a/bundle/direct/dresources/pipeline_test.go +++ b/bundle/direct/dresources/pipeline_test.go @@ -4,6 +4,9 @@ import ( "reflect" "testing" + bundleenv "github.com/databricks/cli/bundle/env" + libsenv "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/auth" @@ -104,8 +107,15 @@ func TestPipelineReadReturnsTesting(t *testing.T) { require.NoError(t, err) r := (&ResourcePipeline{}).New(w) + // Without pipeline testing, the SDK reads the pipeline, as before the feature. remote, err := r.DoRead(t.Context(), "p1") require.NoError(t, err) + assert.Equal(t, "orders", remote.Name) + assert.Nil(t, remote.Testing) + + ctx := libsenv.Set(t.Context(), bundleenv.ExperimentalPipelineTestingVariable, "true") + remote, err = r.DoRead(ctx, "p1") + require.NoError(t, err) assert.Equal(t, "p1", remote.PipelineId) assert.Equal(t, "orders", remote.Name) assert.Equal(t, pipelines.PipelineStateIdle, remote.State) diff --git a/bundle/env/experimental_pipeline_testing.go b/bundle/env/experimental_pipeline_testing.go new file mode 100644 index 00000000000..b00a94176e9 --- /dev/null +++ b/bundle/env/experimental_pipeline_testing.go @@ -0,0 +1,17 @@ +package env + +import ( + "context" + + envlib "github.com/databricks/cli/libs/env" +) + +// ExperimentalPipelineTestingVariable enables the experimental testing property of pipelines. +const ExperimentalPipelineTestingVariable = "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING" + +// ExperimentalPipelineTesting reports whether the experimental testing property of pipelines is +// enabled. While it is off, bundles deploy pipelines exactly as they did before the property existed. +func ExperimentalPipelineTesting(ctx context.Context) bool { + enabled, _ := envlib.GetBool(ctx, ExperimentalPipelineTestingVariable) + return enabled +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 627d3eadb2d..29e57d58ca6 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -36,6 +36,7 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { validate.NoInterpolationInAuthConfig(), validate.NoInterpolationInBundleName(), validate.ValidateEngine(), + validate.PipelineTestingEnabled(), validate.Scripts(), mutator.ValidateSecretValueIsVariable(), From e038a13a69e2a2e4e4d83c2a5e730cdfdd72d156 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 3 Nov 2026 01:02:48 +0000 Subject: [PATCH 6/6] Keep the existing pipeline path rules and reads unchanged, adding the testing ones beside them --- .../mutator/paths/pipeline_paths_visitor.go | 69 +++++++++++++------ bundle/config/mutator/translate_paths.go | 39 ++++++----- bundle/direct/dresources/pipeline.go | 41 +++++------ 3 files changed, 88 insertions(+), 61 deletions(-) diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index 88978e93ae3..83be40afd7d 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -55,31 +55,56 @@ func pipelineRewritePatterns() []pipelineRewritePattern { } func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { - var patterns []pipelineRewritePattern - // Test dependencies take the same forms as the pipeline's environment dependencies. - for _, dependencies := range []dyn.Pattern{ - base.Append(dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), - base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), - } { - patterns = append(patterns, - pipelineRewritePattern{ - pattern: dependencies, - mode: TranslateModeLocalRelativeWithPrefix, - skipRewrite: func(s string) bool { - return !libraries.IsLibraryLocal(s) - }, + pipelineEnvironmentsPatterns := []pipelineRewritePattern{ + { + pattern: dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("pipelines"), + dyn.AnyKey(), + dyn.Key("environment"), + dyn.Key("dependencies"), + dyn.AnyIndex(), + ), + mode: TranslateModeLocalRelativeWithPrefix, + skipRewrite: func(s string) bool { + return !libraries.IsLibraryLocal(s) + }, + }, + // Test dependencies take the same forms as the pipeline's environment dependencies. + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + mode: TranslateModeLocalRelativeWithPrefix, + skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) }, + }, + } + + pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ + { + dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("pipelines"), + dyn.AnyKey(), + dyn.Key("environment"), + dyn.Key("dependencies"), + dyn.AnyIndex(), + ), + TranslateModeEnvironmentPipFlag, + func(s string) bool { + _, _, ok := libraries.IsLocalPathInPipFlag(s) + return !ok }, - pipelineRewritePattern{ - pattern: dependencies, - mode: TranslateModeEnvironmentPipFlag, - skipRewrite: func(s string) bool { - _, _, ok := libraries.IsLocalPathInPipFlag(s) - return !ok - }, + }, + { + base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + TranslateModeEnvironmentPipFlag, + func(s string) bool { + _, _, ok := libraries.IsLocalPathInPipFlag(s) + return !ok }, - ) + }, } - return patterns + + return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) } func VisitPipelinePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index a287567e9b7..1bfc64d7537 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -163,6 +163,15 @@ func (t *translateContext) rewritePath( localPath = filepath.ToSlash(localPath) localRelPath = filepath.ToSlash(localRelPath) + // With pipeline testing enabled, the path after -e can also be a project folder, with optional + // extras such as ./project[test]. + if opts.Mode == paths.TranslateModeEnvironmentPipFlag && flag == "-e" && env.ExperimentalPipelineTesting(ctx) { + if interp, ok := t.translateEditableFolder(localRelPath); ok { + t.seen[key] = interp + return interp, nil + } + } + // Convert local path into workspace path via specified function. var interp string switch opts.Mode { @@ -181,7 +190,9 @@ func (t *translateContext) rewritePath( case paths.TranslateModeLocalRelativeWithPrefix: interp, err = t.translateLocalRelativeWithPrefixPath(ctx, input, localPath, localRelPath) case paths.TranslateModeEnvironmentPipFlag: - interp, err = t.translatePipFlagPath(ctx, flag, input, localPath, localRelPath) + interp, err = t.translateFilePath(ctx, input, localPath, localRelPath) + // Add the flag prefix to the path to indicate it's a local file used in pip flags for environment dependencies. + interp = flag + " " + interp default: return "", fmt.Errorf("unsupported translate mode: %d", opts.Mode) } @@ -250,24 +261,18 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local return path.Join(t.remoteRoot, localRelPath), nil } -// translatePipFlagPath translates the local path after a pip flag of an environment dependency, -// and keeps the flag in front of it. With pipeline testing enabled, the path after -e can also be -// a project folder, with optional extras such as ./project[test]. -func (t *translateContext) translatePipFlagPath(ctx context.Context, flag, literal, localFullPath, localRelPath string) (string, error) { - if flag == "-e" && env.ExperimentalPipelineTesting(ctx) { - folder, extras := localRelPath, "" - if i := strings.LastIndex(localRelPath, "["); i > 0 && strings.HasSuffix(localRelPath, "]") { - folder, extras = localRelPath[:i], localRelPath[i:] - } - if info, err := t.b.SyncRoot.Stat(folder); err == nil && info.IsDir() { - return flag + " " + path.Join(t.remoteRoot, folder) + extras, nil - } +// translateEditableFolder maps a project folder after -e, with optional extras such as [test], to +// its deployed folder. It reports false if the path is not a folder in the sync root. +func (t *translateContext) translateEditableFolder(localRelPath string) (string, bool) { + folder, extras := localRelPath, "" + if i := strings.LastIndex(localRelPath, "["); i > 0 && strings.HasSuffix(localRelPath, "]") { + folder, extras = localRelPath[:i], localRelPath[i:] } - interp, err := t.translateFilePath(ctx, literal, localFullPath, localRelPath) - if err != nil { - return "", err + info, err := t.b.SyncRoot.Stat(folder) + if err != nil || !info.IsDir() { + return "", false } - return flag + " " + interp, nil + return "-e " + path.Join(t.remoteRoot, folder) + extras, true } func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index c6060f5ee63..3eacc3b5fa6 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -78,20 +78,12 @@ func (s PipelineRemote) MarshalJSON() ([]byte, error) { type ResourcePipeline struct { client *databricks.WorkspaceClient - - // apiClient sends the requests that the Go SDK cannot express yet because it lacks the testing - // field. It is built once, so that those requests share one rate limiter. - apiClient *client.DatabricksClient - apiClientErr error } -func (*ResourcePipeline) New(w *databricks.WorkspaceClient) *ResourcePipeline { - r := &ResourcePipeline{client: w, apiClient: nil, apiClientErr: nil} - // The adapter also builds a resource without a client to read its configuration. - if w != nil { - r.apiClient, r.apiClientErr = client.New(w.Config) +func (*ResourcePipeline) New(client *databricks.WorkspaceClient) *ResourcePipeline { + return &ResourcePipeline{ + client: client, } - return r } func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *PipelineState { @@ -118,16 +110,20 @@ type pipelineTestingResponse struct { } `json:"spec"` } -// DoRead sends the same request as GetByPipelineId, but raw, so that it can also read spec.testing, -// which GetPipelineResponse does not have yet. Switch back to GetByPipelineId once it does. func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemote, error) { - if !env.ExperimentalPipelineTesting(ctx) { - resp, err := r.client.Pipelines.GetByPipelineId(ctx, id) - if err != nil { - return nil, err - } - return makePipelineRemote(resp), nil + if env.ExperimentalPipelineTesting(ctx) { + return r.doReadWithTesting(ctx, id) } + resp, err := r.client.Pipelines.GetByPipelineId(ctx, id) + if err != nil { + return nil, err + } + return makePipelineRemote(resp), nil +} + +// doReadWithTesting sends the same request as GetByPipelineId, but raw, so that it can also read +// spec.testing, which GetPipelineResponse does not have yet. Remove it once GetPipelineResponse has it. +func (r *ResourcePipeline) doReadWithTesting(ctx context.Context, id string) (*PipelineRemote, error) { var raw json.RawMessage if err := r.doRaw(ctx, http.MethodGet, "/api/2.0/pipelines/"+id, nil, &raw); err != nil { return nil, err @@ -325,10 +321,11 @@ func (r *ResourcePipeline) doWithTesting(ctx context.Context, method, path strin // doRaw sends a request that the Go SDK cannot express yet because it lacks the testing field. func (r *ResourcePipeline) doRaw(ctx context.Context, method, path string, request, response any) error { - if r.apiClientErr != nil { - return fmt.Errorf("failed to create API client: %w", r.apiClientErr) + apiClient, err := client.New(r.client.Config) + if err != nil { + return fmt.Errorf("failed to create API client: %w", err) } - return r.apiClient.Do(ctx, method, path, auth.WorkspaceIDHeaders(r.client.Config), nil, request, response) + return apiClient.Do(ctx, method, path, auth.WorkspaceIDHeaders(r.client.Config), nil, request, response) } // Note, terraform provider either