diff --git a/.nextchanges/bundles/editable-project-folder.md b/.nextchanges/bundles/editable-project-folder.md new file mode 100644 index 00000000000..23d74409d7a --- /dev/null +++ b/.nextchanges/bundles/editable-project-folder.md @@ -0,0 +1 @@ +* Fix `-e ./folder` in environment dependencies, which failed before deploy. A local project folder after `-e`, with optional extras such as `-e ./project[test]`, now maps to its deployed folder. ([#7274](https://github.com/databricks/cli/pull/7274)) diff --git a/.nextchanges/bundles/pipelines-testing-plan.md b/.nextchanges/bundles/pipelines-testing-plan.md new file mode 100644 index 00000000000..e109905d2bc --- /dev/null +++ b/.nextchanges/bundles/pipelines-testing-plan.md @@ -0,0 +1 @@ +* `bundle plan` now shows a change to a pipeline's `testing` that was made outside the bundle, and `bundle deploy` undoes it. ([#7274](https://github.com/databricks/cli/pull/7274)) diff --git a/.nextchanges/bundles/pipelines-testing.md b/.nextchanges/bundles/pipelines-testing.md new file mode 100644 index 00000000000..956e09a0b93 --- /dev/null +++ b/.nextchanges/bundles/pipelines-testing.md @@ -0,0 +1 @@ +* Add the `testing` property of pipelines, which sets the test sources and test-only dependencies that `databricks pipelines test` runs with. ([#7274](https://github.com/databricks/cli/pull/7274)) diff --git a/.nextchanges/cli/pipelines-test-junit.md b/.nextchanges/cli/pipelines-test-junit.md new file mode 100644 index 00000000000..fff9ab2c679 --- /dev/null +++ b/.nextchanges/cli/pipelines-test-junit.md @@ -0,0 +1 @@ +* Add `--junit-xml` to `databricks pipelines test`, which writes the test results as a JUnit XML report. ([#7273](https://github.com/databricks/cli/pull/7273)) diff --git a/.nextchanges/cli/pipelines-test-results.md b/.nextchanges/cli/pipelines-test-results.md new file mode 100644 index 00000000000..72217847c99 --- /dev/null +++ b/.nextchanges/cli/pipelines-test-results.md @@ -0,0 +1 @@ +* `databricks pipelines test` waits for the test results, prints them, and fails when a test fails or the results are incomplete. ([#7272](https://github.com/databricks/cli/pull/7272)) diff --git a/.nextchanges/cli/pipelines-test.md b/.nextchanges/cli/pipelines-test.md new file mode 100644 index 00000000000..4d548dd6581 --- /dev/null +++ b/.nextchanges/cli/pipelines-test.md @@ -0,0 +1 @@ +* Add the `databricks pipelines test` command, which runs a pipeline's tests in a test-only update. ([#7271](https://github.com/databricks/cli/pull/7271)) diff --git a/acceptance/bundle/environments/editable-folder/databricks.yml b/acceptance/bundle/environments/editable-folder/databricks.yml new file mode 100644 index 00000000000..f16db8d0d95 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/databricks.yml @@ -0,0 +1,33 @@ +bundle: + name: editable-folder + +resources: + jobs: + my_job: + name: my job + tasks: + - task_key: main + python_wheel_task: + package_name: orders + entry_point: main + environment_key: default + environments: + - environment_key: default + spec: + client: "1" + dependencies: + - -e ./project + + pipelines: + my_pipeline: + name: my pipeline + environment: + dependencies: + - -e ./project + testing: + sources: + - glob: + include: /Workspace/Shared/orders/tests/** + environment: + dependencies: + - -e ./project[test] diff --git a/acceptance/bundle/environments/editable-folder/out.test.toml b/acceptance/bundle/environments/editable-folder/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/environments/editable-folder/output.txt b/acceptance/bundle/environments/editable-folder/output.txt new file mode 100644 index 00000000000..5c032cead61 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] bundle validate -o json +{ + "job": [ + "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project" + ], + "pipeline": [ + "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project" + ], + "testing": [ + "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project[test]" + ] +} diff --git a/acceptance/bundle/environments/editable-folder/project/pyproject.toml b/acceptance/bundle/environments/editable-folder/project/pyproject.toml new file mode 100644 index 00000000000..468fddd1ca7 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/project/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "orders" +version = "0.1.0" + +[project.optional-dependencies] +test = ["faker"] diff --git a/acceptance/bundle/environments/editable-folder/script b/acceptance/bundle/environments/editable-folder/script new file mode 100644 index 00000000000..24314e31c00 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/script @@ -0,0 +1 @@ +trace $CLI bundle validate -o json | jq '{job: .resources.jobs.my_job.environments[0].spec.dependencies, pipeline: .resources.pipelines.my_pipeline.environment.dependencies, testing: .resources.pipelines.my_pipeline.testing.environment.dependencies}' diff --git a/acceptance/bundle/environments/editable-folder/test.toml b/acceptance/bundle/environments/editable-folder/test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/bundle/refschema/output/pipelines.txt b/acceptance/bundle/refschema/output/pipelines.txt index c8bc57f512c..9c2b4e56b5c 100644 --- a/acceptance/bundle/refschema/output/pipelines.txt +++ b/acceptance/bundle/refschema/output/pipelines.txt @@ -867,6 +867,14 @@ resources.pipelines.*.storage string ALL resources.pipelines.*.tags map[string]string ALL resources.pipelines.*.tags.* string ALL resources.pipelines.*.target string ALL +resources.pipelines.*.testing *resources.PipelineTesting ALL +resources.pipelines.*.testing.environment *resources.PipelineTestEnvironment ALL +resources.pipelines.*.testing.environment.dependencies []string ALL +resources.pipelines.*.testing.environment.dependencies[*] string ALL +resources.pipelines.*.testing.sources []resources.PipelineTestSource ALL +resources.pipelines.*.testing.sources[*] resources.PipelineTestSource ALL +resources.pipelines.*.testing.sources[*].glob *pipelines.PathPattern ALL +resources.pipelines.*.testing.sources[*].glob.include string ALL resources.pipelines.*.trigger *pipelines.PipelineTrigger ALL resources.pipelines.*.trigger.cron *pipelines.CronTrigger ALL resources.pipelines.*.trigger.cron.quartz_cron_schedule string ALL diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/databricks.yml b/acceptance/bundle/resources/pipelines/testing-dependencies/databricks.yml new file mode 100644 index 00000000000..6bc337c57d3 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: pipeline-testing-dependencies + +resources: + pipelines: + orders: + name: orders + testing: + sources: + - glob: + include: /Workspace/Shared/orders/tests/** + environment: + dependencies: + # Accepted, as by the API. + - --editable /Workspace/Shared/orders[test] + - --requirement=/Workspace/Shared/orders/requirements.txt + - -r ./orders/requirements.txt + # Rejected, as by the API. + - faker==33.0 + - git+https://github.com/example/helpers + - /Workspace/Shared/orders/requirements.txt + - --requirement ./orders/requirements.txt + - /Workspace/Shared/orders -i https://example.com/simple diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/orders/requirements.txt b/acceptance/bundle/resources/pipelines/testing-dependencies/orders/requirements.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/out.test.toml b/acceptance/bundle/resources/pipelines/testing-dependencies/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/output.txt b/acceptance/bundle/resources/pipelines/testing-dependencies/output.txt new file mode 100644 index 00000000000..434913a1bca --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/output.txt @@ -0,0 +1,39 @@ + +>>> musterr [CLI] bundle validate +Error: testing.environment.dependencies must point at files, not "faker==33.0" + at resources.pipelines.orders.testing.environment.dependencies[3] + in databricks.yml:19:15 + +A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported. + +Error: testing.environment.dependencies must point at files, not "git+https://github.com/example/helpers" + at resources.pipelines.orders.testing.environment.dependencies[4] + in databricks.yml:20:15 + +A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported. + +Error: testing.environment.dependencies needs -r before the requirements file "/Workspace/Shared/orders/requirements.txt" + at resources.pipelines.orders.testing.environment.dependencies[5] + in databricks.yml:21:15 + +Without -r, pip treats the file as a package to install. For example: -r ./tests/requirements.txt. + +Error: testing.environment.dependencies must point at files, not "--requirement ./orders/requirements.txt" + at resources.pipelines.orders.testing.environment.dependencies[6] + in databricks.yml:22:15 + +A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported. + +Error: testing.environment.dependencies must point at files, not "/Workspace/Shared/orders -i https://example.com/simple" + at resources.pipelines.orders.testing.environment.dependencies[7] + in databricks.yml:23:15 + +A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported. + +Name: pipeline-testing-dependencies +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dependencies/default + +Found 5 errors diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/script b/acceptance/bundle/resources/pipelines/testing-dependencies/script new file mode 100644 index 00000000000..44dbe24759c --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/script @@ -0,0 +1 @@ +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml b/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml new file mode 100644 index 00000000000..3be7711de0f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml @@ -0,0 +1,2 @@ +# Runs only against the local test server. +Cloud = false diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml new file mode 100644 index 00000000000..381095e1cd3 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: pipeline-testing-same-name-libraries + +resources: + pipelines: + orders: + name: orders + libraries: + - file: + path: ./transformations/orders.py + environment: + dependencies: + - ./pipeline/test_helpers-0.1-py3-none-any.whl + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + # A different wheel with the same file name would be uploaded to the same path. + - ./tests/test_helpers-0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt new file mode 100644 index 00000000000..b9f00092a5b --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt @@ -0,0 +1,12 @@ + +>>> errcode [CLI] bundle deploy +Error: Duplicate local library names: test_helpers-0.1-py3-none-any.whl + at resources.pipelines.orders.environment.dependencies[0] + resources.pipelines.orders.testing.environment.dependencies[0] + in databricks.yml:13:13 + databricks.yml:21:15 + +Local library names must be unique but found libraries with the same name: pipeline/test_helpers-0.1-py3-none-any.whl, tests/test_helpers-0.1-py3-none-any.whl + + +Exit code: 1 diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script new file mode 100644 index 00000000000..e4b9ddaaa91 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script @@ -0,0 +1 @@ +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml new file mode 100644 index 00000000000..3be7711de0f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml @@ -0,0 +1,2 @@ +# Runs only against the local test server. +Cloud = false diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py new file mode 100644 index 00000000000..7d4290a117a --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py @@ -0,0 +1 @@ +x = 1 diff --git a/acceptance/bundle/resources/pipelines/testing/databricks.yml b/acceptance/bundle/resources/pipelines/testing/databricks.yml new file mode 100644 index 00000000000..a751a032169 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: pipeline-testing + +resources: + pipelines: + orders: + name: orders + libraries: + - glob: + include: ./transformations/** + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - --editable ${workspace.file_path}[test] + - -r ./tests/requirements.txt + - ./dist/*.whl diff --git a/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing/out.test.toml b/acceptance/bundle/resources/pipelines/testing/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt new file mode 100644 index 00000000000..e9296bd2e55 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -0,0 +1,215 @@ + +=== Relative test paths resolve to the deployed workspace paths +>>> [CLI] bundle validate -o json +{ + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "./dist/*.whl" + ] + }, + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ] +} + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Created pipelines.orders +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/transformations/**" + } + } + ], + "name": "orders", + "testing": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ], + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + } + } +} + +=== No changes after the deploy +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== A remote edit that drops testing shows in the plan +>>> [CLI] bundle plan -o json +{ + "action": "update", + "old": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ], + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + }, + "new": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ], + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + } +} + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Changing testing updates the pipeline +>>> update_file.py databricks.yml ./tests/** ./tests/unit/** + +>>> [CLI] bundle plan +update pipelines.orders + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[ORDERS_ID]", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/transformations/**" + } + } + ], + "name": "orders", + "testing": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/unit/**" + } + } + ], + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + } + } +} + +=== Removing testing sends an update without it +>>> [CLI] bundle plan +update pipelines.orders + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[ORDERS_ID]", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/transformations/**" + } + } + ], + "name": "orders" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.orders + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.orders + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script new file mode 100644 index 00000000000..c30595708fe --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -0,0 +1,34 @@ +title "Relative test paths resolve to the deployed workspace paths" +trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing + +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp + +title "No changes after the deploy" +trace $CLI bundle plan +rm "$OUT_REQUESTS" + +title "A remote edit that drops testing shows in the plan" +PIPELINE_ID=$(read_id.py orders) +$CLI api get "/api/2.0/pipelines/$PIPELINE_ID" | jq '.spec | del(.testing)' > edit.json +$CLI api put "/api/2.0/pipelines/$PIPELINE_ID" --json @edit.json &> LOG.edit +rm edit.json +trace $CLI bundle plan -o json | jq '.plan["resources.pipelines.orders"].changes.testing' +trace $CLI bundle deploy +trace $CLI bundle plan +rm "$OUT_REQUESTS" + +title "Changing testing updates the pipeline" +trace update_file.py databricks.yml './tests/**' './tests/unit/**' +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp + +title "Removing testing sends an update without it" +update_file.py databricks.yml "$(sed -n '/^ testing:/,$p' databricks.yml)" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp + +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/pipelines/testing/test.toml b/acceptance/bundle/resources/pipelines/testing/test.toml new file mode 100644 index 00000000000..45649fd8fbd --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/test.toml @@ -0,0 +1,6 @@ +# Runs only against the local test server. +Cloud = false +RecordRequests = true +Ignore = [".databricks"] +# The api get and put paths start with "/"; keep Git Bash on Windows from rewriting them. +Env.MSYS_NO_PATHCONV = "1" diff --git a/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt new file mode 100644 index 00000000000..9d7f6e827a1 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt @@ -0,0 +1 @@ +faker diff --git a/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py new file mode 100644 index 00000000000..3a22621c3bf --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py @@ -0,0 +1 @@ +from pyspark import pipelines as dp diff --git a/acceptance/pipelines/databricks-cli-help/output.txt b/acceptance/pipelines/databricks-cli-help/output.txt index 8514fa33b2b..adc729f4f5c 100644 --- a/acceptance/pipelines/databricks-cli-help/output.txt +++ b/acceptance/pipelines/databricks-cli-help/output.txt @@ -30,6 +30,7 @@ Main Commands run Run a pipeline show Preview a table's columns and sample rows stop Stop a pipeline + test *Preview* Run the tests of a pipeline Management Commands apply-environment *Public Preview* Apply the latest environment to the pipeline. diff --git a/acceptance/pipelines/test/arguments/out.flag-error.xml b/acceptance/pipelines/test/arguments/out.flag-error.xml new file mode 100644 index 00000000000..475a397f5fe --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.flag-error.xml @@ -0,0 +1,8 @@ + + + + + unsupported pytest argument "-x": only -k and -m are supported + + + diff --git a/acceptance/pipelines/test/arguments/out.no-wait.xml b/acceptance/pipelines/test/arguments/out.no-wait.xml new file mode 100644 index 00000000000..5f025801a52 --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.no-wait.xml @@ -0,0 +1,8 @@ + + + + + --junit-xml cannot be used with --no-wait because no results are available when the command returns + + + diff --git a/acceptance/pipelines/test/arguments/out.test.toml b/acceptance/pipelines/test/arguments/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt new file mode 100644 index 00000000000..127bcbebf57 --- /dev/null +++ b/acceptance/pipelines/test/arguments/output.txt @@ -0,0 +1,109 @@ + +>>> [CLI] pipelines test --help +Run the tests of a pipeline in a test-only update. + +PIPELINE_KEY is the unique name of the pipeline, as defined in its YAML file. +It is required when test selections are given. If there is only one pipeline +in the project and no arguments are given, it is selected automatically. +Outside a project, identify the pipeline with --pipeline-id; all positional +arguments are then test selections. + +TEST_SELECTION is a test directory, file, or pytest test ID, for example +tests/test_orders.py::test_backfill. In a project, a relative path is a local +path, relative to the working directory, and is converted to its deployed +workspace path. With --pipeline-id, a relative path is relative to the +pipeline's root_path. Paths that start with / are workspace paths. Without a +selection, all of the pipeline's tests run. + +Each --pytest-arg passes one argument to pytest, in the order given. Only -k +and -m are supported, each followed by its value in the next --pytest-arg, for +example: + + databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' + +The command waits for the results and succeeds only if at least one test was +collected and none failed or errored. --no-wait returns after the update starts. +--timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume +to read the results later. To stop the update, run 'pipelines stop'. + +--junit-xml writes one entry per test with its outcome and duration. An +incomplete run, a run without tests, or a timeout adds a failing entry that +names the cause. + +Usage: + databricks pipelines test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...] + +Pipeline Test Flags: + --junit-xml string Write the results as a JUnit XML report to this file, replacing it. + --no-wait Return after the test update starts, without waiting for results. + --pipeline-id string ID of the pipeline to test, instead of a PIPELINE_KEY. + --pytest-arg stringArray Argument to pass to pytest (repeatable). Only -k and -m are supported. + --resume string Read the results of an existing test update instead of starting one. + --timeout duration Stop waiting for results after this duration, for example 30m. The update keeps running. + +Flags: + -h, --help help for test + +Global Flags: + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in project config. Example: --var="foo=bar" + +=== pytest arguments must be passed with --pytest-arg +>>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders +Error: -k is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' + +>>> musterr [CLI] pipelines test --pipeline-id 1234 -msmoke +Error: -m is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' + +>>> musterr [CLI] pipelines test --pipeline-id 1234 -- -k orders +Error: the -- separator is not supported: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' + +=== Only -k and -m are supported +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x +Error: unsupported pytest argument "-x": only -k and -m are supported + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-k +Error: pytest argument -k requires a value: pass it in the next --pytest-arg + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=tests/test_orders.py +Error: unsupported pytest argument "tests/test_orders.py": pass test paths and test IDs as TEST_SELECTION arguments + +=== -k and -m take their value in the next --pytest-arg +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-korders +Error: pytest argument "-korders" must be split: pass --pytest-arg=-k --pytest-arg=orders + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-k --pytest-arg=-m +Error: pytest argument -k requires a value, not "-m": pass it in the next --pytest-arg + +=== Incompatible run-control flags +>>> musterr [CLI] pipelines test --pipeline-id 1234 tests --resume 5678 +Error: --resume cannot be used with test selections + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678 --pytest-arg=-m --pytest-arg=smoke +Error: --resume cannot be used with --pytest-arg + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678 --no-wait +Error: --resume cannot be used with --no-wait + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --timeout 10m +Error: --timeout cannot be used with --no-wait + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678' OR '1'='1 +Error: --resume "5678' OR '1'='1" is not an update ID; pass the update ID that pipelines test printed + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume +Error: --resume "" is not an update ID; pass the update ID that pipelines test printed + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml +Error: --junit-xml cannot be used with --no-wait because no results are available when the command returns + +=== An invalid flag replaces an older report +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml +Error: unsupported pytest argument "-x": only -k and -m are supported + +=== JSON output is not supported +>>> musterr [CLI] pipelines test --pipeline-id 1234 -o json +Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script new file mode 100644 index 00000000000..6b033ca4264 --- /dev/null +++ b/acceptance/pipelines/test/arguments/script @@ -0,0 +1,31 @@ +trace $CLI pipelines test --help + +title "pytest arguments must be passed with --pytest-arg" +trace musterr $CLI pipelines test --pipeline-id 1234 -k orders +trace musterr $CLI pipelines test --pipeline-id 1234 -msmoke +trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders + +title "Only -k and -m are supported" +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-k +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=tests/test_orders.py + +title "-k and -m take their value in the next --pytest-arg" +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-korders +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-k --pytest-arg=-m + +title "Incompatible run-control flags" +trace musterr $CLI pipelines test --pipeline-id 1234 tests --resume 5678 +trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --pytest-arg=-m --pytest-arg=smoke +trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --no-wait +trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --timeout 10m +trace musterr $CLI pipelines test --pipeline-id 1234 --resume "5678' OR '1'='1" +trace musterr $CLI pipelines test --pipeline-id 1234 --resume "" +trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml + +title "An invalid flag replaces an older report" +echo '' > out.flag-error.xml +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml + +title "JSON output is not supported" +trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/acceptance/pipelines/test/immutable/databricks.yml b/acceptance/pipelines/test/immutable/databricks.yml new file mode 100644 index 00000000000..483234d0dd4 --- /dev/null +++ b/acceptance/pipelines/test/immutable/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-pipeline-test-immutable + +workspace: + sync_mode: immutable + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-immutable + libraries: + - file: + path: pipeline_file.py diff --git a/acceptance/pipelines/test/immutable/out.test.toml b/acceptance/pipelines/test/immutable/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/immutable/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/immutable/output.txt b/acceptance/pipelines/test/immutable/output.txt new file mode 100644 index 00000000000..855d885c12e --- /dev/null +++ b/acceptance/pipelines/test/immutable/output.txt @@ -0,0 +1,25 @@ + +>>> [CLI] pipelines deploy +Created internal_immutable_snapshots.immutable +Created pipelines.my_pipeline +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== Local test paths are not supported with workspace.sync_mode immutable +>>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py::test_backfill +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot resolve test selection "tests/test_orders.py::test_backfill": local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path + +=== Absolute workspace paths and running all tests still work +>>> [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: The pipeline has no test sources. Set `testing.sources` to the test files or folders to run. + +Exit code: 1 diff --git a/acceptance/pipelines/test/immutable/pipeline_file.py b/acceptance/pipelines/test/immutable/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/immutable/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/immutable/script b/acceptance/pipelines/test/immutable/script new file mode 100644 index 00000000000..a7bca1bfb07 --- /dev/null +++ b/acceptance/pipelines/test/immutable/script @@ -0,0 +1,12 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "Local test paths are not supported with workspace.sync_mode immutable" +trace musterr $CLI pipelines test my_pipeline tests/test_orders.py::test_backfill + +title "Absolute workspace paths and running all tests still work" +trace $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait +# Like the service, the test server rejects a new update while one is active. +$CLI pipelines stop my_pipeline &>> LOG.stop +trace $CLI pipelines test my_pipeline --no-wait +trace print_requests.py //updates diff --git a/acceptance/pipelines/test/immutable/tests/test_orders.py b/acceptance/pipelines/test/immutable/tests/test_orders.py new file mode 100644 index 00000000000..ec31eaa1b01 --- /dev/null +++ b/acceptance/pipelines/test/immutable/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_backfill(): + assert True diff --git a/acceptance/pipelines/test/live/databricks.yml.tmpl b/acceptance/pipelines/test/live/databricks.yml.tmpl new file mode 100644 index 00000000000..d9bc81ca0ca --- /dev/null +++ b/acceptance/pipelines/test/live/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: test-pipelines-test-live-$UNIQUE_NAME + +resources: + pipelines: + orders: + # The Pipelines API rejects duplicate names, so the name is unique per run. + name: test-pipelines-test-live-$UNIQUE_NAME + catalog: main + schema: default + serverless: true + root_path: . + libraries: + - file: + path: ./transformations/orders.py + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - -e ./helpers diff --git a/acceptance/pipelines/test/live/helpers/order_helpers.py b/acceptance/pipelines/test/live/helpers/order_helpers.py new file mode 100644 index 00000000000..8c963d8c00f --- /dev/null +++ b/acceptance/pipelines/test/live/helpers/order_helpers.py @@ -0,0 +1,2 @@ +def order_total(amounts): + return sum(amounts) diff --git a/acceptance/pipelines/test/live/helpers/pyproject.toml b/acceptance/pipelines/test/live/helpers/pyproject.toml new file mode 100644 index 00000000000..8062e6015d7 --- /dev/null +++ b/acceptance/pipelines/test/live/helpers/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "order-helpers" +version = "0.1.0" + +[tool.setuptools] +py-modules = ["order_helpers"] diff --git a/acceptance/pipelines/test/live/out.test.toml b/acceptance/pipelines/test/live/out.test.toml new file mode 100644 index 00000000000..314df1c2386 --- /dev/null +++ b/acceptance/pipelines/test/live/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +CloudSlow = true diff --git a/acceptance/pipelines/test/live/output.txt b/acceptance/pipelines/test/live/output.txt new file mode 100644 index 00000000000..fe4f3bf3d1f --- /dev/null +++ b/acceptance/pipelines/test/live/output.txt @@ -0,0 +1,22 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipelines-test-live-[UNIQUE_NAME]/default/files... +Created pipelines.orders +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Run the tests and wait for the results +1 test: 1 passed, 0 failed, 0 skipped, 0 errors +>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.orders + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.orders + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipelines-test-live-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/pipelines/test/live/script b/acceptance/pipelines/test/live/script new file mode 100644 index 00000000000..4f2b883cd60 --- /dev/null +++ b/acceptance/pipelines/test/live/script @@ -0,0 +1,20 @@ +if [ -n "$CLOUD_ENV" ] && [ -z "${TEST_PIPELINES_TESTING_ENABLED:-}" ]; then + echo "SKIP_TEST set TEST_PIPELINES_TESTING_ENABLED for a workspace with pipeline testing enabled" + exit 0 +fi + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Run the tests and wait for the results\n" +# Durations differ between runs, so only the summary line and the report are compared. The script +# stops if the command fails. +$CLI pipelines test --junit-xml junit.xml &> LOG.test +grep -E '^[0-9]+ tests?:' LOG.test +grep -o ']*name="[^"]*"' junit.xml diff --git a/acceptance/pipelines/test/live/test.toml b/acceptance/pipelines/test/live/test.toml new file mode 100644 index 00000000000..8a1d4b5164c --- /dev/null +++ b/acceptance/pipelines/test/live/test.toml @@ -0,0 +1,6 @@ +# Also runs against a real workspace, when that workspace has pipeline testing enabled. +# The script skips the cloud run unless TEST_PIPELINES_TESTING_ENABLED is set. +Cloud = true +CloudSlow = true +RecordRequests = false +Ignore = [".databricks", "databricks.yml", "junit.xml"] diff --git a/acceptance/pipelines/test/live/tests/test_orders.py b/acceptance/pipelines/test/live/tests/test_orders.py new file mode 100644 index 00000000000..89d502a0f45 --- /dev/null +++ b/acceptance/pipelines/test/live/tests/test_orders.py @@ -0,0 +1,6 @@ +# order_helpers is not on the import path; only its installation as a test dependency provides it. +from order_helpers import order_total + + +def test_order_total(): + assert order_total([1, 2, 3]) == 6 diff --git a/acceptance/pipelines/test/live/transformations/orders.py b/acceptance/pipelines/test/live/transformations/orders.py new file mode 100644 index 00000000000..c0ec524e2ee --- /dev/null +++ b/acceptance/pipelines/test/live/transformations/orders.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.materialized_view +def orders(): + return spark.range(3) diff --git a/acceptance/pipelines/test/many-cases/databricks.yml b/acceptance/pipelines/test/many-cases/databricks.yml new file mode 100644 index 00000000000..b9a6dd0f214 --- /dev/null +++ b/acceptance/pipelines/test/many-cases/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: test-pipeline-test-many-cases + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-many-cases + libraries: + - file: + path: pipeline_file.py + testing: + sources: + - glob: + include: ./tests/** diff --git a/acceptance/pipelines/test/many-cases/out.test.toml b/acceptance/pipelines/test/many-cases/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/many-cases/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/many-cases/output.txt b/acceptance/pipelines/test/many-cases/output.txt new file mode 100644 index 00000000000..3171229e84b --- /dev/null +++ b/acceptance/pipelines/test/many-cases/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-many-cases/default/files... +Created pipelines.my_pipeline +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== Every case is counted once across pages, polls, and the final read +>>> tail -n 1 LOG.many +200 tests: 200 passed, 0 failed, 0 skipped, 0 errors +Event pages read: 8 +Pages read with page_token: 4 diff --git a/acceptance/pipelines/test/many-cases/pipeline_file.py b/acceptance/pipelines/test/many-cases/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/many-cases/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/many-cases/script b/acceptance/pipelines/test/many-cases/script new file mode 100644 index 00000000000..ddd3e37328f --- /dev/null +++ b/acceptance/pipelines/test/many-cases/script @@ -0,0 +1,9 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "Every case is counted once across pages, polls, and the final read" +$CLI pipelines test my_pipeline &> LOG.many +trace tail -n 1 LOG.many +print_requests.py --get //events --oneline > LOG.events +echo "Event pages read: $(wc -l < LOG.events | tr -d ' ')" +echo "Pages read with page_token: $(grep -c page_token LOG.events)" diff --git a/acceptance/pipelines/test/many-cases/tests/test_many.py b/acceptance/pipelines/test/many-cases/tests/test_many.py new file mode 100644 index 00000000000..c4a7d594fc8 --- /dev/null +++ b/acceptance/pipelines/test/many-cases/tests/test_many.py @@ -0,0 +1,799 @@ +# 200 cases emit 601 events, more than two pages of List Pipeline Events. +def test_case_000(): + assert True + + +def test_case_001(): + assert True + + +def test_case_002(): + assert True + + +def test_case_003(): + assert True + + +def test_case_004(): + assert True + + +def test_case_005(): + assert True + + +def test_case_006(): + assert True + + +def test_case_007(): + assert True + + +def test_case_008(): + assert True + + +def test_case_009(): + assert True + + +def test_case_010(): + assert True + + +def test_case_011(): + assert True + + +def test_case_012(): + assert True + + +def test_case_013(): + assert True + + +def test_case_014(): + assert True + + +def test_case_015(): + assert True + + +def test_case_016(): + assert True + + +def test_case_017(): + assert True + + +def test_case_018(): + assert True + + +def test_case_019(): + assert True + + +def test_case_020(): + assert True + + +def test_case_021(): + assert True + + +def test_case_022(): + assert True + + +def test_case_023(): + assert True + + +def test_case_024(): + assert True + + +def test_case_025(): + assert True + + +def test_case_026(): + assert True + + +def test_case_027(): + assert True + + +def test_case_028(): + assert True + + +def test_case_029(): + assert True + + +def test_case_030(): + assert True + + +def test_case_031(): + assert True + + +def test_case_032(): + assert True + + +def test_case_033(): + assert True + + +def test_case_034(): + assert True + + +def test_case_035(): + assert True + + +def test_case_036(): + assert True + + +def test_case_037(): + assert True + + +def test_case_038(): + assert True + + +def test_case_039(): + assert True + + +def test_case_040(): + assert True + + +def test_case_041(): + assert True + + +def test_case_042(): + assert True + + +def test_case_043(): + assert True + + +def test_case_044(): + assert True + + +def test_case_045(): + assert True + + +def test_case_046(): + assert True + + +def test_case_047(): + assert True + + +def test_case_048(): + assert True + + +def test_case_049(): + assert True + + +def test_case_050(): + assert True + + +def test_case_051(): + assert True + + +def test_case_052(): + assert True + + +def test_case_053(): + assert True + + +def test_case_054(): + assert True + + +def test_case_055(): + assert True + + +def test_case_056(): + assert True + + +def test_case_057(): + assert True + + +def test_case_058(): + assert True + + +def test_case_059(): + assert True + + +def test_case_060(): + assert True + + +def test_case_061(): + assert True + + +def test_case_062(): + assert True + + +def test_case_063(): + assert True + + +def test_case_064(): + assert True + + +def test_case_065(): + assert True + + +def test_case_066(): + assert True + + +def test_case_067(): + assert True + + +def test_case_068(): + assert True + + +def test_case_069(): + assert True + + +def test_case_070(): + assert True + + +def test_case_071(): + assert True + + +def test_case_072(): + assert True + + +def test_case_073(): + assert True + + +def test_case_074(): + assert True + + +def test_case_075(): + assert True + + +def test_case_076(): + assert True + + +def test_case_077(): + assert True + + +def test_case_078(): + assert True + + +def test_case_079(): + assert True + + +def test_case_080(): + assert True + + +def test_case_081(): + assert True + + +def test_case_082(): + assert True + + +def test_case_083(): + assert True + + +def test_case_084(): + assert True + + +def test_case_085(): + assert True + + +def test_case_086(): + assert True + + +def test_case_087(): + assert True + + +def test_case_088(): + assert True + + +def test_case_089(): + assert True + + +def test_case_090(): + assert True + + +def test_case_091(): + assert True + + +def test_case_092(): + assert True + + +def test_case_093(): + assert True + + +def test_case_094(): + assert True + + +def test_case_095(): + assert True + + +def test_case_096(): + assert True + + +def test_case_097(): + assert True + + +def test_case_098(): + assert True + + +def test_case_099(): + assert True + + +def test_case_100(): + assert True + + +def test_case_101(): + assert True + + +def test_case_102(): + assert True + + +def test_case_103(): + assert True + + +def test_case_104(): + assert True + + +def test_case_105(): + assert True + + +def test_case_106(): + assert True + + +def test_case_107(): + assert True + + +def test_case_108(): + assert True + + +def test_case_109(): + assert True + + +def test_case_110(): + assert True + + +def test_case_111(): + assert True + + +def test_case_112(): + assert True + + +def test_case_113(): + assert True + + +def test_case_114(): + assert True + + +def test_case_115(): + assert True + + +def test_case_116(): + assert True + + +def test_case_117(): + assert True + + +def test_case_118(): + assert True + + +def test_case_119(): + assert True + + +def test_case_120(): + assert True + + +def test_case_121(): + assert True + + +def test_case_122(): + assert True + + +def test_case_123(): + assert True + + +def test_case_124(): + assert True + + +def test_case_125(): + assert True + + +def test_case_126(): + assert True + + +def test_case_127(): + assert True + + +def test_case_128(): + assert True + + +def test_case_129(): + assert True + + +def test_case_130(): + assert True + + +def test_case_131(): + assert True + + +def test_case_132(): + assert True + + +def test_case_133(): + assert True + + +def test_case_134(): + assert True + + +def test_case_135(): + assert True + + +def test_case_136(): + assert True + + +def test_case_137(): + assert True + + +def test_case_138(): + assert True + + +def test_case_139(): + assert True + + +def test_case_140(): + assert True + + +def test_case_141(): + assert True + + +def test_case_142(): + assert True + + +def test_case_143(): + assert True + + +def test_case_144(): + assert True + + +def test_case_145(): + assert True + + +def test_case_146(): + assert True + + +def test_case_147(): + assert True + + +def test_case_148(): + assert True + + +def test_case_149(): + assert True + + +def test_case_150(): + assert True + + +def test_case_151(): + assert True + + +def test_case_152(): + assert True + + +def test_case_153(): + assert True + + +def test_case_154(): + assert True + + +def test_case_155(): + assert True + + +def test_case_156(): + assert True + + +def test_case_157(): + assert True + + +def test_case_158(): + assert True + + +def test_case_159(): + assert True + + +def test_case_160(): + assert True + + +def test_case_161(): + assert True + + +def test_case_162(): + assert True + + +def test_case_163(): + assert True + + +def test_case_164(): + assert True + + +def test_case_165(): + assert True + + +def test_case_166(): + assert True + + +def test_case_167(): + assert True + + +def test_case_168(): + assert True + + +def test_case_169(): + assert True + + +def test_case_170(): + assert True + + +def test_case_171(): + assert True + + +def test_case_172(): + assert True + + +def test_case_173(): + assert True + + +def test_case_174(): + assert True + + +def test_case_175(): + assert True + + +def test_case_176(): + assert True + + +def test_case_177(): + assert True + + +def test_case_178(): + assert True + + +def test_case_179(): + assert True + + +def test_case_180(): + assert True + + +def test_case_181(): + assert True + + +def test_case_182(): + assert True + + +def test_case_183(): + assert True + + +def test_case_184(): + assert True + + +def test_case_185(): + assert True + + +def test_case_186(): + assert True + + +def test_case_187(): + assert True + + +def test_case_188(): + assert True + + +def test_case_189(): + assert True + + +def test_case_190(): + assert True + + +def test_case_191(): + assert True + + +def test_case_192(): + assert True + + +def test_case_193(): + assert True + + +def test_case_194(): + assert True + + +def test_case_195(): + assert True + + +def test_case_196(): + assert True + + +def test_case_197(): + assert True + + +def test_case_198(): + assert True + + +def test_case_199(): + assert True diff --git a/acceptance/pipelines/test/pipeline-id/orders/tests/test_orders.py b/acceptance/pipelines/test/pipeline-id/orders/tests/test_orders.py new file mode 100644 index 00000000000..60ebe0db44c --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/orders/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_valid_order(): + pass diff --git a/acceptance/pipelines/test/pipeline-id/out.test.toml b/acceptance/pipelines/test/pipeline-id/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt new file mode 100644 index 00000000000..21ddaafb62a --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -0,0 +1,51 @@ + +=== Relative test selections resolve against the pipeline root_path +>>> [CLI] pipelines test --pipeline-id [UUID] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/orders/tests --no-wait +Pipeline: [UUID] +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill", + "/Workspace/Users/someone@example.com/orders/tests" + ] + } + } +} + +=== Test case IDs are shown relative to root_path +>>> [CLI] pipelines test --pipeline-id [UUID] tests +Pipeline: [UUID] +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_orders.py::test_valid_order PASSED (1.00s) + +1 test: 1 passed, 0 failed, 0 skipped, 0 errors + +=== Without a root_path, test selections must be workspace paths, and test case IDs are shown in full +>>> musterr [CLI] pipelines test --pipeline-id [UUID] tests/test_orders.py +Error: cannot resolve test selection "tests/test_orders.py": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests + +>>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Users/someone@example.com/orders/tests/test_orders.py +Pipeline: [UUID] +Update: [UUID] +Waiting for compute... +Running tests... +/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order PASSED (1.00s) + +1 test: 1 passed, 0 failed, 0 skipped, 0 errors + +=== Without test sources, the service rejects the run +>>> musterr [CLI] pipelines test --pipeline-id [UUID] +Error: The pipeline has no test sources. Set `testing.sources` to the test files or folders to run. + +=== Outside a project, --pipeline-id is required +>>> musterr [CLI] pipelines test tests +Error: outside a project, identify the pipeline with --pipeline-id diff --git a/acceptance/pipelines/test/pipeline-id/script b/acceptance/pipelines/test/pipeline-id/script new file mode 100644 index 00000000000..74e87a82e9a --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/script @@ -0,0 +1,29 @@ +# testing is not in the Go SDK yet, so "pipelines create" would drop it. +orders_tests='{"sources": [{"glob": {"include": "/Workspace/Users/someone@example.com/orders/tests/**"}}]}' +pipeline_id=$($CLI api post /api/2.0/pipelines --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders", "testing": '"$orders_tests"'}' | jq -r .pipeline_id) +no_root_pipeline_id=$($CLI api post /api/2.0/pipelines --json '{"name": "no-root", "testing": '"$orders_tests"'}' | jq -r .pipeline_id) +no_tests_pipeline_id=$($CLI api post /api/2.0/pipelines --json '{"name": "no-tests", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) +$CLI workspace import-dir orders /Workspace/Users/someone@example.com/orders &> LOG.import +rm "$OUT_REQUESTS" + +title "Relative test selections resolve against the pipeline root_path" +trace $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/orders/tests --no-wait +trace print_requests.py //updates +# Like the service, the test server rejects a new update while one is active. +$CLI api post /api/2.0/pipelines/$pipeline_id/stop &>> LOG.stop +rm "$OUT_REQUESTS" + +title "Test case IDs are shown relative to root_path" +trace $CLI pipelines test --pipeline-id $pipeline_id tests +rm "$OUT_REQUESTS" + +title "Without a root_path, test selections must be workspace paths, and test case IDs are shown in full" +trace musterr $CLI pipelines test --pipeline-id $no_root_pipeline_id tests/test_orders.py +trace $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/orders/tests/test_orders.py +rm "$OUT_REQUESTS" + +title "Without test sources, the service rejects the run" +trace musterr $CLI pipelines test --pipeline-id $no_tests_pipeline_id + +title "Outside a project, --pipeline-id is required" +trace musterr $CLI pipelines test tests diff --git a/acceptance/pipelines/test/results/databricks.yml b/acceptance/pipelines/test/results/databricks.yml new file mode 100644 index 00000000000..4bfcd4fa46e --- /dev/null +++ b/acceptance/pipelines/test/results/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-pipeline-test-results + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-results + root_path: . + libraries: + - file: + path: pipeline_file.py + testing: + sources: + - glob: + include: ./tests/** diff --git a/acceptance/pipelines/test/results/out.failures.xml b/acceptance/pipelines/test/results/out.failures.xml new file mode 100644 index 00000000000..5bf009266fd --- /dev/null +++ b/acceptance/pipelines/test/results/out.failures.xml @@ -0,0 +1,15 @@ + + + + + + AssertionError: assert 100 == 150 + + + + + + RuntimeError: fixture 'orders' setup failed + + + diff --git a/acceptance/pipelines/test/results/out.incomplete.xml b/acceptance/pipelines/test/results/out.incomplete.xml new file mode 100644 index 00000000000..c6e324f0f6b --- /dev/null +++ b/acceptance/pipelines/test/results/out.incomplete.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + RUN INCOMPLETE: the update failed before all test results were received. update [UUID] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly. + + + diff --git a/acceptance/pipelines/test/results/out.late-failure.xml b/acceptance/pipelines/test/results/out.late-failure.xml new file mode 100644 index 00000000000..5e9a004942b --- /dev/null +++ b/acceptance/pipelines/test/results/out.late-failure.xml @@ -0,0 +1,12 @@ + + + + + AssertionError: assert 100 == 150 + + + + update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. + + + diff --git a/acceptance/pipelines/test/results/out.no-tests.xml b/acceptance/pipelines/test/results/out.no-tests.xml new file mode 100644 index 00000000000..68f35e0ab2c --- /dev/null +++ b/acceptance/pipelines/test/results/out.no-tests.xml @@ -0,0 +1,8 @@ + + + + + No tests were collected. + + + diff --git a/acceptance/pipelines/test/results/out.passed.xml b/acceptance/pipelines/test/results/out.passed.xml new file mode 100644 index 00000000000..95d1e6f3d0a --- /dev/null +++ b/acceptance/pipelines/test/results/out.passed.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/acceptance/pipelines/test/results/out.test.toml b/acceptance/pipelines/test/results/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/results/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/results/out.update-failed.xml b/acceptance/pipelines/test/results/out.update-failed.xml new file mode 100644 index 00000000000..a115d599dc0 --- /dev/null +++ b/acceptance/pipelines/test/results/out.update-failed.xml @@ -0,0 +1,9 @@ + + + + + + update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. + + + diff --git a/acceptance/pipelines/test/results/output.txt b/acceptance/pipelines/test/results/output.txt new file mode 100644 index 00000000000..d0b3b88be8a --- /dev/null +++ b/acceptance/pipelines/test/results/output.txt @@ -0,0 +1,152 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-results/default/files... +Created pipelines.my_pipeline +Files: 7 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== All selected tests pass +>>> [CLI] pipelines test my_pipeline tests/test_orders.py::test_valid_order --junit-xml out.passed.xml +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_orders.py::test_valid_order PASSED (1.00s) + +1 test: 1 passed, 0 failed, 0 skipped, 0 errors +JUnit report: out.passed.xml + +>>> print_requests.py --get //events +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID]/events", + "q": { + "filter": "update_id='[UUID]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID]/events", + "q": { + "filter": "update_id='[UUID]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID]/events", + "q": { + "filter": "update_id='[UUID]' AND timestamp >= '[TIMESTAMP]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID]/events", + "q": { + "filter": "update_id='[UUID]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} + +=== Failed, errored, and skipped tests +>>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py --junit-xml out.failures.xml +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_orders.py::test_valid_order PASSED (1.00s) +tests/test_orders.py::test_backfill_fails FAILED (1.00s) +tests/test_orders.py::test_legacy_format_skipped SKIPPED (1.00s) +tests/test_orders.py::test_fixture_error ERROR (1.00s) + +FAILURES +tests/test_orders.py::test_backfill_fails + AssertionError: assert 100 == 150 +tests/test_orders.py::test_fixture_error + RuntimeError: fixture 'orders' setup failed + +4 tests: 1 passed, 1 failed, 1 skipped, 1 error +JUnit report: out.failures.xml + +=== No tests collected +>>> musterr [CLI] pipelines test my_pipeline tests/helpers.py --junit-xml out.no-tests.xml +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... + +No tests were collected. +JUnit report: out.no-tests.xml + +=== The update fails after all tests passed +>>> musterr [CLI] pipelines test my_pipeline tests/test_delivery.py --junit-xml out.update-failed.xml +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_delivery.py::test_breaks_update_after_summary PASSED (1.00s) + +1 test: 1 passed, 0 failed, 0 skipped, 0 errors +JUnit report: out.update-failed.xml +Error: update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. + +=== The update fails with an error of its own after a test failed +>>> musterr [CLI] pipelines test my_pipeline tests/test_late_failure.py --junit-xml out.late-failure.xml +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_late_failure.py::test_backfill_fails FAILED (1.00s) +tests/test_late_failure.py::test_breaks_update_after_summary PASSED (1.00s) + +FAILURES +tests/test_late_failure.py::test_backfill_fails + AssertionError: assert 100 == 150 + +2 tests: 1 passed, 1 failed, 0 skipped, 0 errors +JUnit report: out.late-failure.xml +Error: update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. + +=== Compute terminates before all results are received +>>> musterr [CLI] pipelines test my_pipeline tests/test_compute.py --junit-xml out.incomplete.xml +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_compute.py::test_small_order PASSED (1.00s) + +RUN INCOMPLETE: the update failed before all test results were received. +Results received: 1 passed. +JUnit report: out.incomplete.xml +Error: update [UUID] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly. diff --git a/acceptance/pipelines/test/results/pipeline_file.py b/acceptance/pipelines/test/results/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/results/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/results/script b/acceptance/pipelines/test/results/script new file mode 100644 index 00000000000..56779522991 --- /dev/null +++ b/acceptance/pipelines/test/results/script @@ -0,0 +1,22 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "All selected tests pass" +trace $CLI pipelines test my_pipeline tests/test_orders.py::test_valid_order --junit-xml out.passed.xml +trace print_requests.py --get //events + +title "Failed, errored, and skipped tests" +trace musterr $CLI pipelines test my_pipeline tests/test_orders.py --junit-xml out.failures.xml + +title "No tests collected" +trace musterr $CLI pipelines test my_pipeline tests/helpers.py --junit-xml out.no-tests.xml + +title "The update fails after all tests passed" +trace musterr $CLI pipelines test my_pipeline tests/test_delivery.py --junit-xml out.update-failed.xml + +title "The update fails with an error of its own after a test failed" +trace musterr $CLI pipelines test my_pipeline tests/test_late_failure.py --junit-xml out.late-failure.xml + +title "Compute terminates before all results are received" +trace musterr $CLI pipelines test my_pipeline tests/test_compute.py --junit-xml out.incomplete.xml +rm "$OUT_REQUESTS" diff --git a/acceptance/pipelines/test/results/tests/helpers.py b/acceptance/pipelines/test/results/tests/helpers.py new file mode 100644 index 00000000000..5205aa61063 --- /dev/null +++ b/acceptance/pipelines/test/results/tests/helpers.py @@ -0,0 +1,2 @@ +def make_order(): + return {"id": 1} diff --git a/acceptance/pipelines/test/results/tests/test_compute.py b/acceptance/pipelines/test/results/tests/test_compute.py new file mode 100644 index 00000000000..32954a03b8e --- /dev/null +++ b/acceptance/pipelines/test/results/tests/test_compute.py @@ -0,0 +1,11 @@ +# The test server terminates compute when a test name contains "crash". +def test_small_order(): + assert True + + +def test_crash_on_large_order(): + pass + + +def test_last_order(): + assert True diff --git a/acceptance/pipelines/test/results/tests/test_delivery.py b/acceptance/pipelines/test/results/tests/test_delivery.py new file mode 100644 index 00000000000..63022e21729 --- /dev/null +++ b/acceptance/pipelines/test/results/tests/test_delivery.py @@ -0,0 +1,3 @@ +# The test server fails the update after the summary when a test name contains "breaks_update". +def test_breaks_update_after_summary(): + assert True diff --git a/acceptance/pipelines/test/results/tests/test_late_failure.py b/acceptance/pipelines/test/results/tests/test_late_failure.py new file mode 100644 index 00000000000..4880a0bca88 --- /dev/null +++ b/acceptance/pipelines/test/results/tests/test_late_failure.py @@ -0,0 +1,7 @@ +# One test fails, and the test server then fails the update with an error of its own. +def test_backfill_fails(): + assert 100 == 150 + + +def test_breaks_update_after_summary(): + assert True diff --git a/acceptance/pipelines/test/results/tests/test_orders.py b/acceptance/pipelines/test/results/tests/test_orders.py new file mode 100644 index 00000000000..96be34e78a0 --- /dev/null +++ b/acceptance/pipelines/test/results/tests/test_orders.py @@ -0,0 +1,15 @@ +# The test server picks each outcome from the test name. +def test_valid_order(): + assert True + + +def test_backfill_fails(): + assert 100 == 150 + + +def test_legacy_format_skipped(): + pass + + +def test_fixture_error(): + pass diff --git a/acceptance/pipelines/test/run-control/databricks.yml b/acceptance/pipelines/test/run-control/databricks.yml new file mode 100644 index 00000000000..66723f3d9d8 --- /dev/null +++ b/acceptance/pipelines/test/run-control/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-pipeline-test-run-control + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-run-control + root_path: . + libraries: + - file: + path: pipeline_file.py + testing: + sources: + - glob: + include: ./tests/** diff --git a/acceptance/pipelines/test/run-control/out.invalid.xml b/acceptance/pipelines/test/run-control/out.invalid.xml new file mode 100644 index 00000000000..ff48bd9dac9 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.invalid.xml @@ -0,0 +1,8 @@ + + + + + reference does not exist: ${var.missing} + + + diff --git a/acceptance/pipelines/test/run-control/out.not-test-update.xml b/acceptance/pipelines/test/run-control/out.not-test-update.xml new file mode 100644 index 00000000000..89adb9dfed7 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.not-test-update.xml @@ -0,0 +1,8 @@ + + + + + update [UUID] is not a test update; pass the update ID that pipelines test printed + + + diff --git a/acceptance/pipelines/test/run-control/out.outside.xml b/acceptance/pipelines/test/run-control/out.outside.xml new file mode 100644 index 00000000000..1e1f250b228 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.outside.xml @@ -0,0 +1,8 @@ + + + + + outside a project, identify the pipeline with --pipeline-id + + + diff --git a/acceptance/pipelines/test/run-control/out.resumed.xml b/acceptance/pipelines/test/run-control/out.resumed.xml new file mode 100644 index 00000000000..c2fb2be3d65 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.resumed.xml @@ -0,0 +1,11 @@ + + + + + + + + RUN INCOMPLETE: the update was canceled before all test results were received. update [UUID] is CANCELED + + + diff --git a/acceptance/pipelines/test/run-control/out.start-failed.xml b/acceptance/pipelines/test/run-control/out.start-failed.xml new file mode 100644 index 00000000000..fe59c1f29d5 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.start-failed.xml @@ -0,0 +1,8 @@ + + + + + An active update '[UUID]' already exists for pipeline '[UUID]'. + + + diff --git a/acceptance/pipelines/test/run-control/out.test.toml b/acceptance/pipelines/test/run-control/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/run-control/output.txt b/acceptance/pipelines/test/run-control/output.txt new file mode 100644 index 00000000000..95883ca956a --- /dev/null +++ b/acceptance/pipelines/test/run-control/output.txt @@ -0,0 +1,101 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-run-control/default/files... +Created pipelines.my_pipeline +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== --no-wait returns after the update starts +>>> [CLI] pipelines test my_pipeline tests/test_orders.py --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +=== --resume reads the results of an existing update +>>> [CLI] pipelines test my_pipeline --resume [UUID] +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Running tests... +tests/test_orders.py::test_valid_order PASSED (1.00s) + +1 test: 1 passed, 0 failed, 0 skipped, 0 errors + +=== --timeout stops waiting but not the update +>>> [CLI] pipelines test my_pipeline tests/test_slow.py --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s --junit-xml out.resumed.xml +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Running tests... +JUnit report: out.resumed.xml +Error: stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --junit-xml out.resumed.xml --resume [UUID] + +>>> cat out.resumed.xml + + + + + + + + stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --junit-xml out.resumed.xml --resume [UUID] + + + + +=== The command to read the results keeps the profile +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s -p run-control +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Running tests... +Error: stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --profile run-control --resume [UUID] + +=== Another update cannot start while the test update is active, and an older report is replaced +>>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py --junit-xml out.start-failed.xml +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: An active update '[UUID]' already exists for pipeline '[UUID]'. + +=== Stopping the pipeline cancels the test update +>>> [CLI] pipelines stop my_pipeline +Stopping my_pipeline... +my_pipeline has been stopped. + +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --junit-xml out.resumed.xml +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +RUN INCOMPLETE: the update was canceled before all test results were received. +Results received: none. +JUnit report: out.resumed.xml +Error: update [UUID] is CANCELED + +=== --resume rejects an update that is not a test update, and an older report is replaced +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --junit-xml out.not-test-update.xml +Error: update [UUID] is not a test update; pass the update ID that pipelines test printed + +=== Outside a project, an older passing report is replaced +Error: outside a project, identify the pipeline with --pipeline-id + +=== A project with errors names the first error in the report +>>> musterr [CLI] pipelines test my_pipeline --junit-xml ../out.invalid.xml +Error: reference does not exist: ${var.missing} + + +=== When the report cannot be written either, the write error is shown +>>> musterr [CLI] pipelines test my_pipeline --junit-xml missing/report.xml +Error: reference does not exist: ${var.missing} + +Error: failed to write JUnit report: open missing/report.xml: no such file or directory diff --git a/acceptance/pipelines/test/run-control/pipeline_file.py b/acceptance/pipelines/test/run-control/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/run-control/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/run-control/script b/acceptance/pipelines/test/run-control/script new file mode 100644 index 00000000000..5dcc822e165 --- /dev/null +++ b/acceptance/pipelines/test/run-control/script @@ -0,0 +1,60 @@ +trace $CLI pipelines deploy + +title "--no-wait returns after the update starts" +trace $CLI pipelines test my_pipeline tests/test_orders.py --no-wait | tee submit.txt +update_id=$(sed -n 's/^Update: //p' submit.txt) + +title "--resume reads the results of an existing update" +trace $CLI pipelines test my_pipeline --resume $update_id + +title "--timeout stops waiting but not the update" +trace $CLI pipelines test my_pipeline tests/test_slow.py --no-wait | tee submit.txt +hang_update_id=$(sed -n 's/^Update: //p' submit.txt) +rm submit.txt +trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id --timeout 3s --junit-xml out.resumed.xml +trace cat out.resumed.xml + +title "The command to read the results keeps the profile" +cat > .databrickscfg <' > out.start-failed.xml +trace musterr $CLI pipelines test my_pipeline tests/test_orders.py --junit-xml out.start-failed.xml + +title "Stopping the pipeline cancels the test update" +trace $CLI pipelines stop my_pipeline +trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id --junit-xml out.resumed.xml + +title "--resume rejects an update that is not a test update, and an older report is replaced" +pipeline_id=$($CLI bundle summary -o json | jq -r '.resources.pipelines.my_pipeline.id') +regular_update_id=$($CLI pipelines start-update $pipeline_id | jq -r .update_id) +echo '' > out.not-test-update.xml +trace musterr $CLI pipelines test my_pipeline --resume $regular_update_id --junit-xml out.not-test-update.xml + +title "Outside a project, an older passing report is replaced\n" +echo '' > out.outside.xml +here=$PWD +outside=$(mktemp -d) +cd $outside +musterr $CLI pipelines test --junit-xml "$here/out.outside.xml" +cd $here +rm -r $outside + +title "A project with errors names the first error in the report" +mkdir invalid +printf 'bundle:\n name: invalid\nresources:\n pipelines:\n my_pipeline:\n name: ${var.missing}\n' > invalid/databricks.yml +cd invalid +trace musterr $CLI pipelines test my_pipeline --junit-xml ../out.invalid.xml + +title "When the report cannot be written either, the write error is shown" +trace musterr $CLI pipelines test my_pipeline --junit-xml missing/report.xml +cd .. +rm -r invalid diff --git a/acceptance/pipelines/test/run-control/test.toml b/acceptance/pipelines/test/run-control/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/pipelines/test/run-control/test.toml @@ -0,0 +1 @@ +RecordRequests = false diff --git a/acceptance/pipelines/test/run-control/tests/test_orders.py b/acceptance/pipelines/test/run-control/tests/test_orders.py new file mode 100644 index 00000000000..f9b3e3fa5d4 --- /dev/null +++ b/acceptance/pipelines/test/run-control/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_valid_order(): + assert True diff --git a/acceptance/pipelines/test/run-control/tests/test_slow.py b/acceptance/pipelines/test/run-control/tests/test_slow.py new file mode 100644 index 00000000000..f7be6cccad4 --- /dev/null +++ b/acceptance/pipelines/test/run-control/tests/test_slow.py @@ -0,0 +1,3 @@ +# The test server never finishes a test whose name contains "hang". +def test_hang_on_lock(): + pass diff --git a/acceptance/pipelines/test/selection/databricks.yml b/acceptance/pipelines/test/selection/databricks.yml new file mode 100644 index 00000000000..ea0662b1043 --- /dev/null +++ b/acceptance/pipelines/test/selection/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: test-pipeline-test-selection + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-selection + libraries: + - file: + path: pipeline_file.py + testing: + sources: + - glob: + include: ./tests/** diff --git a/acceptance/pipelines/test/selection/out.test.toml b/acceptance/pipelines/test/selection/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/selection/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt new file mode 100644 index 00000000000..8209f96125c --- /dev/null +++ b/acceptance/pipelines/test/selection/output.txt @@ -0,0 +1,140 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files... +Created pipelines.my_pipeline +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== Run all tests of the only pipeline +>>> [CLI] pipelines test --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true + } +} + +=== Select tests by relative path, test ID, and workspace path, with pytest arguments +>>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_valid_order --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg=-m --pytest-arg=smoke,fast --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests", + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_backfill", + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_valid_order" + ], + "pytest_args": [ + "-k", + "orders and not slow", + "-m", + "smoke,fast" + ] + } + } +} + +=== The API rejects selections outside testing.sources +>>> musterr [CLI] pipelines test my_pipeline pipeline_file.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: The test selection '/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/pipeline_file.py' is outside the pipeline's test sources. Select a file or folder inside testing.sources. + +>>> musterr [CLI] pipelines test my_pipeline /Workspace/Shared/tests/test_shared.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: The test selection '/Workspace/Shared/tests/test_shared.py' is outside the pipeline's test sources. Select a file or folder inside testing.sources. + +=== --pytest-arg also takes its value as the next argument +>>> [CLI] pipelines test my_pipeline --pytest-arg -k --pytest-arg orders, returns --pytest-arg -m --pytest-arg smoke --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "pytest_args": [ + "-k", + "orders, returns", + "-m", + "smoke" + ] + } + } +} + +=== Relative paths resolve from the working directory +>>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_valid_order" + ] + } + } +} + +=== Test selections require the pipeline KEY first +>>> musterr [CLI] pipelines test tests/test_orders.py +Error: resource with key "tests/test_orders.py" not found; the first argument must be a pipeline KEY when test selections are given + +=== Test selections must be in the synchronized files +>>> musterr [CLI] pipelines test my_pipeline ../test_outside.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot resolve test selection "../test_outside.py": [TEST_TMP_DIR]_PARENT/test_outside.py is outside the synchronized bundle files in [TEST_TMP_DIR] + +=== Relative test selections must exist locally +>>> musterr [CLI] pipelines test my_pipeline tests/test_ordrs.py::test_backfill +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot resolve test selection "tests/test_ordrs.py::test_backfill": [TEST_TMP_DIR]/tests/test_ordrs.py does not exist; pass a local path or an absolute workspace path diff --git a/acceptance/pipelines/test/selection/pipeline_file.py b/acceptance/pipelines/test/selection/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/selection/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script new file mode 100644 index 00000000000..2807c301373 --- /dev/null +++ b/acceptance/pipelines/test/selection/script @@ -0,0 +1,42 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "Run all tests of the only pipeline" +trace $CLI pipelines test --no-wait +trace print_requests.py //updates +# Like the service, the test server rejects a new update while one is active. +$CLI pipelines stop my_pipeline &>> LOG.stop +rm "$OUT_REQUESTS" + +title "Select tests by relative path, test ID, and workspace path, with pytest arguments" +file_path=$($CLI bundle validate -o json | jq -r .workspace.file_path) +trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill $file_path/tests/test_orders.py::test_valid_order --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg=-m --pytest-arg=smoke,fast --no-wait +trace print_requests.py //updates +$CLI pipelines stop my_pipeline &>> LOG.stop +rm "$OUT_REQUESTS" + +title "The API rejects selections outside testing.sources" +trace musterr $CLI pipelines test my_pipeline pipeline_file.py +trace musterr $CLI pipelines test my_pipeline /Workspace/Shared/tests/test_shared.py +rm "$OUT_REQUESTS" + +title "--pytest-arg also takes its value as the next argument" +trace $CLI pipelines test my_pipeline --pytest-arg -k --pytest-arg 'orders, returns' --pytest-arg -m --pytest-arg smoke --no-wait +trace print_requests.py //updates +$CLI pipelines stop my_pipeline &>> LOG.stop +rm "$OUT_REQUESTS" + +title "Relative paths resolve from the working directory" +cd tests +trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order --no-wait +cd .. +trace print_requests.py //updates + +title "Test selections require the pipeline KEY first" +trace musterr $CLI pipelines test tests/test_orders.py + +title "Test selections must be in the synchronized files" +trace musterr $CLI pipelines test my_pipeline ../test_outside.py + +title "Relative test selections must exist locally" +trace musterr $CLI pipelines test my_pipeline tests/test_ordrs.py::test_backfill diff --git a/acceptance/pipelines/test/selection/tests/test_orders.py b/acceptance/pipelines/test/selection/tests/test_orders.py new file mode 100644 index 00000000000..6058cdd0bba --- /dev/null +++ b/acceptance/pipelines/test/selection/tests/test_orders.py @@ -0,0 +1,6 @@ +def test_valid_order(): + assert True + + +def test_backfill(): + assert True diff --git a/acceptance/pipelines/test/test.toml b/acceptance/pipelines/test/test.toml new file mode 100644 index 00000000000..9fc0358f8ac --- /dev/null +++ b/acceptance/pipelines/test/test.toml @@ -0,0 +1,3 @@ +RecordRequests = true +Ignore = ['.databricks'] +Env.MSYS_NO_PATHCONV = "1" diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index b5af086dd34..88978e93ae3 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -44,48 +44,42 @@ func pipelineRewritePatterns() []pipelineRewritePattern { mode: TranslateModeDirectory, skipRewrite: noSkipRewrite, }, + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("sources"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")), + mode: TranslateModeGlob, + skipRewrite: noSkipRewrite, + }, } return allPatterns } func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { - pipelineEnvironmentsPatterns := []pipelineRewritePattern{ - { - pattern: dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - mode: TranslateModeLocalRelativeWithPrefix, - skipRewrite: func(s string) bool { - return !libraries.IsLibraryLocal(s) + var patterns []pipelineRewritePattern + // Test dependencies take the same forms as the pipeline's environment dependencies. + for _, dependencies := range []dyn.Pattern{ + base.Append(dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + } { + patterns = append(patterns, + pipelineRewritePattern{ + pattern: dependencies, + mode: TranslateModeLocalRelativeWithPrefix, + skipRewrite: func(s string) bool { + return !libraries.IsLibraryLocal(s) + }, }, - }, - } - - pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ - { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - TranslateModeEnvironmentPipFlag, - func(s string) bool { - _, _, ok := libraries.IsLocalPathInPipFlag(s) - return !ok + pipelineRewritePattern{ + pattern: dependencies, + mode: TranslateModeEnvironmentPipFlag, + skipRewrite: func(s string) bool { + _, _, ok := libraries.IsLocalPathInPipFlag(s) + return !ok + }, }, - }, + ) } - - return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) + return patterns } func VisitPipelinePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go index eb9748bba37..85c29065c5b 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go @@ -40,6 +40,13 @@ func TestVisitPipelinePaths(t *testing.T) { }, }, }, + // The acceptance test bundle/resources/pipelines/testing covers the testing paths + // that are rewritten. A package name is not a path. + Testing: &resources.PipelineTesting{ + Environment: &resources.PipelineTestEnvironment{ + Dependencies: []string{"faker"}, + }, + }, }, }, }, diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 956d010ec95..cdf9a95f902 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -180,9 +180,7 @@ func (t *translateContext) rewritePath( case paths.TranslateModeLocalRelativeWithPrefix: interp, err = t.translateLocalRelativeWithPrefixPath(ctx, input, localPath, localRelPath) case paths.TranslateModeEnvironmentPipFlag: - interp, err = t.translateFilePath(ctx, input, localPath, localRelPath) - // Add the flag prefix to the path to indicate it's a local file used in pip flags for environment dependencies. - interp = flag + " " + interp + interp, err = t.translatePipFlagPath(ctx, flag, input, localPath, localRelPath) default: return "", fmt.Errorf("unsupported translate mode: %d", opts.Mode) } @@ -251,6 +249,26 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local return path.Join(t.remoteRoot, localRelPath), nil } +// translatePipFlagPath translates the local path after a pip flag of an environment dependency, +// and keeps the flag in front of it. After -e, the path can also be a project folder, with +// optional extras such as ./project[test]. +func (t *translateContext) translatePipFlagPath(ctx context.Context, flag, literal, localFullPath, localRelPath string) (string, error) { + if flag == "-e" { + folder, extras := localRelPath, "" + if i := strings.LastIndex(localRelPath, "["); i > 0 && strings.HasSuffix(localRelPath, "]") { + folder, extras = localRelPath[:i], localRelPath[i:] + } + if info, err := t.b.SyncRoot.Stat(folder); err == nil && info.IsDir() { + return flag + " " + path.Join(t.remoteRoot, folder) + extras, nil + } + } + interp, err := t.translateFilePath(ctx, literal, localFullPath, localRelPath) + if err != nil { + return "", err + } + return flag + " " + interp, nil +} + func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { if t.skipLocalFileValidation { return path.Join(t.remoteRoot, localRelPath), nil diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 8aaf9ce9c71..b96d1ade278 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -816,6 +816,50 @@ func TestTranslatePathJobEnvironments(t *testing.T) { assert.Equal(t, "https://foo@bar.test/packages/pypi/simple", b.Config.Resources.Jobs["job"].Environments[0].Spec.Dependencies[6]) } +func TestTranslatePathEditableProjectFolder(t *testing.T) { + dir := t.TempDir() + touchEmptyFile(t, filepath.Join(dir, "project", "pyproject.toml")) + touchEmptyFile(t, filepath.Join(dir, "requirements.txt")) + + translate := func(dependencies ...string) (*bundle.Bundle, diag.Diagnostics) { + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Workspace: config.Workspace{ + FilePath: "/bundle", + }, + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Environment: &pipelines.PipelinesEnvironment{Dependencies: dependencies}, + }, + }, + }, + }, + }, + } + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}}) + return b, bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) + } + + b, diags := translate("-e ./project", "-e ./project[test]", "-e .", "-e .[test]", "-r ./requirements.txt") + require.NoError(t, diags.Error()) + assert.Equal(t, []string{ + "-e /bundle/project", + "-e /bundle/project[test]", + "-e /bundle", + "-e /bundle[test]", + "-r /bundle/requirements.txt", + }, b.Config.Resources.Pipelines["pipeline"].Environment.Dependencies) + + // -r takes a file, so a folder after it is still an error. + _, diags = translate("-r ./project") + assert.ErrorContains(t, diags.Error(), "project") +} + func TestTranslatePathWithComplexVariables(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index fd1fc61ce86..1b8b8e0ef43 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -22,6 +22,34 @@ type Pipeline struct { // Nil (unset) uses the server default, which cascades. Set to false to retain datasets // when the pipeline is deleted. Delete-time only: never sent on create/update. CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` + + // Testing is PipelineSpec.testing, which the Go SDK does not have yet. Remove it, and the + // types below, once CreatePipeline has it. + Testing *PipelineTesting `json:"testing,omitempty"` +} + +// PipelineTesting is the test configuration of a pipeline. Only test-only updates use it. +type PipelineTesting struct { + // Sources are the workspace files and folders that hold the pipeline's tests. They must + // not overlap the pipeline's libraries. + Sources []PipelineTestSource `json:"sources,omitempty"` + + // Environment lists the packages that only the tests need. + Environment *PipelineTestEnvironment `json:"environment,omitempty"` +} + +// PipelineTestSource is a workspace location that holds pipeline tests. +type PipelineTestSource struct { + // Glob is a test file, or a folder followed by /**. + Glob *pipelines.PathPattern `json:"glob,omitempty"` +} + +// PipelineTestEnvironment lists the packages that a pipeline's tests need in addition to the +// pipeline's environment. +type PipelineTestEnvironment struct { + // Dependencies are files that declare the packages: a project folder, a requirements + // file, or a wheel. Package names and URLs are rejected by the API. + Dependencies []string `json:"dependencies,omitempty"` } func (p *Pipeline) UnmarshalJSON(b []byte) error { diff --git a/bundle/config/validate/pipeline_testing_dependencies.go b/bundle/config/validate/pipeline_testing_dependencies.go new file mode 100644 index 00000000000..047c55e4f8d --- /dev/null +++ b/bundle/config/validate/pipeline_testing_dependencies.go @@ -0,0 +1,123 @@ +package validate + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" +) + +// pipelineTestingDependencyOptions are the pip options that may come before a test dependency's path. +var pipelineTestingDependencyOptions = []string{"--requirement", "--editable", "-r", "-e"} + +const ( + pipelineTestingDependencyNotFile = "not a file" + pipelineTestingDependencyNeedsRequirementOption = "requirements file without -r" +) + +type pipelineTestingDependencies struct{ bundle.RO } + +// PipelineTestingDependencies rejects the entries of testing.environment.dependencies that the API +// rejects on deploy, such as package names and URLs. It runs after paths are translated. +func PipelineTestingDependencies() bundle.ReadOnlyMutator { + return &pipelineTestingDependencies{} +} + +func (v *pipelineTestingDependencies) Name() string { + return "validate:pipeline_testing_dependencies" +} + +func (v *pipelineTestingDependencies) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + pattern := dyn.NewPattern( + dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), + dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex(), + ) + _, err := dyn.MapByPattern(b.Config.Value(), pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + dependency, ok := v.AsString() + if !ok { + return v, nil + } + summary := fmt.Sprintf("testing.environment.dependencies must point at files, not %q", dependency) + detail := "A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported." + switch pipelineTestingDependencyProblem(dependency) { + case "": + return v, nil + case pipelineTestingDependencyNeedsRequirementOption: + summary = fmt.Sprintf("testing.environment.dependencies needs -r before the requirements file %q", dependency) + detail = "Without -r, pip treats the file as a package to install. For example: -r ./tests/requirements.txt." + } + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: summary, + Detail: detail, + Locations: v.Locations(), + Paths: dyn.ToStructPaths(p), + }) + return v, nil + }) + if err != nil { + return diag.FromErr(err) + } + return diags +} + +// pipelineTestingDependencyProblem returns why dependency is a common mistake that the API rejects, +// or "" if it is not: a package name, a URL, another pip option, "..", a path outside /Workspace and +// /Volumes, or a requirements file without -r. Anything else is left to the API. dependency is a +// value after path translation, so a relative path is a local library that deploy uploads. +func pipelineTestingDependencyProblem(dependency string) string { + // A reference resolved at deploy time, such as a volume path, is checked by the API. + if dynvar.ContainsVariableReference(dependency) { + return "" + } + value := strings.TrimSpace(dependency) + option, filePath := cutPipelineTestingDependencyOption(value) + switch { + case strings.Contains(value, "://"): + return pipelineTestingDependencyNotFile + case strings.HasPrefix(filePath, "-") || strings.Contains(filePath, " -"): + return pipelineTestingDependencyNotFile + case strings.HasPrefix(filePath, "/"): + if !strings.HasPrefix(filePath, "/Workspace/") && !strings.HasPrefix(filePath, "/Volumes/") { + return pipelineTestingDependencyNotFile + } + if slices.Contains(strings.Split(filePath, "/"), "..") { + return pipelineTestingDependencyNotFile + } + // Path translation rewrites a path in the bundle after "-r " or "-e ", so a relative path that + // remains after an option was not rewritten. + case option != "" || !libraries.IsLibraryLocal(filePath): + return pipelineTestingDependencyNotFile + } + if strings.HasSuffix(strings.ToLower(filePath), ".txt") && option != "-r" && option != "--requirement" { + return pipelineTestingDependencyNeedsRequirementOption + } + return "" +} + +// cutPipelineTestingDependencyOption splits a leading pip option, such as "-r", from the path after it. +func cutPipelineTestingDependencyOption(value string) (string, string) { + for _, option := range pipelineTestingDependencyOptions { + rest, ok := strings.CutPrefix(value, option) + if !ok { + continue + } + switch { + case strings.HasPrefix(rest, "=") && strings.HasPrefix(option, "--"): + return option, strings.TrimSpace(rest[1:]) + case strings.HasPrefix(rest, " ") || strings.HasPrefix(rest, "\t"): + return option, strings.TrimSpace(rest) + case strings.HasPrefix(rest, "/") && !strings.HasPrefix(option, "--"): + return option, rest + } + } + return "", value +} diff --git a/bundle/config/validate/pipeline_testing_dependencies_test.go b/bundle/config/validate/pipeline_testing_dependencies_test.go new file mode 100644 index 00000000000..10a8954815d --- /dev/null +++ b/bundle/config/validate/pipeline_testing_dependencies_test.go @@ -0,0 +1,64 @@ +package validate + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPipelineTestingDependencyProblem(t *testing.T) { + accepted := []string{ + "--editable /Workspace/Users/someone@example.com/orders[test]", + "-e /Workspace/orders", + "-e/Workspace/orders", + "--editable=/Workspace/orders", + "-r /Workspace/orders/tests/requirements.txt", + "--requirement=/Workspace/orders/requirements.txt", + "--requirement /Volumes/main/default/reqs/requirements.TXT", + "/Volumes/main/default/wheels/helpers-0.1-py3-none-any.whl", + "/Workspace/Users/someone@example.com/my orders/project", + "./dist/*.whl", + "dist/helpers-0.1-py3-none-any.whl", + "${resources.volumes.wheels.volume_path}/helpers-0.1-py3-none-any.whl", + // Not a common mistake, so the API decides. + "/Workspace/orders\n/Workspace/shared", + } + for _, dependency := range accepted { + assert.Empty(t, pipelineTestingDependencyProblem(dependency), dependency) + } + + notFiles := []string{ + // Package names. + "faker", + "faker==33.0", + "requests>=2", + // URLs. + "git+https://github.com/example/helpers", + "https://example.com/helpers-0.1-py3-none-any.whl", + "--editable git+https://github.com/example/helpers", + // Other pip options. + "-i https://example.com/simple", + "/Workspace/orders -i example", + "--no-deps /Workspace/orders", + // Paths outside /Workspace and /Volumes. + "/dbfs/helpers-0.1-py3-none-any.whl", + "/Users/someone@example.com/helpers-0.1-py3-none-any.whl", + // "..". + "/Workspace/orders/../shared", + // A path in the bundle that path translation did not rewrite. + "--editable ./project", + "--requirement ./tests/requirements.txt", + } + for _, dependency := range notFiles { + assert.Equal(t, pipelineTestingDependencyNotFile, pipelineTestingDependencyProblem(dependency), dependency) + } + + needsRequirementOption := []string{ + "/Workspace/orders/requirements.txt", + "--editable /Workspace/orders/requirements.txt", + "./tests/requirements.txt", + } + for _, dependency := range needsRequirementOption { + assert.Equal(t, pipelineTestingDependencyNeedsRequirementOption, pipelineTestingDependencyProblem(dependency), dependency) + } +} diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 6f367d4e498..2f5517556bb 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -2,13 +2,18 @@ package dresources import ( "context" + "encoding/json" + "fmt" + "net/http" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/pipelines" ) @@ -22,6 +27,9 @@ type PipelineState struct { // STs, Views). Nil means the server default (cascade) applies. Read from persisted state at // delete time; never sent on create/update. CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` + + // Testing is PipelineSpec.testing, which the Go SDK does not have yet. + Testing *resources.PipelineTesting `json:"testing,omitempty"` } func (s *PipelineState) UnmarshalJSON(b []byte) error { @@ -52,6 +60,9 @@ type PipelineRemote struct { PipelineId string `json:"pipeline_id,omitempty"` RunAsUserName string `json:"run_as_user_name,omitempty"` State pipelines.PipelineState `json:"state,omitempty"` + + // Testing is spec.testing of the Get response, read by DoRead because the Go SDK does not have it yet. + Testing *resources.PipelineTesting `json:"testing,omitempty"` } // Custom marshaler needed because embedded CreatePipeline has its own MarshalJSON @@ -66,18 +77,27 @@ func (s PipelineRemote) MarshalJSON() ([]byte, error) { type ResourcePipeline struct { client *databricks.WorkspaceClient + + // apiClient sends the requests that the Go SDK cannot express yet because it lacks the testing + // field. It is built once, so that those requests share one rate limiter. + apiClient *client.DatabricksClient + apiClientErr error } -func (*ResourcePipeline) New(client *databricks.WorkspaceClient) *ResourcePipeline { - return &ResourcePipeline{ - client: client, +func (*ResourcePipeline) New(w *databricks.WorkspaceClient) *ResourcePipeline { + r := &ResourcePipeline{client: w, apiClient: nil, apiClientErr: nil} + // The adapter also builds a resource without a client to read its configuration. + if w != nil { + r.apiClient, r.apiClientErr = client.New(w.Config) } + return r } func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *PipelineState { return &PipelineState{ CreatePipeline: input.CreatePipeline, CascadeOnDestroy: input.CascadeOnDestroy, + Testing: input.Testing, } } @@ -86,15 +106,35 @@ func (*ResourcePipeline) RemapState(remote *PipelineRemote) *PipelineState { CreatePipeline: remote.CreatePipeline, // cascade_on_destroy is input-only and absent from PipelineRemote, so it stays nil here. CascadeOnDestroy: nil, + Testing: remote.Testing, } } +// pipelineTestingResponse holds spec.testing of a Get Pipeline response. +type pipelineTestingResponse struct { + Spec struct { + Testing *resources.PipelineTesting `json:"testing"` + } `json:"spec"` +} + +// DoRead sends the same request as GetByPipelineId, but raw, so that it can also read spec.testing, +// which GetPipelineResponse does not have yet. Switch back to GetByPipelineId once it does. func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemote, error) { - resp, err := r.client.Pipelines.GetByPipelineId(ctx, id) - if err != nil { + var raw json.RawMessage + if err := r.doRaw(ctx, http.MethodGet, "/api/2.0/pipelines/"+id, nil, &raw); err != nil { return nil, err } - return makePipelineRemote(resp), nil + var resp pipelines.GetPipelineResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, err + } + var testing pipelineTestingResponse + if err := json.Unmarshal(raw, &testing); err != nil { + return nil, err + } + remote := makePipelineRemote(&resp) + remote.Testing = testing.Spec.Testing + return remote, nil } func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { @@ -155,10 +195,20 @@ func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { PipelineId: p.PipelineId, RunAsUserName: p.RunAsUserName, State: p.State, + // GetPipelineResponse has no spec.testing; DoRead sets it from the raw response. + Testing: nil, } } func (r *ResourcePipeline) DoCreate(ctx context.Context, config *PipelineState) (string, *PipelineRemote, error) { + if config.Testing != nil { + var response pipelines.CreatePipelineResponse + err := r.doWithTesting(ctx, http.MethodPost, "/api/2.0/pipelines", config.CreatePipeline, config.Testing, &response) + if err != nil { + return "", nil, err + } + return response.PipelineId, nil, nil + } response, err := r.client.Pipelines.Create(ctx, config.CreatePipeline) if err != nil { return "", nil, err @@ -212,6 +262,9 @@ func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *Pipe ForceSendFields: utils.FilterFields[pipelines.EditPipeline](config.ForceSendFields), } + if config.Testing != nil { + return nil, r.doWithTesting(ctx, http.MethodPut, "/api/2.0/pipelines/"+id, request, config.Testing, nil) + } return nil, r.client.Pipelines.Update(ctx, request) } @@ -242,6 +295,34 @@ func (*ResourcePipeline) OverrideChangeDesc(_ context.Context, path *structpath. return nil } +// doWithTesting sends a Create or Edit request with the testing field, which the Go SDK does not +// have yet. The body is the SDK's JSON for the request plus testing. Remove this once +// CreatePipeline and EditPipeline have the field; pipelines without testing already use the SDK. +func (r *ResourcePipeline) doWithTesting(ctx context.Context, method, path string, request any, testing *resources.PipelineTesting, response any) error { + data, err := json.Marshal(request) + if err != nil { + return err + } + // RawMessage values keep the SDK's encoding, including large integers, unchanged. + var body map[string]json.RawMessage + if err := json.Unmarshal(data, &body); err != nil { + return err + } + body["testing"], err = json.Marshal(testing) + if err != nil { + return err + } + return r.doRaw(ctx, method, path, body, response) +} + +// doRaw sends a request that the Go SDK cannot express yet because it lacks the testing field. +func (r *ResourcePipeline) doRaw(ctx context.Context, method, path string, request, response any) error { + if r.apiClientErr != nil { + return fmt.Errorf("failed to create API client: %w", r.apiClientErr) + } + return r.apiClient.Do(ctx, method, path, auth.WorkspaceIDHeaders(r.client.Config), nil, request, response) +} + // Note, terraform provider either // a) reads back state at least once and fails create if state is "failed" // b) repeatededly reads state until state is "running" (if spec.Contionous is set). diff --git a/bundle/direct/dresources/pipeline_test.go b/bundle/direct/dresources/pipeline_test.go index 3629255a600..418555d0578 100644 --- a/bundle/direct/dresources/pipeline_test.go +++ b/bundle/direct/dresources/pipeline_test.go @@ -4,8 +4,12 @@ import ( "reflect" "testing" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -53,3 +57,60 @@ func TestPipelineOverrideChangeDescCascadeOnDestroy(t *testing.T) { assert.Equal(t, deployplan.Skip, change.Action) }) } + +func TestPipelineRawRequests(t *testing.T) { + server := testserver.New(t) + var bodies, workspaceIDs []string + server.Handle("POST", "/api/2.0/pipelines", func(req testserver.Request) any { + bodies = append(bodies, string(req.Body)) + workspaceIDs = append(workspaceIDs, req.Headers.Get(auth.WorkspaceIDHeader)) + return pipelines.CreatePipelineResponse{PipelineId: "p1"} + }) + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req testserver.Request) any { + workspaceIDs = append(workspaceIDs, req.Headers.Get(auth.WorkspaceIDHeader)) + return map[string]any{"pipeline_id": "p1", "spec": map[string]any{"name": "orders"}} + }) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token", WorkspaceID: "123"}) + require.NoError(t, err) + r := (&ResourcePipeline{}).New(w) + + testing := &resources.PipelineTesting{Sources: []resources.PipelineTestSource{{Glob: &pipelines.PathPattern{Include: "/Workspace/orders/tests/**"}}}} + _, _, err = r.DoCreate(t.Context(), &PipelineState{Name: "orders", Testing: testing}) + require.NoError(t, err) + _, err = r.DoRead(t.Context(), "p1") + require.NoError(t, err) + // Without testing, the request is the SDK's. + _, _, err = r.DoCreate(t.Context(), &PipelineState{Name: "orders"}) + require.NoError(t, err) + + // The raw requests carry the workspace ID, as the SDK's do. + assert.Equal(t, []string{"123", "123", "123"}, workspaceIDs) + assert.JSONEq(t, `{"name": "orders"}`, bodies[1]) +} + +func TestPipelineReadReturnsTesting(t *testing.T) { + server := testserver.New(t) + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req testserver.Request) any { + return map[string]any{ + "pipeline_id": req.Vars["pipeline_id"], + "state": "IDLE", + "spec": map[string]any{ + "name": "orders", + "testing": map[string]any{"sources": []any{map[string]any{"glob": map[string]any{"include": "/Workspace/orders/tests/**"}}}}, + }, + } + }) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + r := (&ResourcePipeline{}).New(w) + + remote, err := r.DoRead(t.Context(), "p1") + require.NoError(t, err) + assert.Equal(t, "p1", remote.PipelineId) + assert.Equal(t, "orders", remote.Name) + assert.Equal(t, pipelines.PipelineStateIdle, remote.State) + require.NotNil(t, remote.Testing) + require.Len(t, remote.Testing.Sources, 1) + assert.Equal(t, "/Workspace/orders/tests/**", remote.Testing.Sources[0].Glob.Include) + assert.Same(t, remote.Testing, r.RemapState(remote).Testing) +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index d4885599805..28ab10430c1 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1592,6 +1592,26 @@ resources: "user_name": "description": |- The name of the user granted the permission level. + "testing": + "description": |- + The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. + "x-databricks-launch-stage": |- + PRIVATE_PREVIEW + "$fields": + "environment": + "description": |- + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + "$fields": + "dependencies": + "description": |- + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and an extra, for example `--editable ${workspace.file_path}[test]`. Package names and URLs are rejected. + "sources": + "description": |- + The workspace files and folders that hold the pipeline's tests. They must not overlap the pipeline's `libraries`. + "$fields": + "glob": + "description": |- + A test file, or a folder followed by `/**`. "trigger": "deprecation_message": |- Use continuous instead diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index df089f880a7..91f5ab261b1 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -198,6 +198,15 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) +var pipelineTestingEnvDepsPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("pipelines"), + dyn.AnyKey(), + dyn.Key("testing"), + dyn.Key("environment"), + dyn.Key("dependencies"), +) + var clusterLibrariesPattern = dyn.NewPattern( dyn.Key("resources"), dyn.Key("clusters"), @@ -223,6 +232,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { pattern: pipelineEnvDepsPattern, fn: expandEnvironmentDeps, }, + { + pattern: pipelineTestingEnvDepsPattern, + fn: expandEnvironmentDeps, + }, { pattern: clusterLibrariesPattern, fn: expandLibraries, diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index 70e577a1a2a..359744328b4 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -143,6 +143,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), // The AI Runtime task's code_source_path is a local archive (typically an // artifact-built .tar.gz) that must be uploaded and referenced by its remote // path, exactly like a wheel or jar library. diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 722d42c7f64..e4e1c62c614 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -22,6 +22,7 @@ var patterns = []dyn.Pattern{ clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), } type libData struct { diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index ef74a786ca1..627d3eadb2d 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -222,6 +222,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { mutator.TranslatePaths(), + // Reject the pipelines' testing.environment.dependencies that the API rejects. Runs after + // TranslatePaths, so that it checks the paths that deploy sends. + validate.PipelineTestingDependencies(), + // Reads (typed): b.Config.Experimental.PythonWheelWrapper, b.Config.Workspace.SyncMode (checks Python wheel wrapper and deployment mode settings) // Reads (dynamic): resources.jobs.*.tasks (checks for tasks with local libraries and incompatible DBR versions) // Provides warnings when Python wheel tasks are used with DBR < 13.3 or when wheel wrapper is incompatible with source-linked deployment diff --git a/bundle/root.go b/bundle/root.go index 5f7317a5e99..c41c52edfb7 100644 --- a/bundle/root.go +++ b/bundle/root.go @@ -70,6 +70,13 @@ func mustGetRoot(ctx context.Context) (string, diag.Diagnostics) { return getRootWithTraversal() } +// HasRoot reports whether a bundle root is configured or can be found without loading it. +// An invalid bundle root environment variable counts as a root so that loading reports the error. +func HasRoot(ctx context.Context) bool { + path, diags := tryGetRoot(ctx) + return path != "" || diags.HasError() +} + // tryGetRoot returns a bundle root or an empty string if one cannot be found. func tryGetRoot(ctx context.Context) (string, diag.Diagnostics) { // Note: an invalid value in the environment variable is still an error. diff --git a/bundle/root_test.go b/bundle/root_test.go index 27109f2aeff..20b160b8b40 100644 --- a/bundle/root_test.go +++ b/bundle/root_test.go @@ -110,3 +110,24 @@ func TestRootLookupError(t *testing.T) { require.True(t, diags.HasError()) require.Contains(t, diags[0].Summary, "Unable to locate the bundle root") } + +func TestHasRoot(t *testing.T) { + ctx := t.Context() + + // Have to set then unset to allow the testing package to revert it to its original value. + t.Setenv(env.RootVariable, "") + os.Unsetenv(env.RootVariable) + + dir := t.TempDir() + t.Chdir(dir) + require.False(t, HasRoot(ctx)) + + f, err := os.Create(filepath.Join(dir, config.FileNames[0])) + require.NoError(t, err) + f.Close() + require.True(t, HasRoot(ctx)) + + // An invalid environment variable is reported when the bundle is loaded. + t.Setenv(env.RootVariable, filepath.Join(dir, "doesntexist")) + require.True(t, HasRoot(ctx)) +} diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 959273d2586..f07b44dcbaa 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2309,6 +2309,12 @@ "deprecationMessage": "This field is deprecated", "deprecated": true }, + "testing": { + "description": "[Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTesting", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, "trigger": { "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger", @@ -2365,6 +2371,64 @@ } ] }, + "resources.PipelineTestEnvironment": { + "oneOf": [ + { + "type": "object", + "properties": { + "dependencies": { + "description": "A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and an extra, for example `--editable ${workspace.file_path}[test]`. Package names and URLs are rejected.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelineTestSource": { + "oneOf": [ + { + "type": "object", + "properties": { + "glob": { + "description": "A test file, or a folder followed by `/**`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PathPattern" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelineTesting": { + "oneOf": [ + { + "type": "object", + "properties": { + "environment": { + "description": "The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTestEnvironment" + }, + "sources": { + "description": "The workspace files and folders that hold the pipeline's tests. They must not overlap the pipeline's `libraries`.", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.PipelineTestSource" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresBranch": { "oneOf": [ { @@ -20875,6 +20939,20 @@ } ] }, + "resources.PipelineTestSource": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTestSource" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.SecretScopePermission": { "oneOf": [ { diff --git a/cmd/pipelines/commands.go b/cmd/pipelines/commands.go index 6a6b6931313..24882f97864 100644 --- a/cmd/pipelines/commands.go +++ b/cmd/pipelines/commands.go @@ -17,6 +17,7 @@ func Commands() []*cobra.Command { destroyCommand(), runCommand(), dryRunCommand(), + testCommand(), historyCommand(), logsCommand(), openCommand(), diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go new file mode 100644 index 00000000000..efc3bcf9c54 --- /dev/null +++ b/cmd/pipelines/test.go @@ -0,0 +1,701 @@ +package pipelines + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "os/signal" + "path" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdgroup" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/shellquote" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/google/uuid" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// startTestUpdateRequest is the Start Update body of a test run. +// test_only and test_options are not in the Go SDK yet, so the request is sent directly. +type startTestUpdateRequest struct { + TestOnly bool `json:"test_only"` + TestOptions *testOptions `json:"test_options,omitempty"` +} + +type testOptions struct { + TestSelection []string `json:"test_selection,omitempty"` + PytestArgs []string `json:"pytest_args,omitempty"` +} + +func newStartTestUpdateRequest(selection, pytestArgs []string) startTestUpdateRequest { + req := startTestUpdateRequest{TestOnly: true} + if len(selection) > 0 || len(pytestArgs) > 0 { + req.TestOptions = &testOptions{TestSelection: selection, PytestArgs: pytestArgs} + } + return req +} + +const pytestArgHint = "pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'" + +// validatePytestArgs accepts only -k and -m, each followed by its value in the next argument, as the API +// requires. The value cannot be empty or start with "-", because pytest reads it as an option. +func validatePytestArgs(args []string) error { + for i := 0; i < len(args); i++ { + arg := args[i] + switch { + case arg == "-k" || arg == "-m": + if i == len(args)-1 { + return fmt.Errorf("pytest argument %s requires a value: pass it in the next --pytest-arg", arg) + } + next := args[i+1] + if strings.TrimSpace(next) == "" { + return fmt.Errorf("pytest argument %s requires a non-empty value", arg) + } + if strings.HasPrefix(next, "-") { + return fmt.Errorf("pytest argument %s requires a value, not %q: pass it in the next --pytest-arg", arg, next) + } + i++ + case strings.HasPrefix(arg, "-k") || strings.HasPrefix(arg, "-m"): + option, value := arg[:2], strings.TrimPrefix(arg[2:], "=") + return fmt.Errorf("pytest argument %q must be split: pass --pytest-arg=%s --pytest-arg=%s", arg, option, shellquote.BashArg(value)) + case !strings.HasPrefix(arg, "-"): + return fmt.Errorf("unsupported pytest argument %q: pass test paths and test IDs as TEST_SELECTION arguments", arg) + default: + return fmt.Errorf("unsupported pytest argument %q: only -k and -m are supported", arg) + } + } + return nil +} + +// testFlagErrorFunc adds guidance when pytest's -k or -m is passed as a CLI flag. +func testFlagErrorFunc(cmd *cobra.Command, err error) error { + if notExist, ok := errors.AsType[*pflag.NotExistError](err); ok { + if name := notExist.GetSpecifiedName(); name == "k" || name == "m" { + return fmt.Errorf("-%s is a pytest argument: %s", name, pytestArgHint) + } + } + return cmd.Parent().FlagErrorFunc()(cmd, err) +} + +// resolveTestSelections converts relative selection paths with toWorkspacePath and keeps +// the pytest test ID after "::" unchanged. Paths that start with "/" are workspace paths. +func resolveTestSelections(selections []string, toWorkspacePath func(string) (string, error)) ([]string, error) { + var resolved []string + for _, selection := range selections { + p, testID, _ := strings.Cut(selection, "::") + if p == "" { + return nil, fmt.Errorf("test selection %q has no path", selection) + } + if strings.HasPrefix(p, "/") { + p = path.Clean(p) + } else { + var err error + p, err = toWorkspacePath(p) + if err != nil { + return nil, fmt.Errorf("cannot resolve test selection %q: %w", selection, err) + } + } + if strings.Contains(selection, "::") { + p += "::" + testID + } + resolved = append(resolved, p) + } + return resolved, nil +} + +// bundleWorkspacePath maps a local path, relative to the working directory, to its deployed +// path under filesRoot, the deployed path of the bundle's synchronized files. +func bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath string) (string, error) { + if !filepath.IsAbs(localPath) { + localPath = filepath.Join(cwd, localPath) + } + rel, err := filepath.Rel(syncRoot, localPath) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("%s is outside the synchronized bundle files in %s", filepath.ToSlash(localPath), filepath.ToSlash(syncRoot)) + } + return path.Join(filesRoot, filepath.ToSlash(rel)), nil +} + +// bundleFilesRoot returns the deployed path of the bundle's synchronized files, as TranslatePaths +// maps resource paths, or "" for an immutable deployment, whose snapshot path the command does not know. +func bundleFilesRoot(b *bundle.Bundle) string { + switch { + case b.SyncMode() == config.SyncModeImmutable: + return "" + case b.IsSourceLinkedDeployment(): + return b.SyncRootPath + default: + return b.Config.Workspace.FilePath + } +} + +// bundleTestSelectionPath maps a local path, relative to the working directory, to its deployed path +// under filesRoot. The path must exist locally, so that a typo fails before compute starts. +func bundleTestSelectionPath(cwd, syncRoot, filesRoot, localPath string) (string, error) { + if filesRoot == "" { + return "", errors.New("local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path") + } + workspacePath, err := bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath) + if err != nil { + return "", err + } + if !filepath.IsAbs(localPath) { + localPath = filepath.Join(cwd, localPath) + } + if _, err := os.Stat(localPath); errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("%s does not exist; pass a local path or an absolute workspace path", filepath.ToSlash(localPath)) + } else if err != nil { + return "", err + } + return workspacePath, nil +} + +// bundleShortTestID shows a test ID whose path is under filesRoot, the deployed path of the bundle's +// synchronized files, as the local path relative to the working directory, which selects the test again. +// Other test IDs are returned unchanged. +func bundleShortTestID(cwd, syncRoot, filesRoot, id string) string { + rel, testID, ok := cutTestIDFolder(filesRoot, id) + if !ok { + return id + } + local, err := filepath.Rel(cwd, filepath.Join(syncRoot, filepath.FromSlash(rel))) + if err != nil { + return id + } + return filepath.ToSlash(local) + testID +} + +// relativeTestID shows a test ID whose path is under folder relative to it. Other test IDs are returned +// unchanged. Relative to root_path, it is what --pipeline-id resolves to select the test again. +func relativeTestID(folder, id string) string { + rel, testID, ok := cutTestIDFolder(folder, id) + if !ok { + return id + } + return rel + testID +} + +// cutTestIDFolder splits a test ID whose path is under folder into the path relative to folder and the +// "::" suffix that names the test, if any. +func cutTestIDFolder(folder, id string) (rel, testID string, ok bool) { + if folder == "" { + return "", "", false + } + p, suffix, hasSuffix := strings.Cut(id, "::") + rel, ok = strings.CutPrefix(withWorkspacePrefix(p), strings.TrimSuffix(withWorkspacePrefix(folder), "/")+"/") + if !ok { + return "", "", false + } + if hasSuffix { + testID = "::" + suffix + } + return rel, testID, true +} + +// withWorkspacePrefix adds /Workspace to an absolute workspace path without it, as the service does. +func withWorkspacePrefix(p string) string { + if p == "/Workspace" || strings.HasPrefix(p, "/Workspace/") || !strings.HasPrefix(p, "/") { + return p + } + return "/Workspace" + p +} + +// rootPathSelection resolves a relative path against the pipeline's root_path. +func rootPathSelection(rootPath, relPath string) (string, error) { + if rootPath == "" { + return "", errors.New("the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") + } + return path.Join(rootPath, filepath.ToSlash(relPath)), nil +} + +// getRootPath reads the root_path of the deployed pipeline. +func getRootPath(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) (string, error) { + pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineID}) + if err != nil { + return "", err + } + if pipeline.Spec == nil { + return "", nil + } + return pipeline.Spec.RootPath, nil +} + +// updateTestOnly holds UpdateInfo.test_only from a Get Update response; the Go SDK does not have it yet. +// The service leaves it out where it cannot report it, such as before pipeline testing is enabled. +type updateTestOnly struct { + Update struct { + TestOnly *bool `json:"test_only"` + } `json:"update"` +} + +// checkTestUpdate returns an error if the service reports that the update is not a test-only update. +func checkTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipelineID, updateID string) error { + apiClient, err := client.New(w.Config) + if err != nil { + return fmt.Errorf("failed to create API client: %w", err) + } + var response updateTestOnly + err = apiClient.Do(ctx, http.MethodGet, fmt.Sprintf("/api/2.0/pipelines/%s/updates/%s", pipelineID, updateID), auth.WorkspaceIDHeaders(w.Config), nil, nil, &response) + if err != nil { + return err + } + if testOnly := response.Update.TestOnly; testOnly != nil && !*testOnly { + return fmt.Errorf("update %s is not a test update; pass the update ID that pipelines test printed", updateID) + } + return nil +} + +// startTestUpdate starts a test-only update and returns its ID. +func startTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string, req startTestUpdateRequest) (string, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return "", fmt.Errorf("failed to create API client: %w", err) + } + + var response pipelines.StartUpdateResponse + err = apiClient.Do( + ctx, + http.MethodPost, + fmt.Sprintf("/api/2.0/pipelines/%s/updates", pipelineID), + auth.WorkspaceIDHeaders(w.Config), + nil, + req, + &response, + ) + if err != nil { + return "", err + } + return response.UpdateId, nil +} + +// testTarget is the pipeline a test run is submitted to. +type testTarget struct { + key string + pipelineID string + selections []string + w *databricks.WorkspaceClient + // shortID shows a test ID in the shortest form that selects the test again. + shortID func(id string) string + // projectFolder is the deployed folder that names test cases independently of the working + // directory: the bundle's synchronized files, or root_path. It is "" when unknown. + projectFolder string +} + +func (t testTarget) printHeader(out io.Writer, updateID string) { + if t.key != "" { + fmt.Fprintf(out, "Pipeline: %s (%s)\n", t.key, t.pipelineID) + } else { + fmt.Fprintf(out, "Pipeline: %s\n", t.pipelineID) + } + fmt.Fprintf(out, "Update: %s\n", updateID) +} + +// resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. +func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) (testTarget, error) { + if cmd.Flag("target").Value.String() == "" { + cmd.SetContext(root.SkipLoadBundle(cmd.Context())) + } + if err := root.MustWorkspaceClient(cmd, args); err != nil { + return testTarget{}, err + } + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + rootPath, err := getRootPath(ctx, w, pipelineID) + if err != nil { + return testTarget{}, err + } + selections, err := resolveTestSelections(args, func(p string) (string, error) { + return rootPathSelection(rootPath, p) + }) + if err != nil { + return testTarget{}, err + } + shortID := func(id string) string { return relativeTestID(rootPath, id) } + return testTarget{pipelineID: pipelineID, selections: selections, w: w, shortID: shortID, projectFolder: rootPath}, nil +} + +// resolveTestTargetInBundle identifies the pipeline by its bundle KEY, which must precede any selections. +func resolveTestTargetInBundle(cmd *cobra.Command, args []string, resume bool) (testTarget, error) { + var key string + b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ + PostInitFunc: func(ctx context.Context, b *bundle.Bundle) error { + var err error + key, err = resolvePipelineArgument(ctx, b, args[:min(len(args), 1)]) + return err + }, + ErrorOnEmptyState: true, + SkipInitContext: true, + }) + if err != nil { + return testTarget{}, err + } + ctx := cmd.Context() + + if _, err := resources.Lookup(b, key, isPipeline); err != nil { + return testTarget{}, fmt.Errorf("%w; the first argument must be a pipeline KEY when test selections are given", err) + } + pipelineID, err := resolvePipelineIdFromKey(ctx, b, key) + if err != nil { + return testTarget{}, err + } + if !resume { + suggestPipelineDeploy(ctx, cmd) + } + + cwd, err := os.Getwd() + if err != nil { + return testTarget{}, err + } + var selectionArgs []string + if len(args) > 1 { + selectionArgs = args[1:] + } + filesRoot := bundleFilesRoot(b) + selections, err := resolveTestSelections(selectionArgs, func(p string) (string, error) { + return bundleTestSelectionPath(cwd, b.SyncRootPath, filesRoot, p) + }) + if err != nil { + return testTarget{}, err + } + shortID := func(id string) string { return bundleShortTestID(cwd, b.SyncRootPath, filesRoot, id) } + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx), shortID: shortID, projectFolder: filesRoot}, nil +} + +// resumeCommand returns the command that reads the result of the update later. +func (t testTarget) resumeCommand(cmd *cobra.Command, updateID string) string { + parts := []string{cmd.CommandPath()} + if t.key != "" { + parts = append(parts, shellquote.BashArg(t.key)) + } else { + parts = append(parts, "--pipeline-id", shellquote.BashArg(t.pipelineID)) + } + // Quoted so that the command can be pasted into a shell as is. + if target := cmd.Flag("target").Value.String(); target != "" { + parts = append(parts, "--target", shellquote.BashArg(target)) + } + if profile := cmd.Flag("profile").Value.String(); profile != "" { + parts = append(parts, "--profile", shellquote.BashArg(profile)) + } + // Reading the results again replaces the partial report. + if flag := cmd.Flag("junit-xml"); flag != nil && flag.Value.String() != "" { + parts = append(parts, "--junit-xml", shellquote.BashArg(flag.Value.String())) + } + return strings.Join(append(parts, "--resume", updateID), " ") +} + +const testPollInterval = time.Second + +// testWatcher follows a test update and prints its cases as they finish. +type testWatcher struct { + w *databricks.WorkspaceClient + pipelineID string + updateID string + events *testEventReader + printer *testProgressPrinter + run *testRun +} + +func (tw *testWatcher) addEvents(events []testPipelineEvent) { + for _, e := range events { + if c := tw.run.add(e); c != nil { + tw.printer.caseFinished(c) + } + } +} + +// watch polls the update until it ends and returns its final state. +func (tw *testWatcher) watch(ctx context.Context) (pipelines.UpdateInfoState, error) { + lastTimestamp := "" + for { + response, err := tw.w.Pipelines.GetUpdate(ctx, pipelines.GetUpdateRequest{ + PipelineId: tw.pipelineID, + UpdateId: tw.updateID, + }) + if err != nil { + return "", err + } + if response.Update == nil { + return "", fmt.Errorf("no update found with id %s for pipeline %s", tw.updateID, tw.pipelineID) + } + state := response.Update.State + tw.printer.updateState(state) + + // Reading from the last timestamp returns its events again; the run ignores duplicates. + events, err := tw.events.read(ctx, lastTimestamp) + if err != nil { + return "", err + } + if len(events) > 0 { + lastTimestamp = events[len(events)-1].Timestamp + } + tw.addEvents(events) + + switch state { + case pipelines.UpdateInfoStateCompleted, pipelines.UpdateInfoStateFailed, pipelines.UpdateInfoStateCanceled: + // Events can become readable after later ones, so read all of them once more. + events, err := tw.events.read(ctx, "") + if err != nil { + return "", err + } + tw.addEvents(events) + return state, nil + default: + } + + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(testPollInterval): + } + } +} + +// testRunError returns the error the command exits with after the update ended, or nil +// if every collected test passed or was skipped. +func testRunError(updateID string, state pipelines.UpdateInfoState, run *testRun, incompleteReason string) error { + if incompleteReason == "" { + summary := run.summaries[0] + testsFailed := summary.Total == 0 || summary.Failed > 0 || summary.Errors > 0 + if !testsFailed && state == pipelines.UpdateInfoStateCompleted { + return nil + } + // A test update fails when its tests fail; the printed result explains it, unless the + // update ended in another way or reports an error of its own. The service's own verdict, + // such as "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed", is not. + ended := state == pipelines.UpdateInfoStateFailed || state == pipelines.UpdateInfoStateCompleted + if testsFailed && ended && run.fatalError == "" { + return root.ErrAlreadyPrinted + } + } else if state == pipelines.UpdateInfoStateCompleted { + return root.ErrAlreadyPrinted + } + if updateError := run.updateError(); updateError != "" { + return fmt.Errorf("update %s is %s: %s", updateID, state, updateError) + } + return fmt.Errorf("update %s is %s", updateID, state) +} + +// testRunProblem describes why a run that ended did not pass for a reason other than +// failed tests, or returns "" otherwise. +func testRunProblem(run *testRun, incompleteReason string, runErr error) string { + var lines []string + switch { + case incompleteReason != "": + lines = append(lines, fmt.Sprintf("RUN INCOMPLETE: %s.", incompleteReason)) + case run.summaries[0].Total == 0: + lines = append(lines, "No tests were collected.") + } + if runErr != nil && !errors.Is(runErr, root.ErrAlreadyPrinted) { + lines = append(lines, runErr.Error()) + } + return strings.Join(lines, "\n") +} + +var errTestWaitTimeout = errors.New("test wait timed out") + +func testCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...]", + Short: "*Preview* Run the tests of a pipeline", + Long: `Run the tests of a pipeline in a test-only update. + +PIPELINE_KEY is the unique name of the pipeline, as defined in its YAML file. +It is required when test selections are given. If there is only one pipeline +in the project and no arguments are given, it is selected automatically. +Outside a project, identify the pipeline with --pipeline-id; all positional +arguments are then test selections. + +TEST_SELECTION is a test directory, file, or pytest test ID, for example +tests/test_orders.py::test_backfill. In a project, a relative path is a local +path, relative to the working directory, and is converted to its deployed +workspace path. With --pipeline-id, a relative path is relative to the +pipeline's root_path. Paths that start with / are workspace paths. Without a +selection, all of the pipeline's tests run. + +Each --pytest-arg passes one argument to pytest, in the order given. Only -k +and -m are supported, each followed by its value in the next --pytest-arg, for +example: + + databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' + +The command waits for the results and succeeds only if at least one test was +collected and none failed or errored. --no-wait returns after the update starts. +--timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume +to read the results later. To stop the update, run 'pipelines stop'. + +--junit-xml writes one entry per test with its outcome and duration. An +incomplete run, a run without tests, or a timeout adds a failing entry that +names the cause.`, + } + + var pipelineID string + var pytestArgs []string + var noWait bool + var timeout time.Duration + var resumeUpdateID string + var junitPath string + + testGroup := cmdgroup.NewFlagGroup("Pipeline Test") + testGroup.FlagSet().StringVar(&pipelineID, "pipeline-id", "", "ID of the pipeline to test, instead of a PIPELINE_KEY.") + testGroup.FlagSet().StringArrayVar(&pytestArgs, "pytest-arg", nil, "Argument to pass to pytest (repeatable). Only -k and -m are supported.") + testGroup.FlagSet().BoolVar(&noWait, "no-wait", false, "Return after the test update starts, without waiting for results.") + testGroup.FlagSet().DurationVar(&timeout, "timeout", 0, "Stop waiting for results after this duration, for example 30m. The update keeps running.") + testGroup.FlagSet().StringVar(&resumeUpdateID, "resume", "", "Read the results of an existing test update instead of starting one.") + testGroup.FlagSet().StringVar(&junitPath, "junit-xml", "", "Write the results as a JUnit XML report to this file, replacing it.") + + wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) + wrappedCmd.AddFlagGroup(testGroup) + + cmd.SetFlagErrorFunc(testFlagErrorFunc) + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + // An empty --resume, such as from an unset shell variable, must not start a new run. + resume := cmd.Flags().Changed("resume") + checkFlags := func() error { + if cmd.ArgsLenAtDash() >= 0 { + return errors.New("the -- separator is not supported: " + pytestArgHint) + } + if root.OutputType(cmd) != flags.OutputText { + return fmt.Errorf("--output %s is not supported by pipelines test", root.OutputType(cmd)) + } + if err := validatePytestArgs(pytestArgs); err != nil { + return err + } + if timeout < 0 { + return errors.New("--timeout must not be negative") + } + if noWait && timeout > 0 { + return errors.New("--timeout cannot be used with --no-wait") + } + if noWait && junitPath != "" { + return errors.New("--junit-xml cannot be used with --no-wait because no results are available when the command returns") + } + if resume { + selectionCount := len(args) + if pipelineID == "" { + selectionCount = max(len(args)-1, 0) + } + switch { + case noWait: + return errors.New("--resume cannot be used with --no-wait") + case len(pytestArgs) > 0: + return errors.New("--resume cannot be used with --pytest-arg") + case selectionCount > 0: + return errors.New("--resume cannot be used with test selections") + } + if _, err := uuid.Parse(resumeUpdateID); err != nil { + return fmt.Errorf("--resume %q is not an update ID; pass the update ID that pipelines test printed", resumeUpdateID) + } + } + return nil + } + + // Every exit writes the report, so that an older report never survives. + report := &testReport{path: junitPath, out: cmd.OutOrStdout(), suiteName: pipelineID} + if report.suiteName == "" && len(args) > 0 { + report.suiteName = args[0] + } + if err := checkFlags(); err != nil { + return report.failed(cmd.Context(), err) + } + + var target testTarget + var err error + if pipelineID != "" { + target, err = resolveTestTargetByID(cmd, args, pipelineID) + } else { + ctx := logdiag.InitContext(cmd.Context()) + cmd.SetContext(ctx) + if !bundle.HasRoot(ctx) { + return report.failed(ctx, errors.New("outside a project, identify the pipeline with --pipeline-id")) + } + target, err = resolveTestTargetInBundle(cmd, args, resume) + } + if err != nil { + return report.failed(cmd.Context(), err) + } + report.suiteName = target.key + if report.suiteName == "" { + report.suiteName = target.pipelineID + } + report.projectFolder = target.projectFolder + // The CLI root doesn't cancel the context on signals, so stop waiting here. + ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer stop() + out := cmd.OutOrStdout() + + updateID := resumeUpdateID + if !resume { + req := newStartTestUpdateRequest(target.selections, pytestArgs) + updateID, err = startTestUpdate(ctx, target.w, target.pipelineID, req) + if err != nil { + return report.failed(ctx, err) + } + } else if err := checkTestUpdate(ctx, target.w, target.pipelineID, updateID); err != nil { + return report.failed(ctx, err) + } + target.printHeader(out, updateID) + if noWait { + return nil + } + + events, err := newTestEventReader(target.w, target.pipelineID, updateID) + if err != nil { + return report.failed(ctx, err) + } + watcher := &testWatcher{ + w: target.w, + pipelineID: target.pipelineID, + updateID: updateID, + events: events, + printer: &testProgressPrinter{out: out, shortID: target.shortID}, + run: newTestRun(), + } + + waitCtx := ctx + if timeout > 0 { + var cancel context.CancelFunc + waitCtx, cancel = context.WithTimeoutCause(ctx, timeout, errTestWaitTimeout) + defer cancel() + } + state, err := watcher.watch(waitCtx) + if err != nil { + if waitCtx.Err() == nil { + return report.finish(updateID, watcher.run, err.Error(), err) + } + resume := target.resumeCommand(cmd, updateID) + stopErr := fmt.Errorf("stopped waiting for update %s; the update keeps running, to get its results run: %s", updateID, resume) + if errors.Is(context.Cause(waitCtx), errTestWaitTimeout) { + stopErr = fmt.Errorf("stopped waiting for update %s after %s; the update keeps running, to get its results run: %s", updateID, timeout, resume) + } + return report.finish(updateID, watcher.run, stopErr.Error(), stopErr) + } + + incompleteReason := watcher.run.incompleteReason(state) + renderTestResult(out, watcher.run, incompleteReason, target.shortID) + runErr := testRunError(updateID, state, watcher.run, incompleteReason) + return report.finish(updateID, watcher.run, testRunProblem(watcher.run, incompleteReason, runErr), runErr) + } + + return cmd +} diff --git a/cmd/pipelines/test_events.go b/cmd/pipelines/test_events.go new file mode 100644 index 00000000000..4db20555587 --- /dev/null +++ b/cmd/pipelines/test_events.go @@ -0,0 +1,352 @@ +package pipelines + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/databricks/cli/libs/auth" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/pipelines" +) + +const ( + testCaseProgressEventType = "test_case_progress" + testRunSummaryEventType = "test_run_summary" +) + +type testCaseStatus string + +const ( + testCasePending testCaseStatus = "PENDING" + testCaseRunning testCaseStatus = "RUNNING" + testCasePassed testCaseStatus = "PASSED" + testCaseFailed testCaseStatus = "FAILED" + testCaseSkipped testCaseStatus = "SKIPPED" + testCaseError testCaseStatus = "ERROR" +) + +// parseTestCaseStatus accepts both the short status and the proto enum value name. +func parseTestCaseStatus(s string) testCaseStatus { + return testCaseStatus(strings.TrimPrefix(s, "PIPELINE_TEST_CASE_STATUS_")) +} + +func (s testCaseStatus) isFinal() bool { + switch s { + case testCasePassed, testCaseFailed, testCaseSkipped, testCaseError: + return true + default: + return false + } +} + +type testRunSummary struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Skipped int `json:"skipped"` + Errors int `json:"errored"` +} + +// testPipelineEvent is an entry of List Pipeline Events. The Go SDK PipelineEvent has no +// details or origin.test_case_id, so test events are decoded from the raw response. +// w.Pipelines.ListPipelineEvents can replace testEventReader only once the SDK has those +// fields and its iterator stops repeating order_by with page_token, which the service rejects. +type testPipelineEvent struct { + Id string `json:"id"` + EventType string `json:"event_type"` + Timestamp string `json:"timestamp"` + Origin struct { + TestCaseId string `json:"test_case_id"` + } `json:"origin"` + Error *pipelines.ErrorDetail `json:"error"` + Truncation *pipelines.Truncation `json:"truncation"` + Details struct { + TestCaseProgress *struct { + Status string `json:"status"` + } `json:"test_case_progress"` + TestRunSummary *testRunSummary `json:"test_run_summary"` + } `json:"details"` +} + +func (e testPipelineEvent) isTruncated(field string) bool { + if e.Truncation == nil { + return false + } + for _, f := range e.Truncation.TruncatedFields { + if f.FieldName == field { + return true + } + } + return false +} + +// testEventsPageSize is the largest page List Pipeline Events returns. +const testEventsPageSize = 250 + +// testEventReader reads the events of one update. fetchAllPipelineEvents in utils.go is not reused: +// it reads one page into SDK events, which drop the test fields, and a test run needs every page. +type testEventReader struct { + apiClient *client.DatabricksClient + headers map[string]string + pipelineID string + updateID string +} + +func newTestEventReader(w *databricks.WorkspaceClient, pipelineID, updateID string) (*testEventReader, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return nil, fmt.Errorf("failed to create API client: %w", err) + } + return &testEventReader{ + apiClient: apiClient, + headers: auth.WorkspaceIDHeaders(w.Config), + pipelineID: pipelineID, + updateID: updateID, + }, nil +} + +// read returns the update's events in timestamp order, following every page. With +// sinceTimestamp it returns only the events at or after that timestamp, which must be a +// timestamp string exactly as the API returned it. +func (r *testEventReader) read(ctx context.Context, sinceTimestamp string) ([]testPipelineEvent, error) { + path := fmt.Sprintf("/api/2.0/pipelines/%s/events", r.pipelineID) + filter := fmt.Sprintf("update_id='%s'", r.updateID) + if sinceTimestamp != "" { + filter += fmt.Sprintf(" AND timestamp >= '%s'", sinceTimestamp) + } + request := pipelines.ListPipelineEventsRequest{ + Filter: filter, + OrderBy: []string{"timestamp asc"}, + MaxResults: testEventsPageSize, + } + + var events []testPipelineEvent + for { + var response struct { + Events []testPipelineEvent `json:"events"` + NextPageToken string `json:"next_page_token"` + } + err := r.apiClient.Do(ctx, http.MethodGet, path, r.headers, nil, request, &response) + if err != nil { + return nil, fmt.Errorf("failed to read events of update %s: %w", r.updateID, err) + } + events = append(events, response.Events...) + if response.NextPageToken == "" { + return events, nil + } + // A request with page_token must not repeat filter or order_by; the token carries + // them. The SDK iterator repeats them, which is another reason not to use it here. + request = pipelines.ListPipelineEventsRequest{ + PageToken: response.NextPageToken, + MaxResults: testEventsPageSize, + } + } +} + +// testCaseResult is what the events report about one test case. +type testCaseResult struct { + // id is origin.test_case_id: the pytest test ID, relative to pytest's rootdir, such as + // tests/test_orders.py::test_backfill. + id string + status testCaseStatus + // finals counts final outcomes; a complete result has exactly one per case. + finals int + started time.Time + finished time.Time + errorDetail *pipelines.ErrorDetail + detailsTruncated bool +} + +// duration returns the time between the RUNNING and the final event, if both are known. +func (c *testCaseResult) duration() (time.Duration, bool) { + if c.started.IsZero() || c.finished.IsZero() || c.finished.Before(c.started) { + return 0, false + } + return c.finished.Sub(c.started), true +} + +// testRun accumulates the events of a test update, ignoring events it has already seen. +type testRun struct { + seen map[string]bool + cases []*testCaseResult + byID map[string]*testCaseResult + summaries []testRunSummary + // unreadable counts test events whose outcome is missing. + unreadable int + // fatalError is the first fatal error that the update reported, other than the test verdict. + fatalError string + // verdictError is the service's verdict that tests failed, if the update reported it. + verdictError string +} + +// testVerdictErrorPrefix starts the message of the error by which the service reports that tests +// failed, such as "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed (1 passed, 0 skipped).". +const testVerdictErrorPrefix = "[PIPELINE_TEST_FAILED]" + +// isTestVerdict reports whether an update error is the service's verdict that tests failed. +func isTestVerdict(e *pipelines.ErrorDetail) bool { + for _, exception := range e.Exceptions { + if strings.HasPrefix(exception.Message, testVerdictErrorPrefix) { + return true + } + } + return false +} + +func newTestRun() *testRun { + return &testRun{seen: map[string]bool{}, byID: map[string]*testCaseResult{}} +} + +// add records an event and returns the case it gave a final outcome to, if any. +func (r *testRun) add(e testPipelineEvent) *testCaseResult { + if r.seen[e.Id] { + return nil + } + r.seen[e.Id] = true + + if e.Error != nil && e.Error.Fatal { + switch { + case isTestVerdict(e.Error): + r.verdictError = errorDetailText(e.Error) + case r.fatalError == "": + r.fatalError = errorDetailText(e.Error) + } + } + + switch e.EventType { + case testRunSummaryEventType: + if e.Details.TestRunSummary == nil { + r.unreadable++ + return nil + } + r.summaries = append(r.summaries, *e.Details.TestRunSummary) + case testCaseProgressEventType: + progress := e.Details.TestCaseProgress + if progress == nil || e.Origin.TestCaseId == "" { + r.unreadable++ + return nil + } + status := parseTestCaseStatus(progress.Status) + if !status.isFinal() && status != testCasePending && status != testCaseRunning { + r.unreadable++ + return nil + } + + c, ok := r.byID[e.Origin.TestCaseId] + if !ok { + c = &testCaseResult{id: e.Origin.TestCaseId, status: testCasePending} + r.byID[c.id] = c + r.cases = append(r.cases, c) + } + timestamp, _ := time.Parse(time.RFC3339Nano, e.Timestamp) + switch { + case status == testCaseRunning: + c.started = timestamp + if !c.status.isFinal() { + c.status = testCaseRunning + } + case status.isFinal(): + c.finals++ + c.status = status + c.finished = timestamp + c.errorDetail = e.Error + c.detailsTruncated = e.isTruncated("error") + if c.finals == 1 { + return c + } + } + } + return nil +} + +// counts returns the number of cases with each final outcome. +func (r *testRun) counts() testRunSummary { + var s testRunSummary + for _, c := range r.cases { + if !c.status.isFinal() { + continue + } + s.Total++ + switch c.status { + case testCasePassed: + s.Passed++ + case testCaseFailed: + s.Failed++ + case testCaseSkipped: + s.Skipped++ + case testCaseError: + s.Errors++ + default: + } + } + return s +} + +// incompleteReason explains why the events of an ended update are not a complete result, +// or returns "" if they are: one summary and one final outcome per case, matching its counts. +func (r *testRun) incompleteReason(state pipelines.UpdateInfoState) string { + if r.unreadable > 0 { + return "some test results could not be read" + } + switch len(r.summaries) { + case 0: + switch state { + case pipelines.UpdateInfoStateFailed: + return "the update failed before all test results were received" + case pipelines.UpdateInfoStateCanceled: + return "the update was canceled before all test results were received" + default: + return "the update ended without a test summary" + } + case 1: + default: + return fmt.Sprintf("received %d test summaries instead of one", len(r.summaries)) + } + + for _, c := range r.cases { + if c.finals > 1 { + return fmt.Sprintf("test %s has more than one result", c.id) + } + } + summary := r.summaries[0] + received := r.counts() + if received.Total != summary.Total { + return fmt.Sprintf("received %d of %d test results", received.Total, summary.Total) + } + if received != summary { + return "the test results do not match the test summary" + } + for _, c := range r.cases { + if c.finals == 0 { + return fmt.Sprintf("test %s has no result", c.id) + } + } + return "" +} + +// updateError returns the error that the update reported: its own, or else the test verdict. +func (r *testRun) updateError() string { + if r.fatalError != "" { + return r.fatalError + } + return r.verdictError +} + +func errorDetailText(detail *pipelines.ErrorDetail) string { + var parts []string + for _, e := range detail.Exceptions { + switch { + case e.ClassName != "" && e.Message != "": + parts = append(parts, e.ClassName+": "+e.Message) + case e.Message != "": + parts = append(parts, e.Message) + case e.ClassName != "": + parts = append(parts, e.ClassName) + } + } + return strings.Join(parts, "\n") +} diff --git a/cmd/pipelines/test_events_test.go b/cmd/pipelines/test_events_test.go new file mode 100644 index 00000000000..d76cc1efb4f --- /dev/null +++ b/cmd/pipelines/test_events_test.go @@ -0,0 +1,218 @@ +package pipelines + +import ( + "encoding/json" + "fmt" + "net/url" + "strings" + "testing" + "time" + + "github.com/databricks/cli/libs/testserver" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var testEventSeq int + +func caseEvent(t *testing.T, id, status, timestamp string, extra ...string) testPipelineEvent { + testEventSeq++ + return decodeTestEvent(t, fmt.Sprintf(`{ + "id": "e%d", "event_type": "test_case_progress", "timestamp": %q, + "origin": {"test_case_id": %q}, + "details": {"test_case_progress": {"status": %q}}%s + }`, testEventSeq, timestamp, id, status, strings.Join(append([]string{""}, extra...), ","))) +} + +func summaryEvent(t *testing.T, total, passed, failed, skipped, errored int) testPipelineEvent { + testEventSeq++ + return decodeTestEvent(t, fmt.Sprintf(`{ + "id": "e%d", "event_type": "test_run_summary", + "details": {"test_run_summary": {"total": %d, "passed": %d, "failed": %d, "skipped": %d, "errored": %d}} + }`, testEventSeq, total, passed, failed, skipped, errored)) +} + +func decodeTestEvent(t *testing.T, data string) testPipelineEvent { + var e testPipelineEvent + require.NoError(t, json.Unmarshal([]byte(data), &e)) + return e +} + +func TestTestRunAdd(t *testing.T) { + run := newTestRun() + events := []testPipelineEvent{ + caseEvent(t, "tests/test_a.py::test_ok", "PENDING", "2026-01-01T00:00:00.000Z"), + caseEvent(t, "tests/test_a.py::test_ok", "RUNNING", "2026-01-01T00:00:01.000Z"), + caseEvent(t, "tests/test_a.py::test_ok", "PASSED", "2026-01-01T00:00:02.840Z"), + caseEvent(t, "tests/test_a.py::test_bad", "PIPELINE_TEST_CASE_STATUS_FAILED", "2026-01-01T00:00:03.000Z", + `"error": {"exceptions": [{"class_name": "AssertionError", "message": "assert 1 == 2"}]}`, + `"truncation": {"truncated_fields": [{"field_name": "error"}]}`), + // A late RUNNING event does not undo the final outcome. + caseEvent(t, "tests/test_a.py::test_bad", "RUNNING", "2026-01-01T00:00:02.000Z"), + summaryEvent(t, 2, 1, 1, 0, 0), + } + + var finished []string + for _, e := range events { + if c := run.add(e); c != nil { + finished = append(finished, c.id) + } + } + // Reading every page again adds nothing. + for _, e := range events { + assert.Nil(t, run.add(e)) + } + + assert.Equal(t, []string{"tests/test_a.py::test_ok", "tests/test_a.py::test_bad"}, finished) + require.Len(t, run.cases, 2) + + ok := run.cases[0] + assert.Equal(t, testCasePassed, ok.status) + d, known := ok.duration() + assert.True(t, known) + assert.Equal(t, 1840*time.Millisecond, d) + + bad := run.cases[1] + assert.Equal(t, testCaseFailed, bad.status) + assert.True(t, bad.detailsTruncated) + assert.Equal(t, "AssertionError: assert 1 == 2", errorDetailText(bad.errorDetail)) + d, known = bad.duration() + assert.True(t, known) + assert.Equal(t, time.Second, d) + + assert.Empty(t, run.incompleteReason(pipelines.UpdateInfoStateFailed)) +} + +func TestTestRunIncompleteReason(t *testing.T) { + passed := func() testPipelineEvent { + return caseEvent(t, "t::a", "PASSED", "") + } + tests := []struct { + name string + events []testPipelineEvent + state pipelines.UpdateInfoState + want string + }{ + { + name: "no summary after failure", + events: []testPipelineEvent{passed()}, + state: pipelines.UpdateInfoStateFailed, + want: "the update failed before all test results were received", + }, + { + name: "no summary after cancel", + events: nil, + state: pipelines.UpdateInfoStateCanceled, + want: "the update was canceled before all test results were received", + }, + { + name: "no summary after completion", + events: []testPipelineEvent{passed()}, + state: pipelines.UpdateInfoStateCompleted, + want: "the update ended without a test summary", + }, + { + name: "two summaries", + events: []testPipelineEvent{passed(), summaryEvent(t, 1, 1, 0, 0, 0), summaryEvent(t, 1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "received 2 test summaries instead of one", + }, + { + name: "missing outcome", + events: []testPipelineEvent{passed(), caseEvent(t, "t::b", "RUNNING", ""), summaryEvent(t, 2, 2, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "received 1 of 2 test results", + }, + { + name: "case without outcome that the summary leaves out", + events: []testPipelineEvent{passed(), caseEvent(t, "t::b", "RUNNING", ""), summaryEvent(t, 1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "test t::b has no result", + }, + { + name: "two outcomes for one case", + events: []testPipelineEvent{passed(), caseEvent(t, "t::a", "FAILED", ""), summaryEvent(t, 1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "test t::a has more than one result", + }, + { + name: "outcomes differ from summary", + events: []testPipelineEvent{caseEvent(t, "t::a", "FAILED", ""), summaryEvent(t, 1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "the test results do not match the test summary", + }, + { + name: "outcome without details", + events: []testPipelineEvent{ + passed(), + decodeTestEvent(t, `{"id": "truncated", "event_type": "test_case_progress", "origin": {"test_case_id": "t::b"}}`), + summaryEvent(t, 1, 1, 0, 0, 0), + }, + state: pipelines.UpdateInfoStateCompleted, + want: "some test results could not be read", + }, + { + name: "unknown status", + events: []testPipelineEvent{passed(), caseEvent(t, "t::b", "PIPELINE_TEST_CASE_STATUS_UNSPECIFIED", ""), summaryEvent(t, 1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "some test results could not be read", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + run := newTestRun() + for _, e := range tc.events { + run.add(e) + } + assert.Equal(t, tc.want, run.incompleteReason(tc.state)) + }) + } +} + +func TestTestRunFatalError(t *testing.T) { + run := newTestRun() + run.add(decodeTestEvent(t, `{ + "id": "fatal", "event_type": "update_progress", "level": "ERROR", + "error": {"fatal": true, "exceptions": [{"class_name": "ComputeTerminated", "message": "Compute stopped."}]} + }`)) + assert.Equal(t, "ComputeTerminated: Compute stopped.", run.fatalError) +} + +func TestTestEventReaderFollowsPages(t *testing.T) { + server := testserver.New(t) + var queries []url.Values + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/events", func(req testserver.Request) any { + queries = append(queries, req.URL.Query()) + if req.URL.Query().Get("page_token") == "" { + return map[string]any{"events": []map[string]string{{"id": "e1"}}, "next_page_token": "page2"} + } + return map[string]any{"events": []map[string]string{{"id": "e2"}}} + }) + + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + reader, err := newTestEventReader(w, "p1", "u1") + require.NoError(t, err) + + events, err := reader.read(t.Context(), "2026-01-01T00:00:01.000Z") + require.NoError(t, err) + + var ids []string + for _, e := range events { + ids = append(ids, e.Id) + } + assert.Equal(t, []string{"e1", "e2"}, ids) + assert.Equal(t, []url.Values{ + { + "filter": {"update_id='u1' AND timestamp >= '2026-01-01T00:00:01.000Z'"}, + "order_by": {"timestamp asc"}, + "max_results": {"250"}, + }, + { + "page_token": {"page2"}, + "max_results": {"250"}, + }, + }, queries) +} diff --git a/cmd/pipelines/test_junit.go b/cmd/pipelines/test_junit.go new file mode 100644 index 00000000000..75939821082 --- /dev/null +++ b/cmd/pipelines/test_junit.go @@ -0,0 +1,190 @@ +package pipelines + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/logdiag" +) + +type junitTestSuites struct { + XMLName xml.Name `xml:"testsuites"` + Suites []junitTestSuite `xml:"testsuite"` +} + +type junitTestSuite struct { + Name string `xml:"name,attr"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Errors int `xml:"errors,attr"` + Skipped int `xml:"skipped,attr"` + Cases []junitTestCase `xml:"testcase"` +} + +type junitTestCase struct { + ClassName string `xml:"classname,attr,omitempty"` + Name string `xml:"name,attr"` + Time string `xml:"time,attr,omitempty"` + Failure *junitResult `xml:"failure"` + Error *junitResult `xml:"error"` + Skipped *junitResult `xml:"skipped"` +} + +type junitResult struct { + Message string `xml:"message,attr,omitempty"` + Text string `xml:",chardata"` +} + +// junitCaseName splits a pytest test ID the way pytest's JUnit report does: the classname +// is the module path plus any class names, joined by dots. See +// https://docs.pytest.org/en/stable/how-to/output.html#creating-junitxml-format-files. +// A parameter ID in brackets, such as test_cast[a::int], stays with the test name. +func junitCaseName(testID string) (string, string) { + params := "" + if i := strings.Index(testID, "["); i >= 0 { + testID, params = testID[:i], testID[i:] + } + parts := strings.Split(testID, "::") + parts[len(parts)-1] += params + module := strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(parts[0], "/"), ".py"), "/", ".") + if len(parts) == 1 { + return "", module + } + classes := append([]string{module}, parts[1:len(parts)-1]...) + return strings.Join(classes, "."), parts[len(parts)-1] +} + +func newJUnitResult(text string) *junitResult { + message, _, _ := strings.Cut(text, "\n") + return &junitResult{Message: message, Text: text} +} + +// junitNoResult is the error message of a collected case that has no final outcome. +const junitNoResult = "no result received" + +// newJUnitReport has one entry per collected case; a case without a final outcome is an error, +// because CI tools treat skipped entries as passing. +// A non-empty problem, such as an incomplete run, adds a failing entry for the update so the +// report never passes by omission. +// Cases are named relative to projectFolder, as pytest names them, so that the names do not depend on +// the working directory. +func newJUnitReport(suiteName, updateID, projectFolder string, run *testRun, problem string) junitTestSuites { + suite := junitTestSuite{Name: suiteName} + for _, c := range run.cases { + entry := junitTestCase{} + entry.ClassName, entry.Name = junitCaseName(relativeTestID(projectFolder, c.id)) + if d, ok := c.duration(); ok { + entry.Time = fmt.Sprintf("%.3f", d.Seconds()) + } + + details := "" + if c.errorDetail != nil { + details = errorDetailText(c.errorDetail) + } + if c.detailsTruncated { + details = strings.TrimPrefix(details+"\n(details truncated)", "\n") + } + switch c.status { + case testCasePassed: + case testCaseFailed: + entry.Failure = newJUnitResult(details) + suite.Failures++ + case testCaseError: + entry.Error = newJUnitResult(details) + suite.Errors++ + case testCaseSkipped: + entry.Skipped = &junitResult{} + suite.Skipped++ + default: + entry.Error = &junitResult{Message: junitNoResult} + suite.Errors++ + } + suite.Cases = append(suite.Cases, entry) + } + + if problem != "" { + suite.Cases = append(suite.Cases, junitTestCase{ + Name: "update " + updateID, + Error: newJUnitResult(problem), + }) + suite.Errors++ + } + suite.Tests = len(suite.Cases) + return junitTestSuites{Suites: []junitTestSuite{suite}} +} + +// testReport writes the JUnit report of a test run to path, if --junit-xml is set. +type testReport struct { + path string + out io.Writer + suiteName string + projectFolder string +} + +// finish writes the report of a run that ended with runErr. A write error is joined with runErr, +// so that neither is lost. +func (r *testReport) finish(updateID string, run *testRun, problem string, runErr error) error { + if r.path == "" { + return runErr + } + if err := writeJUnitReport(r.path, newJUnitReport(r.suiteName, updateID, r.projectFolder, run, problem)); err != nil { + if runErr == nil || errors.Is(runErr, root.ErrAlreadyPrinted) { + return err + } + return errors.Join(runErr, err) + } + fmt.Fprintf(r.out, "JUnit report: %s\n", filepath.ToSlash(r.path)) + return runErr +} + +// failed writes a report with one failing entry for an error that ended the command before the +// update could be read, so that an older report does not survive. A write error is joined with +// err, as in finish. +func (r *testReport) failed(ctx context.Context, err error) error { + if r.path == "" { + return err + } + message := err.Error() + printed := errors.Is(err, root.ErrAlreadyPrinted) + if printed { + // The error was printed as a diagnostic. Name it instead of the sentinel's text. + message = "the project has errors, see the command output" + if logdiag.IsSetup(ctx) && logdiag.GetFirstErrorSummary(ctx) != "" { + message = logdiag.GetFirstErrorSummary(ctx) + } + } + report := junitTestSuites{Suites: []junitTestSuite{{ + Name: r.suiteName, + Tests: 1, + Errors: 1, + Cases: []junitTestCase{{Name: "pipelines test", Error: newJUnitResult(message)}}, + }}} + if writeErr := writeJUnitReport(r.path, report); writeErr != nil { + if printed { + return writeErr + } + return errors.Join(err, writeErr) + } + return err +} + +// writeJUnitReport replaces the file at path with the report. +func writeJUnitReport(path string, report junitTestSuites) error { + data, err := xml.MarshalIndent(report, "", " ") + if err != nil { + return err + } + data = append([]byte(xml.Header), data...) + data = append(data, '\n') + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("failed to write JUnit report: %w", err) + } + return nil +} diff --git a/cmd/pipelines/test_junit_test.go b/cmd/pipelines/test_junit_test.go new file mode 100644 index 00000000000..ae60f20a582 --- /dev/null +++ b/cmd/pipelines/test_junit_test.go @@ -0,0 +1,139 @@ +package pipelines + +import ( + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/logdiag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJUnitCaseName(t *testing.T) { + tests := []struct { + testID, className, name string + }{ + {"tests/test_orders.py::test_backfill", "tests.test_orders", "test_backfill"}, + {"tests/test_orders.py::TestOrders::test_tax[a-b]", "tests.test_orders.TestOrders", "test_tax[a-b]"}, + // A parameter ID may contain "::". + {"tests/test_a.py::test_cast[a::int]", "tests.test_a", "test_cast[a::int]"}, + {"tests/test_a.py::TestCast::test_cast[a::int]", "tests.test_a.TestCast", "test_cast[a::int]"}, + {"tests/test_orders.py", "", "tests.test_orders"}, + {"/Workspace/Shared/tests/test_x.py::test_x", "Workspace.Shared.tests.test_x", "test_x"}, + } + for _, tc := range tests { + className, name := junitCaseName(tc.testID) + assert.Equal(t, tc.className, className, tc.testID) + assert.Equal(t, tc.name, name, tc.testID) + } +} + +func TestWriteJUnitReport(t *testing.T) { + // Cases are named relative to the project folder, as pytest names them, wherever the command runs. + const project = "/Workspace/Users/someone/.bundle/orders/dev/files" + run := newTestRun() + for _, e := range []testPipelineEvent{ + caseEvent(t, project+"/tests/test_a.py::test_ok", "RUNNING", "2026-01-01T00:00:00.000Z"), + caseEvent(t, project+"/tests/test_a.py::test_ok", "PASSED", "2026-01-01T00:00:01.840Z"), + caseEvent(t, project+"/tests/test_a.py::test_bad", "FAILED", "", + `"error": {"exceptions": [{"class_name": "AssertionError", "message": "assert 1 == 2\nwhere 1 = f()"}]}`, + `"truncation": {"truncated_fields": [{"field_name": "error"}]}`), + caseEvent(t, project+"/tests/test_a.py::test_setup", "ERROR", "", + `"error": {"exceptions": [{"class_name": "RuntimeError", "message": "fixture failed"}]}`), + caseEvent(t, project+"/tests/test_a.py::test_old", "SKIPPED", ""), + caseEvent(t, project+"/tests/test_a.py::test_unfinished", "RUNNING", ""), + } { + run.add(e) + } + + path := filepath.Join(t.TempDir(), "report.xml") + require.NoError(t, os.WriteFile(path, []byte("previous report"), 0o644)) + require.NoError(t, writeJUnitReport(path, newJUnitReport("orders", "u1", project, run, "RUN INCOMPLETE: received 4 of 5 test results."))) + + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, ` + + + + + AssertionError: assert 1 == 2 where 1 = f() (details truncated) + + + RuntimeError: fixture <db> failed + + + + + + + + + RUN INCOMPLETE: received 4 of 5 test results. + + + +`, string(data)) +} + +func TestTestRunProblem(t *testing.T) { + empty := newTestRun() + empty.add(summaryEvent(t, 0, 0, 0, 0, 0)) + assert.Equal(t, "No tests were collected.", testRunProblem(empty, "", root.ErrAlreadyPrinted)) + + failed := newTestRun() + failed.add(caseEvent(t, "t::a", "FAILED", "")) + failed.add(summaryEvent(t, 1, 0, 1, 0, 0)) + assert.Empty(t, testRunProblem(failed, "", root.ErrAlreadyPrinted)) + + passed := newTestRun() + passed.add(caseEvent(t, "t::a", "PASSED", "")) + passed.add(summaryEvent(t, 1, 1, 0, 0, 0)) + assert.Equal(t, "update u1 is FAILED", testRunProblem(passed, "", errors.New("update u1 is FAILED"))) + + assert.Equal(t, "RUN INCOMPLETE: the update was canceled before all test results were received.\nupdate u1 is CANCELED", + testRunProblem(newTestRun(), "the update was canceled before all test results were received", errors.New("update u1 is CANCELED"))) +} + +func TestTestReportJoinsWriteErrors(t *testing.T) { + report := &testReport{path: filepath.Join(t.TempDir(), "missing", "report.xml"), out: io.Discard, suiteName: "orders"} + runErr := errors.New("update u1 is FAILED") + err := report.finish("u1", newTestRun(), runErr.Error(), runErr) + require.ErrorIs(t, err, runErr) + assert.ErrorContains(t, err, "failed to write JUnit report") +} + +func TestTestReportReplacesAStaleReportOnError(t *testing.T) { + path := filepath.Join(t.TempDir(), "report.xml") + require.NoError(t, os.WriteFile(path, []byte(""), 0o644)) + report := &testReport{path: path, out: io.Discard, suiteName: "orders"} + startErr := errors.New("An active update already exists.") + assert.Equal(t, startErr, report.failed(t.Context(), startErr)) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(data), ``) + assert.Contains(t, string(data), ``) +} + +func TestTestReportNamesAPrintedError(t *testing.T) { + ctx := logdiag.InitContext(t.Context()) + logdiag.SetCollect(ctx, true) + logdiag.LogDiag(ctx, diag.Diagnostic{Severity: diag.Error, Summary: "reference does not exist: ${var.missing}"}) + path := filepath.Join(t.TempDir(), "report.xml") + report := &testReport{path: path, out: io.Discard, suiteName: "orders"} + assert.Equal(t, root.ErrAlreadyPrinted, report.failed(ctx, root.ErrAlreadyPrinted)) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(data), ``) + + // The sentinel would hide a joined write error, so the write error is returned alone. + report.path = filepath.Join(t.TempDir(), "missing", "report.xml") + err = report.failed(ctx, root.ErrAlreadyPrinted) + assert.NotErrorIs(t, err, root.ErrAlreadyPrinted) + assert.ErrorContains(t, err, "failed to write JUnit report") +} diff --git a/cmd/pipelines/test_render.go b/cmd/pipelines/test_render.go new file mode 100644 index 00000000000..c16c83689c9 --- /dev/null +++ b/cmd/pipelines/test_render.go @@ -0,0 +1,135 @@ +package pipelines + +import ( + "fmt" + "io" + "strings" + + "github.com/databricks/databricks-sdk-go/service/pipelines" +) + +// testProgressPrinter prints the phase of a test update and each case as it finishes. +type testProgressPrinter struct { + out io.Writer + shortID func(id string) string + waiting bool + running bool +} + +func (p *testProgressPrinter) updateState(state pipelines.UpdateInfoState) { + switch state { + case pipelines.UpdateInfoStateQueued, + pipelines.UpdateInfoStateCreated, + pipelines.UpdateInfoStateWaitingForResources, + pipelines.UpdateInfoStateInitializing, + pipelines.UpdateInfoStateResetting, + pipelines.UpdateInfoStateSettingUpTables: + if !p.waiting && !p.running { + fmt.Fprintln(p.out, "Waiting for compute...") + p.waiting = true + } + case pipelines.UpdateInfoStateRunning: + p.startRunning() + default: + } +} + +func (p *testProgressPrinter) startRunning() { + if !p.running { + fmt.Fprintln(p.out, "Running tests...") + p.running = true + } +} + +func (p *testProgressPrinter) caseFinished(c *testCaseResult) { + p.startRunning() + if d, ok := c.duration(); ok { + fmt.Fprintf(p.out, "%s %s (%.2fs)\n", p.shortID(c.id), c.status, d.Seconds()) + } else { + fmt.Fprintf(p.out, "%s %s\n", p.shortID(c.id), c.status) + } +} + +// renderTestFailures prints the details of every failed or errored case. +func renderTestFailures(out io.Writer, run *testRun, shortID func(id string) string) { + var failed []*testCaseResult + for _, c := range run.cases { + if c.status == testCaseFailed || c.status == testCaseError { + failed = append(failed, c) + } + } + if len(failed) == 0 { + return + } + + fmt.Fprintln(out) + fmt.Fprintln(out, "FAILURES") + for _, c := range failed { + fmt.Fprintln(out, shortID(c.id)) + details := "" + if c.errorDetail != nil { + details = errorDetailText(c.errorDetail) + } + if details == "" { + details = "No details were reported." + } + for line := range strings.SplitSeq(details, "\n") { + fmt.Fprintf(out, " %s\n", line) + } + if c.detailsTruncated { + fmt.Fprintln(out, " (details truncated)") + } + } +} + +func errorsLabel(n int) string { + if n == 1 { + return "error" + } + return "errors" +} + +func formatTestCounts(s testRunSummary) string { + return fmt.Sprintf("%d passed, %d failed, %d skipped, %d %s", s.Passed, s.Failed, s.Skipped, s.Errors, errorsLabel(s.Errors)) +} + +// renderTestResult prints the end of a test run: failure details, then the summary or why +// the result is incomplete. +func renderTestResult(out io.Writer, run *testRun, incompleteReason string, shortID func(id string) string) { + renderTestFailures(out, run, shortID) + fmt.Fprintln(out) + + if incompleteReason != "" { + fmt.Fprintf(out, "RUN INCOMPLETE: %s.\n", incompleteReason) + received := run.counts() + var parts []string + for _, count := range []struct { + n int + label string + }{ + {received.Passed, "passed"}, + {received.Failed, "failed"}, + {received.Skipped, "skipped"}, + {received.Errors, errorsLabel(received.Errors)}, + } { + if count.n > 0 { + parts = append(parts, fmt.Sprintf("%d %s", count.n, count.label)) + } + } + if len(parts) == 0 { + parts = []string{"none"} + } + fmt.Fprintf(out, "Results received: %s.\n", strings.Join(parts, ", ")) + return + } + + summary := run.summaries[0] + switch summary.Total { + case 0: + fmt.Fprintln(out, "No tests were collected.") + case 1: + fmt.Fprintf(out, "1 test: %s\n", formatTestCounts(summary)) + default: + fmt.Fprintf(out, "%d tests: %s\n", summary.Total, formatTestCounts(summary)) + } +} diff --git a/cmd/pipelines/test_render_test.go b/cmd/pipelines/test_render_test.go new file mode 100644 index 00000000000..7ca6aab7bfc --- /dev/null +++ b/cmd/pipelines/test_render_test.go @@ -0,0 +1,149 @@ +package pipelines + +import ( + "bytes" + "strings" + "testing" + + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" +) + +func sameTestID(id string) string { return id } + +func TestRenderShortTestIDs(t *testing.T) { + shortID := func(id string) string { return strings.TrimPrefix(id, "/Workspace/root/") } + var out bytes.Buffer + p := &testProgressPrinter{out: &out, shortID: shortID} + run := newTestRun() + p.caseFinished(run.add(caseEvent(t, "/Workspace/root/t.py::test_x", "FAILED", "2026-01-01T00:00:00.000Z"))) + renderTestFailures(&out, run, shortID) + + assert.Equal(t, `Running tests... +t.py::test_x FAILED + +FAILURES +t.py::test_x + No details were reported. +`, out.String()) +} + +func TestTestProgressPrinter(t *testing.T) { + var out bytes.Buffer + p := &testProgressPrinter{out: &out, shortID: sameTestID} + run := newTestRun() + + p.updateState(pipelines.UpdateInfoStateQueued) + p.updateState(pipelines.UpdateInfoStateWaitingForResources) + run.add(caseEvent(t, "t::timed", "RUNNING", "2026-01-01T00:00:00.000Z")) + p.caseFinished(run.add(caseEvent(t, "t::timed", "PASSED", "2026-01-01T00:01:15.300Z"))) + p.updateState(pipelines.UpdateInfoStateRunning) + p.caseFinished(run.add(caseEvent(t, "t::untimed", "SKIPPED", "2026-01-01T00:01:16.000Z"))) + + assert.Equal(t, `Waiting for compute... +Running tests... +t::timed PASSED (75.30s) +t::untimed SKIPPED +`, out.String()) +} + +func TestRenderTestResult(t *testing.T) { + tests := []struct { + name string + events func() []testPipelineEvent + state pipelines.UpdateInfoState + want string + }{ + { + name: "failures", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + caseEvent(t, "t::ok", "PASSED", ""), + caseEvent(t, "t::bad", "FAILED", "", + `"error": {"exceptions": [{"class_name": "AssertionError", "message": "assert 1 == 2\nwhere 1 = f()"}]}`, + `"truncation": {"truncated_fields": [{"field_name": "error"}]}`), + caseEvent(t, "t::setup", "ERROR", ""), + summaryEvent(t, 3, 1, 1, 0, 1), + } + }, + state: pipelines.UpdateInfoStateFailed, + want: ` +FAILURES +t::bad + AssertionError: assert 1 == 2 + where 1 = f() + (details truncated) +t::setup + No details were reported. + +3 tests: 1 passed, 1 failed, 0 skipped, 1 error +`, + }, + { + name: "several errors", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + caseEvent(t, "t::setup", "ERROR", ""), + caseEvent(t, "t::teardown", "ERROR", ""), + summaryEvent(t, 2, 0, 0, 0, 2), + } + }, + state: pipelines.UpdateInfoStateFailed, + want: ` +FAILURES +t::setup + No details were reported. +t::teardown + No details were reported. + +2 tests: 0 passed, 0 failed, 0 skipped, 2 errors +`, + }, + { + name: "incomplete", + events: func() []testPipelineEvent { + return []testPipelineEvent{caseEvent(t, "t::ok", "PASSED", ""), caseEvent(t, "t::skip", "SKIPPED", "")} + }, + state: pipelines.UpdateInfoStateFailed, + want: ` +RUN INCOMPLETE: the update failed before all test results were received. +Results received: 1 passed, 1 skipped. +`, + }, + { + name: "incomplete with one error", + events: func() []testPipelineEvent { + return []testPipelineEvent{caseEvent(t, "t::setup", "ERROR", "")} + }, + state: pipelines.UpdateInfoStateFailed, + want: ` +FAILURES +t::setup + No details were reported. + +RUN INCOMPLETE: the update failed before all test results were received. +Results received: 1 error. +`, + }, + { + name: "incomplete without results", + events: func() []testPipelineEvent { return nil }, + state: pipelines.UpdateInfoStateCanceled, + want: ` +RUN INCOMPLETE: the update was canceled before all test results were received. +Results received: none. +`, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + run := newTestRun() + for _, e := range tc.events() { + run.add(e) + } + var out bytes.Buffer + renderTestResult(&out, run, run.incompleteReason(tc.state), sameTestID) + assert.Equal(t, tc.want, out.String()) + }) + } +} diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go new file mode 100644 index 00000000000..7a7e23e23f1 --- /dev/null +++ b/cmd/pipelines/test_test.go @@ -0,0 +1,406 @@ +package pipelines + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/cmd/root" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidatePytestArgs(t *testing.T) { + tests := []struct { + args []string + wantErr string + }{ + {args: nil}, + {args: []string{"-k", "orders and not slow", "-m", "smoke,fast"}}, + {args: []string{"-korders"}, wantErr: `pytest argument "-korders" must be split: pass --pytest-arg=-k --pytest-arg=orders`}, + {args: []string{"-msmoke,fast"}, wantErr: `pytest argument "-msmoke,fast" must be split: pass --pytest-arg=-m --pytest-arg='smoke,fast'`}, + {args: []string{"-k=orders and not slow"}, wantErr: `pytest argument "-k=orders and not slow" must be split: pass --pytest-arg=-k --pytest-arg='orders and not slow'`}, + {args: []string{"-k", " "}, wantErr: "pytest argument -k requires a non-empty value"}, + {args: []string{"-k", "-m"}, wantErr: `pytest argument -k requires a value, not "-m": pass it in the next --pytest-arg`}, + {args: []string{"-m", "--maxfail=1"}, wantErr: `pytest argument -m requires a value, not "--maxfail=1": pass it in the next --pytest-arg`}, + {args: []string{"-k"}, wantErr: "pytest argument -k requires a value: pass it in the next --pytest-arg"}, + {args: []string{"-m", "smoke", "-m"}, wantErr: "pytest argument -m requires a value: pass it in the next --pytest-arg"}, + {args: []string{"-x"}, wantErr: `unsupported pytest argument "-x": only -k and -m are supported`}, + {args: []string{"--maxfail=1"}, wantErr: `unsupported pytest argument "--maxfail=1": only -k and -m are supported`}, + {args: []string{"tests/test_orders.py"}, wantErr: `unsupported pytest argument "tests/test_orders.py": pass test paths and test IDs as TEST_SELECTION arguments`}, + } + for _, tc := range tests { + err := validatePytestArgs(tc.args) + if tc.wantErr == "" { + assert.NoError(t, err, tc.args) + } else { + assert.EqualError(t, err, tc.wantErr, tc.args) + } + } +} + +// TestValidatePytestArgsRejectsUnsupportedOptions covers each form of option that the API rejects. +func TestValidatePytestArgsRejectsUnsupportedOptions(t *testing.T) { + for _, arg := range []string{"--collect-only", "-n", "--rootdir=/", "--junit-xml=out.xml"} { + err := validatePytestArgs([]string{arg}) + assert.EqualError(t, err, `unsupported pytest argument "`+arg+`": only -k and -m are supported`) + } +} + +func TestResolveTestSelections(t *testing.T) { + toWorkspacePath := func(p string) (string, error) { + if p == "missing" { + return "", errors.New("not found") + } + return "/Workspace/root/" + p, nil + } + + resolved, err := resolveTestSelections([]string{ + "tests", + "tests/test_orders.py::TestOrders::test_backfill[a::b]", + "/Workspace/Users/someone/../someone/tests/test_x.py::test_x", + }, toWorkspacePath) + require.NoError(t, err) + assert.Equal(t, []string{ + "/Workspace/root/tests", + "/Workspace/root/tests/test_orders.py::TestOrders::test_backfill[a::b]", + "/Workspace/Users/someone/tests/test_x.py::test_x", + }, resolved) + + _, err = resolveTestSelections([]string{"::test_x"}, toWorkspacePath) + assert.EqualError(t, err, `test selection "::test_x" has no path`) + + _, err = resolveTestSelections([]string{"missing::test_x"}, toWorkspacePath) + assert.EqualError(t, err, `cannot resolve test selection "missing::test_x": not found`) +} + +func TestBundleWorkspacePath(t *testing.T) { + syncRoot := t.TempDir() + bundleRoot := filepath.Join(syncRoot, "orders") + filePath := "/Workspace/Users/someone/.bundle/orders/dev/files" + + got, err := bundleWorkspacePath(bundleRoot, syncRoot, filePath, filepath.Join("tests", "test_orders.py")) + require.NoError(t, err) + assert.Equal(t, filePath+"/orders/tests/test_orders.py", got) + + got, err = bundleWorkspacePath(filepath.Join(bundleRoot, "tests"), syncRoot, filePath, filepath.Join("..", "conftest.py")) + require.NoError(t, err) + assert.Equal(t, filePath+"/orders/conftest.py", got) + + got, err = bundleWorkspacePath(syncRoot, syncRoot, filePath, ".") + require.NoError(t, err) + assert.Equal(t, filePath, got) + + _, err = bundleWorkspacePath(syncRoot, syncRoot, filePath, filepath.Join("..", "tests")) + assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+syncRoot) +} + +func TestBundleShortTestID(t *testing.T) { + syncRoot := t.TempDir() + filesRoot := "/Workspace/Users/someone/.bundle/orders/dev/files" + id := filesRoot + "/tests/test_a.py::TestA::test_x[a::b]" + + // The local path, relative to where the user stands, selects the test again. + assert.Equal(t, "tests/test_a.py::TestA::test_x[a::b]", bundleShortTestID(syncRoot, syncRoot, filesRoot, id)) + assert.Equal(t, "test_a.py::TestA::test_x[a::b]", bundleShortTestID(filepath.Join(syncRoot, "tests"), syncRoot, filesRoot, id)) + assert.Equal(t, "../tests/test_a.py::TestA::test_x[a::b]", bundleShortTestID(filepath.Join(syncRoot, "src"), syncRoot, filesRoot, id)) + assert.Equal(t, "tests", bundleShortTestID(syncRoot, syncRoot, filesRoot, filesRoot+"/tests")) + + // Other IDs are shown as the service sends them. + for _, other := range []string{ + "/Workspace/Users/someone/tests/test_s.py::test_s", + filesRoot + "2/tests/test_a.py::test_x", + "tests/test_a.py::test_x", + } { + assert.Equal(t, other, bundleShortTestID(syncRoot, syncRoot, filesRoot, other)) + } + assert.Equal(t, id, bundleShortTestID(syncRoot, syncRoot, "", id)) +} + +func TestRelativeTestID(t *testing.T) { + rootPath := "/Workspace/Users/someone/orders" + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID(rootPath, rootPath+"/tests/test_a.py::test_x")) + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID(rootPath+"/", rootPath+"/tests/test_a.py::test_x")) + // The service accepts workspace paths with or without the /Workspace prefix. + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID("/Users/someone/orders", rootPath+"/tests/test_a.py::test_x")) + for _, other := range []string{ + "/Workspace/Users/someone/shared/test_s.py::test_s", + rootPath + "2/test_a.py::test_x", + "tests/test_a.py::test_x", + } { + assert.Equal(t, other, relativeTestID(rootPath, other)) + } + assert.Equal(t, rootPath+"/tests/test_a.py::test_x", relativeTestID("", rootPath+"/tests/test_a.py::test_x")) +} + +func TestRootPathSelection(t *testing.T) { + got, err := rootPathSelection("/Workspace/Users/someone/orders", "tests/test_orders.py") + require.NoError(t, err) + assert.Equal(t, "/Workspace/Users/someone/orders/tests/test_orders.py", got) + + _, err = rootPathSelection("", "tests") + assert.EqualError(t, err, "the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") +} + +func TestBundleFilesRoot(t *testing.T) { + filePath := "/Workspace/Users/someone/.bundle/orders/dev/files" + syncRoot := "/Workspace/Users/someone/orders" + for _, tc := range []struct { + mode config.SyncMode + want string + }{ + {mode: config.SyncModeOverwrite, want: filePath}, + // Source-linked deployments run the source files, as TranslatePaths maps them. + {mode: config.SyncModeInPlace, want: syncRoot}, + // Immutable deployments run a snapshot, whose path the command does not know. + {mode: config.SyncModeImmutable, want: ""}, + } { + b := &bundle.Bundle{SyncRootPath: syncRoot} + b.Config.Workspace.FilePath = filePath + b.Config.Workspace.SyncMode = tc.mode + assert.Equal(t, tc.want, bundleFilesRoot(b), tc.mode) + } +} + +func TestBundleTestSelectionPath(t *testing.T) { + syncRoot := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(syncRoot, "pipeline", "tests"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(syncRoot, "pipeline", "tests", "test_a.py"), nil, 0o644)) + filesRoot := "/Workspace/Users/someone/.bundle/orders/dev/files" + + // A local path maps to its deployed path, relative to where the user stands. + got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, filesRoot, "tests/test_a.py") + require.NoError(t, err) + assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) + + got, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "pipeline/tests") + require.NoError(t, err) + assert.Equal(t, filesRoot+"/pipeline/tests", got) + + // A path that does not exist locally is an error, so that a typo fails before compute starts. + _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "tests/test_a.py") + assert.EqualError(t, err, filepath.ToSlash(filepath.Join(syncRoot, "tests", "test_a.py"))+" does not exist; pass a local path or an absolute workspace path") + + // A path outside the synchronized files is an error. The error shows forward slashes. + _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, filepath.Join("..", "tests")) + assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+filepath.ToSlash(syncRoot)) + + // When the deployed path of local files is unknown, a local path is an error. + _, err = bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, "", "tests/test_a.py") + assert.EqualError(t, err, "local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path") +} + +func TestNewStartTestUpdateRequest(t *testing.T) { + tests := []struct { + selection []string + pytestArgs []string + want string + }{ + {want: `{"test_only":true}`}, + { + selection: []string{"/Workspace/tests"}, + want: `{"test_only":true,"test_options":{"test_selection":["/Workspace/tests"]}}`, + }, + { + pytestArgs: []string{"-m", "smoke,fast"}, + want: `{"test_only":true,"test_options":{"pytest_args":["-m","smoke,fast"]}}`, + }, + } + for _, tc := range tests { + body, err := json.Marshal(newStartTestUpdateRequest(tc.selection, tc.pytestArgs)) + require.NoError(t, err) + assert.JSONEq(t, tc.want, string(body)) + } +} + +func TestTestTargetResumeCommand(t *testing.T) { + tests := []struct { + name string + target testTarget + flags map[string]string + want string + }{ + { + name: "pipeline key", + target: testTarget{key: "my_pipeline", pipelineID: "p1"}, + want: "databricks pipelines test my_pipeline --resume u1", + }, + { + name: "pipeline key with target and profile", + target: testTarget{key: "my_pipeline", pipelineID: "p1"}, + flags: map[string]string{"target": "dev", "profile": "staging"}, + want: "databricks pipelines test my_pipeline --target dev --profile staging --resume u1", + }, + { + name: "names that need quoting", + target: testTarget{key: "my_pipeline", pipelineID: "p1"}, + flags: map[string]string{"target": "dev west", "profile": "CI West"}, + want: "databricks pipelines test my_pipeline --target 'dev west' --profile 'CI West' --resume u1", + }, + { + name: "JUnit report", + target: testTarget{key: "my_pipeline", pipelineID: "p1"}, + flags: map[string]string{"junit-xml": "reports/test results.xml"}, + want: "databricks pipelines test my_pipeline --junit-xml 'reports/test results.xml' --resume u1", + }, + { + name: "pipeline ID with profile", + target: testTarget{pipelineID: "p1"}, + flags: map[string]string{"profile": "staging"}, + want: "databricks pipelines test --pipeline-id p1 --profile staging --resume u1", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + rootCmd := &cobra.Command{Use: "databricks"} + rootCmd.PersistentFlags().StringP("profile", "p", "", "") + rootCmd.PersistentFlags().StringP("target", "t", "", "") + pipelinesCmd := &cobra.Command{Use: "pipelines"} + cmd := &cobra.Command{Use: "test"} + cmd.Flags().String("junit-xml", "", "") + rootCmd.AddCommand(pipelinesCmd) + pipelinesCmd.AddCommand(cmd) + for name, value := range tc.flags { + require.NoError(t, cmd.Flag(name).Value.Set(value)) + } + assert.Equal(t, tc.want, tc.target.resumeCommand(cmd, "u1")) + }) + } +} + +func TestCheckTestUpdate(t *testing.T) { + const updateID = "11111111-1111-1111-1111-111111111111" + tests := []struct { + name string + update string + wantErr string + }{ + // Where the service cannot report test_only, as before pipeline testing is enabled, the + // update may be a test update, so the command reads its results. + {name: "test_only not reported", update: `{"state": "COMPLETED"}`}, + {name: "test update", update: `{"state": "COMPLETED", "test_only": true}`}, + { + name: "regular update", + update: `{"state": "COMPLETED", "test_only": false}`, + wantErr: "update " + updateID + " is not a test update; pass the update ID that pipelines test printed", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.0/pipelines/p1/updates/"+updateID { + _, _ = w.Write([]byte(`{"update": ` + tc.update + `}`)) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: srv.URL, Token: "token"}) + require.NoError(t, err) + + err = checkTestUpdate(t.Context(), w, "p1", updateID) + if tc.wantErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.wantErr) + } + }) + } +} + +func TestTestRunError(t *testing.T) { + tests := []struct { + name string + events func() []testPipelineEvent + state pipelines.UpdateInfoState + wantErr string + }{ + { + name: "passed", + events: func() []testPipelineEvent { + return []testPipelineEvent{caseEvent(t, "t::a", "PASSED", ""), summaryEvent(t, 1, 1, 0, 0, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + }, + { + name: "incomplete after completion", + events: func() []testPipelineEvent { + return []testPipelineEvent{caseEvent(t, "t::a", "PASSED", "")} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "update failed with an error of its own and then the service's verdict", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + caseEvent(t, "t::a", "FAILED", ""), + summaryEvent(t, 1, 0, 1, 0, 0), + decodeTestEvent(t, `{"id": "own", "error": {"fatal": true, "exceptions": [{"message": "Commit failed."}]}}`), + decodeTestEvent(t, `{"id": "verdict", "error": {"fatal": true, "exceptions": [{"class_name": "TestFailure", "message": "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed."}]}}`), + } + }, + state: pipelines.UpdateInfoStateFailed, + wantErr: "update u1 is FAILED: Commit failed.", + }, + { + name: "update failed with the service's verdict although no test failed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + caseEvent(t, "t::a", "PASSED", ""), + summaryEvent(t, 1, 1, 0, 0, 0), + decodeTestEvent(t, `{"id": "fatal", "error": {"fatal": true, "exceptions": [{"message": "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed."}]}}`), + } + }, + state: pipelines.UpdateInfoStateFailed, + wantErr: "update u1 is FAILED: [PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed.", + }, + { + name: "tests failed although the update completed", + events: func() []testPipelineEvent { + return []testPipelineEvent{caseEvent(t, "t::a", "FAILED", ""), summaryEvent(t, 1, 0, 1, 0, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "no tests collected although the update completed", + events: func() []testPipelineEvent { + return []testPipelineEvent{summaryEvent(t, 0, 0, 0, 0, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "canceled after tests failed", + events: func() []testPipelineEvent { + return []testPipelineEvent{caseEvent(t, "t::a", "FAILED", ""), summaryEvent(t, 1, 0, 1, 0, 0)} + }, + state: pipelines.UpdateInfoStateCanceled, + wantErr: "update u1 is CANCELED", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + run := newTestRun() + for _, e := range tc.events() { + run.add(e) + } + err := testRunError("u1", tc.state, run, run.incompleteReason(tc.state)) + if tc.wantErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.wantErr) + } + }) + } +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index dd5050f6f49..b1a28872475 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -216,7 +216,10 @@ type FakeWorkspace struct { JobRunOutputs map[int64]jobs.RunOutput JobRunIdempotency map[string]int64 Pipelines map[string]pipelines.GetPipelineResponse - PipelineUpdates map[string]bool + PipelineUpdates map[string]*fakePipelineUpdate + pipelineEvents []fakePipelineEvent + pipelineTesting map[string]json.RawMessage + pipelineEventTime time.Time Monitors map[string]catalog.MonitorInfo Apps map[string]apps.App Schemas map[string]catalog.SchemaInfo @@ -516,7 +519,9 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { JobRunIdempotency: map[string]int64{}, Grants: map[string][]catalog.PrivilegeAssignment{}, Pipelines: map[string]pipelines.GetPipelineResponse{}, - PipelineUpdates: map[string]bool{}, + PipelineUpdates: map[string]*fakePipelineUpdate{}, + pipelineEventTime: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + pipelineTesting: map[string]json.RawMessage{}, Monitors: map[string]catalog.MonitorInfo{}, Apps: map[string]apps.App{}, Catalogs: map[string]catalog.CatalogInfo{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 50ff904a573..f42a74636e6 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -648,11 +648,11 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("POST", "/api/2.0/pipelines/{pipeline_id}/updates", func(req Request) any { - return req.Workspace.PipelineStartUpdate(req.Vars["pipeline_id"]) + return req.Workspace.PipelineStartUpdate(req.Vars["pipeline_id"], req.Body) }) server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/events", func(req Request) any { - return req.Workspace.PipelineEvents(req.Vars["pipeline_id"]) + return req.Workspace.PipelineEvents(req.Vars["pipeline_id"], req.URL.Query()) }) server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/updates/{update_id}", func(req Request) any { diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 0aedc44c5d3..d5488f17bc7 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -18,6 +18,13 @@ func (s *FakeWorkspace) PipelineGet(pipelineId string) Response { Body: map[string]string{"message": fmt.Sprintf("The specified pipeline %s was not found.", pipelineId)}, } } + if testing, ok := s.pipelineTesting[pipelineId]; ok { + body, err := withPipelineTesting(value, testing) + if err != nil { + return Response{StatusCode: 500, Body: fmt.Sprintf("internal error: %s", err)} + } + return Response{Body: body} + } return Response{ Body: value, } @@ -80,6 +87,7 @@ func (s *FakeWorkspace) PipelineCreate(req Request) Response { setSpecDefaults(&spec, pipelineId) s.Pipelines[pipelineId] = r + s.setPipelineTesting(pipelineId, req.Body) return Response{ Body: pipelines.CreatePipelineResponse{ @@ -139,45 +147,81 @@ func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response } setSpecDefaults(&spec, pipelineId) s.Pipelines[pipelineId] = item + s.setPipelineTesting(pipelineId, req.Body) return Response{} } -func (s *FakeWorkspace) PipelineStartUpdate(pipelineId string) Response { +func (s *FakeWorkspace) PipelineStartUpdate(pipelineId string, body []byte) Response { defer s.LockUnlock()() - _, exists := s.Pipelines[pipelineId] - if !exists { + if _, exists := s.Pipelines[pipelineId]; !exists { return Response{ StatusCode: 404, Body: map[string]string{"message": fmt.Sprintf("The specified pipeline %s was not found.", pipelineId)}, } } - updateId := nextUUID() - s.PipelineUpdates[updateId] = true - - return Response{ - Body: pipelines.StartUpdateResponse{ - UpdateId: updateId, - }, + for id, update := range s.PipelineUpdates { + if update.pipelineID == pipelineId && update.active() { + return Response{ + StatusCode: 409, + Body: map[string]string{ + "error_code": "INVALID_STATE_TRANSITION", + "message": fmt.Sprintf("An active update '%s' already exists for pipeline '%s'.", id, pipelineId), + }, + } + } } -} -func (s *FakeWorkspace) PipelineEvents(pipelineId string) Response { - defer s.LockUnlock()() - - _, exists := s.Pipelines[pipelineId] - if !exists { + var test pipelineTestRequest + if len(body) > 0 { + if err := json.Unmarshal(body, &test); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("cannot unmarshal request body: %s", err), + } + } + } + if test.TestOptions != nil && !test.TestOnly { return Response{ - StatusCode: 404, - Body: map[string]string{"message": fmt.Sprintf("The specified pipeline %s was not found.", pipelineId)}, + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": "test_options requires test_only to be true."}, + } + } + if test.TestOptions != nil { + if message := pipelineTestOptionsError(test.TestOptions.PytestArgs, test.TestOptions.TestSelection); message != "" { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, + } + } + } + + updateId := nextUUID() + update := &fakePipelineUpdate{ + pipelineID: pipelineId, + states: []pipelines.UpdateInfoState{pipelines.UpdateInfoStateCompleted}, + } + if test.TestOnly { + var selection []string + if test.TestOptions != nil { + selection = test.TestOptions.TestSelection + } + targets, message := pipelineTestTargets(s.pipelineTestSources(pipelineId), selection) + if message != "" { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, + } } + update = s.startPipelineTestRun(pipelineId, updateId, s.collectPipelineTests(targets)) } + s.PipelineUpdates[updateId] = update return Response{ - Body: map[string]any{ - "events": []pipelines.PipelineEvent{}, + Body: pipelines.StartUpdateResponse{ + UpdateId: updateId, }, } } @@ -193,25 +237,38 @@ func (s *FakeWorkspace) PipelineGetUpdate(pipelineId, updateId string) Response } } - // Check if the update exists - _, updateExists := s.PipelineUpdates[updateId] - if !updateExists { + update, updateExists := s.PipelineUpdates[updateId] + if !updateExists || update.pipelineID != pipelineId { return Response{ StatusCode: 404, Body: map[string]string{"message": fmt.Sprintf("The specified update %s was not found.", updateId)}, } } + update.polls++ return Response{ - Body: pipelines.GetUpdateResponse{ - Update: &pipelines.UpdateInfo{ + Body: fakeGetUpdateResponse{ + Update: fakeUpdateInfo{ UpdateId: updateId, - State: pipelines.UpdateInfoStateCompleted, + State: update.states[update.stateIndex()], + TestOnly: update.testOnly, }, }, } } +// fakeGetUpdateResponse is a Get Update response with UpdateInfo.test_only, which the Go SDK does not have +// yet. Like the service where pipeline testing is enabled, it reports test_only for every update. +type fakeGetUpdateResponse struct { + Update fakeUpdateInfo `json:"update"` +} + +type fakeUpdateInfo struct { + UpdateId string `json:"update_id"` + State pipelines.UpdateInfoState `json:"state"` + TestOnly bool `json:"test_only"` +} + func (s *FakeWorkspace) PipelineStop(pipelineId string) Response { defer s.LockUnlock()() @@ -223,6 +280,13 @@ func (s *FakeWorkspace) PipelineStop(pipelineId string) Response { } } + // Stop is pipeline-wide: it cancels whichever update is active. + for _, update := range s.PipelineUpdates { + if update.pipelineID == pipelineId && update.active() { + update.cancel() + } + } + return Response{ Body: pipelines.GetPipelineResponse{ PipelineId: pipelineId, diff --git a/libs/testserver/pipelines_test.go b/libs/testserver/pipelines_test.go index 39114714082..d8a9391c6d5 100644 --- a/libs/testserver/pipelines_test.go +++ b/libs/testserver/pipelines_test.go @@ -51,7 +51,7 @@ func TestPipelineCreate_AllowsSingleSegmentTargetSchemaName(t *testing.T) { func TestPipelineStartUpdate_HandlesNonExistentPipeline(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") - response := workspace.PipelineStartUpdate("non-existent-pipeline") + response := workspace.PipelineStartUpdate("non-existent-pipeline", nil) assert.Equal(t, 404, response.StatusCode) assert.Contains(t, response.Body.(map[string]string)["message"], "The specified pipeline non-existent-pipeline was not found") } @@ -74,7 +74,7 @@ func TestPipelineStop_AfterUpdate(t *testing.T) { pipelineId := createTestPipeline(t, workspace) - startResponse := workspace.PipelineStartUpdate(pipelineId) + startResponse := workspace.PipelineStartUpdate(pipelineId, nil) assert.Equal(t, 0, startResponse.StatusCode) stopResponse := workspace.PipelineStop(pipelineId) diff --git a/libs/testserver/pipelines_testing.go b/libs/testserver/pipelines_testing.go new file mode 100644 index 00000000000..0cff5fe5890 --- /dev/null +++ b/libs/testserver/pipelines_testing.go @@ -0,0 +1,481 @@ +package testserver + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/url" + "path" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "github.com/databricks/databricks-sdk-go/service/pipelines" +) + +// Pipeline event types, which also name the event's details field. +const ( + pipelineEventUpdateProgress = "update_progress" + pipelineEventTestCaseProgress = "test_case_progress" + pipelineEventTestRunSummary = "test_run_summary" +) + +// fakePipelineUpdate is a pipeline update whose state advances by one step on each Get Update call. +type fakePipelineUpdate struct { + pipelineID string + testOnly bool + states []pipelines.UpdateInfoState + polls int + + // canceled is set by cancel. CANCELED then holds the index of a state the update never reached. + canceled bool +} + +// stateIndex returns the index of the state returned by the last Get Update call, or -1 before the +// first call. +func (u *fakePipelineUpdate) stateIndex() int { + return min(u.polls, len(u.states)) - 1 +} + +// eventIndex returns the index of the last state whose events are visible. Events emitted for a +// later state are not visible yet, and never are for a state that a canceled update did not reach. +func (u *fakePipelineUpdate) eventIndex() int { + if u.canceled { + return len(u.states) - 2 + } + return u.stateIndex() +} + +// active reports whether the update has not ended yet: the state of the last Get Update call, or +// the first state before that call, is not terminal. Like the service, an update that will end on +// its own is active until it does. +func (u *fakePipelineUpdate) active() bool { + return !isTerminalUpdateState(u.states[max(u.stateIndex(), 0)]) +} + +// cancel ends the update at its current state, as Stop does. CANCELED is the update's state from +// now on, even before the next Get Update call. +func (u *fakePipelineUpdate) cancel() { + u.states = append(u.states[:u.stateIndex()+1], pipelines.UpdateInfoStateCanceled) + u.polls = len(u.states) + u.canceled = true +} + +func isTerminalUpdateState(state pipelines.UpdateInfoState) bool { + switch state { + case pipelines.UpdateInfoStateCompleted, pipelines.UpdateInfoStateFailed, pipelines.UpdateInfoStateCanceled: + return true + default: + return false + } +} + +type fakePipelineEventOrigin struct { + PipelineId string `json:"pipeline_id"` + UpdateId string `json:"update_id"` + TestCaseId string `json:"test_case_id,omitempty"` +} + +// fakePipelineEvent is a pipeline event as returned by List Pipeline Events. The Go SDK +// PipelineEvent has no details or origin.test_case_id, so the fake defines its own shape. +type fakePipelineEvent struct { + Id string `json:"id"` + EventType string `json:"event_type"` + Timestamp string `json:"timestamp"` + Level pipelines.EventLevel `json:"level"` + Message string `json:"message"` + Origin fakePipelineEventOrigin `json:"origin"` + Error *pipelines.ErrorDetail `json:"error,omitempty"` + Details map[string]any `json:"details"` + + // visibleAt is the index of the update state from which the event can be read. + visibleAt int +} + +// pipelineTestRequest holds the Start Update fields of a test run, which are not in the Go SDK yet. +type pipelineTestRequest struct { + TestOnly bool `json:"test_only"` + TestOptions *struct { + TestSelection []string `json:"test_selection"` + PytestArgs []string `json:"pytest_args"` + } `json:"test_options"` +} + +var pipelineTestFunctionRegexp = regexp.MustCompile(`(?m)^def (test_\w+)\(`) + +// pipelineTestOptionsError returns the service's message for test options that it rejects, or "". Like +// PipelineTestSelection in the service, pytest_args holds -k and -m, each followed by a value that is not +// empty and does not start with "-", and each test selection is an absolute workspace path. +func pipelineTestOptionsError(pytestArgs, selection []string) string { + for i := 0; i < len(pytestArgs); i += 2 { + option := pytestArgs[i] + switch { + case option != "-k" && option != "-m": + return fmt.Sprintf("pytest_args accepts only -k and -m, each followed by its value. Got '%s'.", option) + case i+1 == len(pytestArgs): + return fmt.Sprintf("The pytest_args option '%s' must be followed by its value.", option) + case strings.TrimSpace(pytestArgs[i+1]) == "": + return fmt.Sprintf("The value of the pytest_args option '%s' must not be empty.", option) + case strings.HasPrefix(pytestArgs[i+1], "-"): + return fmt.Sprintf("The value of the pytest_args option '%s' must not start with '-'. Got '%s'.", option, pytestArgs[i+1]) + } + } + for _, entry := range selection { + if p, _, _ := strings.Cut(strings.TrimSpace(entry), "::"); !strings.HasPrefix(p, "/") { + return fmt.Sprintf("The test selection '%s' must be an absolute workspace path, such as /Workspace/Users/someone@example.com/orders/tests/test_orders.py.", entry) + } + } + return "" +} + +// pipelineTestingRequest holds the testing field of a Create or Edit request, which is not in the +// Go SDK yet. +type pipelineTestingRequest struct { + Testing json.RawMessage `json:"testing"` +} + +// pipelineTesting is the part of the testing field that test-only updates read. +type pipelineTesting struct { + Sources []struct { + Glob *struct { + Include string `json:"include"` + } `json:"glob"` + } `json:"sources"` +} + +// setPipelineTesting records the testing field of a Create or Edit body, so that Get returns it. +// Like the rest of the spec, an Edit without testing removes it. +func (s *FakeWorkspace) setPipelineTesting(pipelineID string, body []byte) { + var request pipelineTestingRequest + _ = json.Unmarshal(body, &request) + if len(request.Testing) == 0 || string(request.Testing) == "null" { + delete(s.pipelineTesting, pipelineID) + return + } + s.pipelineTesting[pipelineID] = request.Testing +} + +// pipelineTestSources returns the glob includes of the pipeline's testing.sources. +func (s *FakeWorkspace) pipelineTestSources(pipelineID string) []string { + var testing pipelineTesting + _ = json.Unmarshal(s.pipelineTesting[pipelineID], &testing) + var sources []string + for _, source := range testing.Sources { + if source.Glob != nil { + sources = append(sources, source.Glob.Include) + } + } + return sources +} + +// withPipelineTesting adds spec.testing to a Get Pipeline response, which the Go SDK type cannot +// hold. The JSON is merged as raw values, so numbers and the other fields are kept as they are. +func withPipelineTesting(value pipelines.GetPipelineResponse, testing json.RawMessage) (map[string]json.RawMessage, error) { + data, err := json.Marshal(value) + if err != nil { + return nil, err + } + var body map[string]json.RawMessage + if err := json.Unmarshal(data, &body); err != nil { + return nil, err + } + var spec map[string]json.RawMessage + if err := json.Unmarshal(body["spec"], &spec); err != nil { + return nil, err + } + spec["testing"] = testing + body["spec"], err = json.Marshal(spec) + return body, err +} + +// pipelineTestTargets returns the paths to collect tests from: the selection, which must be +// within the test sources, or the test sources themselves. A non-empty message rejects the request. +func pipelineTestTargets(sources, selection []string) ([]string, string) { + if len(sources) == 0 { + return nil, "The pipeline has no test sources. Set `testing.sources` to the test files or folders to run." + } + if len(selection) == 0 { + var targets []string + for _, source := range sources { + targets = append(targets, strings.TrimSuffix(source, "/**")) + } + return targets, "" + } + for _, target := range selection { + targetPath, _, _ := strings.Cut(target, "::") + if !slices.ContainsFunc(sources, func(source string) bool { + folder, recursive := strings.CutSuffix(source, "/**") + return targetPath == folder || (recursive && strings.HasPrefix(targetPath, folder+"/")) + }) { + return nil, fmt.Sprintf("The test selection '%s' is outside the pipeline's test sources. Select a file or folder inside testing.sources.", target) + } + } + return selection, "" +} + +// collectPipelineTests stands in for pytest collection: it finds the top-level "def test_*" +// functions in the workspace test_*.py files under each target. Like origin.test_case_id, a test ID +// is the absolute workspace path of its file followed by the pytest test ID, such as +// /Workspace/Users/someone/orders/tests/test_orders.py::test_backfill. +func (s *FakeWorkspace) collectPipelineTests(targets []string) []string { + var ids []string + for _, target := range targets { + targetPath, testName, _ := strings.Cut(target, "::") + var files []string + if _, ok := s.files[targetPath]; ok { + files = []string{targetPath} + } else { + for filePath := range s.files { + base := path.Base(filePath) + if strings.HasPrefix(filePath, targetPath+"/") && strings.HasPrefix(base, "test_") && strings.HasSuffix(base, ".py") { + files = append(files, filePath) + } + } + slices.Sort(files) + } + + for _, filePath := range files { + for _, match := range pipelineTestFunctionRegexp.FindAllSubmatch(s.files[filePath].Data, -1) { + name := string(match[1]) + id := filePath + "::" + name + if (testName == "" || testName == name) && !slices.Contains(ids, id) { + ids = append(ids, id) + } + } + } + } + return ids +} + +// startPipelineTestRun records the events of a test-only update. Tests do not run here; each +// test name selects its outcome: +// - "fail": FAILED, "error": ERROR (a fixture failed), "skip": SKIPPED, otherwise PASSED. +// - "crash": compute terminates when the case starts and the update fails without a summary. +// - "hang": the case never finishes and the update stays RUNNING until the pipeline is stopped. +// - "breaks_update": the case passes, but the update fails after the summary. +func (s *FakeWorkspace) startPipelineTestRun(pipelineID, updateID string, caseIDs []string) *fakePipelineUpdate { + const ( + running = 1 + ended = 2 + ) + update := &fakePipelineUpdate{ + pipelineID: pipelineID, + testOnly: true, + states: []pipelines.UpdateInfoState{pipelines.UpdateInfoStateWaitingForResources, pipelines.UpdateInfoStateRunning}, + } + + emit := func(visibleAt int, event fakePipelineEvent) { + s.pipelineEventTime = s.pipelineEventTime.Add(time.Second) + event.Id = nextUUID() + event.Timestamp = s.pipelineEventTime.UTC().Format("2006-01-02T15:04:05.000Z") + event.Origin.PipelineId = pipelineID + event.Origin.UpdateId = updateID + if event.Level == "" { + event.Level = pipelines.EventLevelInfo + } + event.visibleAt = visibleAt + s.pipelineEvents = append(s.pipelineEvents, event) + } + emitCase := func(id, status string, errorDetail *pipelines.ErrorDetail) { + event := fakePipelineEvent{ + EventType: pipelineEventTestCaseProgress, + Message: fmt.Sprintf("Test case %s is %s.", id, status), + Origin: fakePipelineEventOrigin{TestCaseId: id}, + Error: errorDetail, + Details: map[string]any{pipelineEventTestCaseProgress: map[string]any{"status": status}}, + } + if errorDetail != nil { + event.Level = pipelines.EventLevelError + } + emit(running, event) + } + exception := func(className, message string) *pipelines.ErrorDetail { + return &pipelines.ErrorDetail{Exceptions: []pipelines.SerializedException{{ClassName: className, Message: message}}} + } + + for _, id := range caseIDs { + emitCase(id, "PENDING", nil) + } + + counts := map[string]int{} + breaksUpdate := false + for _, id := range caseIDs { + emitCase(id, "RUNNING", nil) + _, name, _ := strings.Cut(id, "::") + switch { + case strings.Contains(name, "hang"): + return update + case strings.Contains(name, "crash"): + emit(ended, fakePipelineEvent{ + EventType: pipelineEventUpdateProgress, + Level: pipelines.EventLevelError, + Message: fmt.Sprintf("Update %s is FAILED.", updateID[:6]), + Error: &pipelines.ErrorDetail{Fatal: true, Exceptions: []pipelines.SerializedException{{ClassName: "ComputeTerminatedException", Message: "Compute terminated unexpectedly."}}}, + Details: map[string]any{pipelineEventUpdateProgress: map[string]any{"state": "FAILED"}}, + }) + update.states = append(update.states, pipelines.UpdateInfoStateFailed) + return update + case strings.Contains(name, "breaks_update"): + emitCase(id, "PASSED", nil) + counts["passed"]++ + breaksUpdate = true + case strings.Contains(name, "fail"): + emitCase(id, "FAILED", exception("AssertionError", "assert 100 == 150")) + counts["failed"]++ + case strings.Contains(name, "error"): + emitCase(id, "ERROR", exception("RuntimeError", "fixture 'orders' setup failed")) + counts["errors"]++ + case strings.Contains(name, "skip"): + emitCase(id, "SKIPPED", nil) + counts["skipped"]++ + default: + emitCase(id, "PASSED", nil) + counts["passed"]++ + } + } + + emit(ended, fakePipelineEvent{ + EventType: pipelineEventTestRunSummary, + Message: fmt.Sprintf("Collected %d tests.", len(caseIDs)), + Details: map[string]any{pipelineEventTestRunSummary: map[string]any{ + "total": len(caseIDs), + "passed": counts["passed"], + "failed": counts["failed"], + "skipped": counts["skipped"], + "errored": counts["errors"], + }}, + }) + // Like the service, a run whose tests failed or errored ends with the test verdict as the + // update's error. + if !breaksUpdate && (counts["failed"] > 0 || counts["errors"] > 0) { + failed := counts["failed"] + counts["errors"] + emit(ended, fakePipelineEvent{ + EventType: pipelineEventUpdateProgress, + Level: pipelines.EventLevelError, + Message: fmt.Sprintf("%d pipeline tests failed.", failed), + Error: &pipelines.ErrorDetail{Fatal: true, Exceptions: []pipelines.SerializedException{{ + ClassName: "TestFailure", + Message: fmt.Sprintf("[PIPELINE_TEST_FAILED] %d of the pipeline's tests failed (%d passed, %d skipped).", failed, counts["passed"], counts["skipped"]), + }}}, + Details: map[string]any{pipelineEventUpdateProgress: map[string]any{"state": "FAILED"}}, + }) + } + if breaksUpdate { + emit(ended, fakePipelineEvent{ + EventType: pipelineEventUpdateProgress, + Level: pipelines.EventLevelError, + Message: fmt.Sprintf("Update %s is FAILED.", updateID[:6]), + Error: &pipelines.ErrorDetail{Fatal: true, Exceptions: []pipelines.SerializedException{{ClassName: "ResultDeliveryException", Message: "Test results could not be verified."}}}, + Details: map[string]any{pipelineEventUpdateProgress: map[string]any{"state": "FAILED"}}, + }) + } + if len(caseIDs) == 0 || counts["failed"] > 0 || counts["errors"] > 0 || breaksUpdate { + update.states = append(update.states, pipelines.UpdateInfoStateFailed) + } else { + update.states = append(update.states, pipelines.UpdateInfoStateCompleted) + } + return update +} + +var ( + pipelineEventsUpdateIDFilter = regexp.MustCompile(`update_id\s*=\s*'([^']*)'`) + pipelineEventsTimestampFilter = regexp.MustCompile(`timestamp\s*>=\s*'([^']*)'`) +) + +// pipelineEventsPageToken is the query of the first page plus the offset of the next one. +type pipelineEventsPageToken struct { + Query url.Values `json:"query"` + Offset int `json:"offset"` +} + +// PipelineEvents lists the visible events of a pipeline. Only the update_id and timestamp >= +// conditions of the filter are applied. +func (s *FakeWorkspace) PipelineEvents(pipelineId string, query url.Values) Response { + defer s.LockUnlock()() + + _, exists := s.Pipelines[pipelineId] + if !exists { + return Response{ + StatusCode: 404, + Body: map[string]string{"message": fmt.Sprintf("The specified pipeline %s was not found.", pipelineId)}, + } + } + + invalid := func(message string) Response { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, + } + } + + maxResults := 25 + if value := query.Get("max_results"); value != "" { + n, err := strconv.Atoi(value) + if err != nil || n <= 0 { + return invalid("max_results must be a positive integer.") + } + maxResults = min(n, 250) + } + + offset := 0 + if token := query.Get("page_token"); token != "" { + for key := range query { + if key != "page_token" && key != "max_results" { + return invalid(fmt.Sprintf("page_token cannot be used with %s.", key)) + } + } + data, err := base64.URLEncoding.DecodeString(token) + var decoded pipelineEventsPageToken + if err == nil { + err = json.Unmarshal(data, &decoded) + } + if err != nil { + return invalid("Invalid page_token.") + } + query = decoded.Query + offset = decoded.Offset + } + + updateID := "" + if match := pipelineEventsUpdateIDFilter.FindStringSubmatch(query.Get("filter")); match != nil { + updateID = match[1] + } + var since time.Time + if match := pipelineEventsTimestampFilter.FindStringSubmatch(query.Get("filter")); match != nil { + var err error + since, err = time.Parse(time.RFC3339Nano, match[1]) + if err != nil { + return invalid(fmt.Sprintf("Invalid timestamp in filter: %s.", match[1])) + } + } + + var events []fakePipelineEvent + for _, event := range s.pipelineEvents { + update := s.PipelineUpdates[event.Origin.UpdateId] + if update.pipelineID != pipelineId || event.visibleAt > update.eventIndex() { + continue + } + if updateID != "" && event.Origin.UpdateId != updateID { + continue + } + if timestamp, _ := time.Parse(time.RFC3339Nano, event.Timestamp); timestamp.Before(since) { + continue + } + events = append(events, event) + } + + // Events are returned newest first unless ascending order is requested. + if query.Get("order_by") != "timestamp asc" { + slices.Reverse(events) + } + + page := events[min(offset, len(events)):] + body := map[string]any{"events": append([]fakePipelineEvent{}, page[:min(maxResults, len(page))]...)} + if len(page) > maxResults { + data, _ := json.Marshal(pipelineEventsPageToken{Query: query, Offset: offset + maxResults}) + body["next_page_token"] = base64.URLEncoding.EncodeToString(data) + } + return Response{Body: body} +} diff --git a/libs/testserver/pipelines_testing_test.go b/libs/testserver/pipelines_testing_test.go new file mode 100644 index 00000000000..a73b8310350 --- /dev/null +++ b/libs/testserver/pipelines_testing_test.go @@ -0,0 +1,280 @@ +package testserver + +import ( + "encoding/json" + "net/url" + "testing" + + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testRootPath = "/Workspace/Users/someone/orders" + +func createTestPipelineWithTests(t *testing.T, workspace *FakeWorkspace) string { + response := workspace.PipelineCreate(Request{Body: []byte(`{"name": "orders", "testing": {"sources": [{"glob": {"include": "` + testRootPath + `/tests/**"}}]}}`)}) + require.Equal(t, 0, response.StatusCode) + workspace.files[testRootPath+"/tests/test_orders.py"] = FileEntry{Data: []byte("def test_ok():\n pass\n\ndef test_fails():\n pass\n\n def test_nested():\n pass\n")} + workspace.files[testRootPath+"/tests/test_slow.py"] = FileEntry{Data: []byte("def test_hang():\n pass\n")} + workspace.files[testRootPath+"/tests/helpers.py"] = FileEntry{Data: []byte("def test_helper():\n pass\n")} + return response.Body.(pipelines.CreatePipelineResponse).PipelineId +} + +func startTestUpdate(t *testing.T, workspace *FakeWorkspace, pipelineID, body string) string { + response := workspace.PipelineStartUpdate(pipelineID, []byte(body)) + require.Equal(t, 0, response.StatusCode, response.Body) + return response.Body.(pipelines.StartUpdateResponse).UpdateId +} + +func getUpdateState(workspace *FakeWorkspace, pipelineID, updateID string) pipelines.UpdateInfoState { + return workspace.PipelineGetUpdate(pipelineID, updateID).Body.(fakeGetUpdateResponse).Update.State +} + +func TestPipelineGetUpdateReportsTestOnly(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + + testUpdateID := startTestUpdate(t, workspace, pipelineID, `{"test_only": true}`) + assert.True(t, workspace.PipelineGetUpdate(pipelineID, testUpdateID).Body.(fakeGetUpdateResponse).Update.TestOnly) + + workspace.PipelineStop(pipelineID) + updateID := startTestUpdate(t, workspace, pipelineID, `{}`) + assert.False(t, workspace.PipelineGetUpdate(pipelineID, updateID).Body.(fakeGetUpdateResponse).Update.TestOnly) +} + +func listEvents(t *testing.T, workspace *FakeWorkspace, pipelineID string, query url.Values) ([]fakePipelineEvent, string) { + response := workspace.PipelineEvents(pipelineID, query) + require.Equal(t, 0, response.StatusCode, response.Body) + body := response.Body.(map[string]any) + token, _ := body["next_page_token"].(string) + return body["events"].([]fakePipelineEvent), token +} + +func TestCollectPipelineTests(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + createTestPipelineWithTests(t, workspace) + + // Like origin.test_case_id, an ID is the absolute workspace path of its file and the pytest test ID. + tests := testRootPath + "/tests/" + assert.Equal(t, []string{tests + "test_orders.py::test_ok", tests + "test_orders.py::test_fails", tests + "test_slow.py::test_hang"}, + workspace.collectPipelineTests([]string{testRootPath + "/tests"})) + assert.Equal(t, []string{tests + "test_orders.py::test_fails"}, + workspace.collectPipelineTests([]string{testRootPath + "/tests/test_orders.py::test_fails", testRootPath + "/tests/test_orders.py::test_fails"})) + assert.Equal(t, []string{tests + "helpers.py::test_helper"}, + workspace.collectPipelineTests([]string{testRootPath + "/tests/helpers.py"})) + assert.Empty(t, workspace.collectPipelineTests(nil)) +} + +func TestPipelineTestTargets(t *testing.T) { + sources := []string{"/Workspace/orders/tests/**", "/Workspace/orders/checks/test_x.py"} + + targets, message := pipelineTestTargets(sources, nil) + assert.Empty(t, message) + assert.Equal(t, []string{"/Workspace/orders/tests", "/Workspace/orders/checks/test_x.py"}, targets) + + selection := []string{"/Workspace/orders/tests/unit/test_a.py::test_a", "/Workspace/orders/tests", "/Workspace/orders/checks/test_x.py::test_x"} + targets, message = pipelineTestTargets(sources, selection) + assert.Empty(t, message) + assert.Equal(t, selection, targets) + + _, message = pipelineTestTargets(sources, []string{"/Workspace/orders/transformations/orders.py"}) + // The messages are the service's. + assert.Equal(t, "The test selection '/Workspace/orders/transformations/orders.py' is outside the pipeline's test sources. Select a file or folder inside testing.sources.", message) + _, message = pipelineTestTargets(sources, []string{"/Workspace/orders/tests2/test_a.py"}) + assert.NotEmpty(t, message) + _, message = pipelineTestTargets(nil, nil) + assert.Equal(t, "The pipeline has no test sources. Set `testing.sources` to the test files or folders to run.", message) +} + +func TestPipelineEditReplacesTestSources(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + assert.Equal(t, []string{testRootPath + "/tests/**"}, workspace.pipelineTestSources(pipelineID)) + body, err := json.Marshal(workspace.PipelineGet(pipelineID).Body) + require.NoError(t, err) + assert.Contains(t, string(body), `"testing":{"sources":[{"glob":{"include":"`+testRootPath+`/tests/**"}}]}`) + + response := workspace.PipelineUpdate(Request{Body: []byte(`{"name": "orders"}`)}, pipelineID) + require.Equal(t, 0, response.StatusCode) + assert.Empty(t, workspace.pipelineTestSources(pipelineID)) + body, err = json.Marshal(workspace.PipelineGet(pipelineID).Body) + require.NoError(t, err) + assert.NotContains(t, string(body), `"testing"`) + assert.Equal(t, 400, workspace.PipelineStartUpdate(pipelineID, []byte(`{"test_only": true}`)).StatusCode) +} + +func TestPipelineTestUpdate(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + updateID := startTestUpdate(t, workspace, pipelineID, `{"test_only": true, "test_options": {"test_selection": ["`+testRootPath+`/tests/test_orders.py"]}}`) + query := url.Values{"filter": {"update_id='" + updateID + "'"}, "order_by": {"timestamp asc"}} + + events, _ := listEvents(t, workspace, pipelineID, query) + assert.Empty(t, events) + + assert.Equal(t, pipelines.UpdateInfoStateWaitingForResources, getUpdateState(workspace, pipelineID, updateID)) + assert.Equal(t, pipelines.UpdateInfoStateRunning, getUpdateState(workspace, pipelineID, updateID)) + events, _ = listEvents(t, workspace, pipelineID, query) + var statuses []string + for _, e := range events { + statuses = append(statuses, e.Origin.TestCaseId+" "+e.Details["test_case_progress"].(map[string]any)["status"].(string)) + } + file := testRootPath + "/tests/test_orders.py" + assert.Equal(t, []string{ + file + "::test_ok PENDING", + file + "::test_fails PENDING", + file + "::test_ok RUNNING", + file + "::test_ok PASSED", + file + "::test_fails RUNNING", + file + "::test_fails FAILED", + }, statuses) + + assert.Equal(t, pipelines.UpdateInfoStateFailed, getUpdateState(workspace, pipelineID, updateID)) + last := events[len(events)-1] + query.Set("filter", "update_id='"+updateID+"' AND timestamp >= '"+last.Timestamp+"'") + events, _ = listEvents(t, workspace, pipelineID, query) + require.Len(t, events, 3) + assert.Equal(t, last.Id, events[0].Id) + assert.Equal(t, map[string]any{"total": 2, "passed": 1, "failed": 1, "skipped": 0, "errored": 0}, events[1].Details["test_run_summary"]) + // Like the service, the update's error is the test verdict. + require.NotNil(t, events[2].Error) + assert.True(t, events[2].Error.Fatal) + assert.Equal(t, "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed (1 passed, 0 skipped).", events[2].Error.Exceptions[0].Message) + + query.Set("filter", "timestamp >= 'yesterday'") + assert.Equal(t, 400, workspace.PipelineEvents(pipelineID, query).StatusCode) +} + +func TestPipelineEventsPages(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + updateID := startTestUpdate(t, workspace, pipelineID, `{"test_only": true}`) + for range 3 { + getUpdateState(workspace, pipelineID, updateID) + } + + all, token := listEvents(t, workspace, pipelineID, url.Values{"order_by": {"timestamp asc"}, "max_results": {"100"}}) + assert.Empty(t, token) + require.Len(t, all, 8) + + var paged []fakePipelineEvent + query := url.Values{"filter": {"update_id='" + updateID + "'"}, "order_by": {"timestamp asc"}, "max_results": {"3"}} + for { + events, token := listEvents(t, workspace, pipelineID, query) + paged = append(paged, events...) + if token == "" { + break + } + query = url.Values{"page_token": {token}, "max_results": {"3"}} + } + assert.Equal(t, all, paged) + + // Events are newest first by default. + newest, _ := listEvents(t, workspace, pipelineID, url.Values{"max_results": {"1"}}) + assert.Equal(t, all[len(all)-1].Id, newest[0].Id) + + response := workspace.PipelineEvents(pipelineID, url.Values{"page_token": {"x"}, "filter": {"level='INFO'"}}) + assert.Equal(t, 400, response.StatusCode) +} + +func TestPipelineTestUpdateConflictAndStop(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + updateID := startTestUpdate(t, workspace, pipelineID, `{"test_only": true, "test_options": {"test_selection": ["`+testRootPath+`/tests/test_slow.py"]}}`) + + response := workspace.PipelineStartUpdate(pipelineID, nil) + assert.Equal(t, 409, response.StatusCode) + assert.Equal(t, "An active update '"+updateID+"' already exists for pipeline '"+pipelineID+"'.", response.Body.(map[string]string)["message"]) + + workspace.PipelineStop(pipelineID) + for range 2 { + getUpdateState(workspace, pipelineID, updateID) + } + assert.Equal(t, pipelines.UpdateInfoStateCanceled, getUpdateState(workspace, pipelineID, updateID)) + startTestUpdate(t, workspace, pipelineID, `{}`) +} + +func TestPipelineStartUpdateRejectsTestOptionsWithoutTestOnly(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + + response := workspace.PipelineStartUpdate(pipelineID, []byte(`{"test_options": {"pytest_args": ["-k", "x"]}}`)) + assert.Equal(t, 400, response.StatusCode) +} + +func TestPipelineStartUpdateRejectsInvalidTestOptions(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + + // The messages are the service's, from PipelineTestSelection. + for options, message := range map[string]string{ + `{"pytest_args": ["-korders"]}`: "pytest_args accepts only -k and -m, each followed by its value. Got '-korders'.", + `{"pytest_args": ["-x"]}`: "pytest_args accepts only -k and -m, each followed by its value. Got '-x'.", + `{"pytest_args": ["-m"]}`: "The pytest_args option '-m' must be followed by its value.", + `{"pytest_args": ["-k", " "]}`: "The value of the pytest_args option '-k' must not be empty.", + `{"pytest_args": ["-k", "-m"]}`: "The value of the pytest_args option '-k' must not start with '-'. Got '-m'.", + `{"test_selection": ["tests/test_a.py::t"]}`: "The test selection 'tests/test_a.py::t' must be an absolute workspace path, such as /Workspace/Users/someone@example.com/orders/tests/test_orders.py.", + } { + response := workspace.PipelineStartUpdate(pipelineID, []byte(`{"test_only": true, "test_options": `+options+`}`)) + assert.Equal(t, 400, response.StatusCode, options) + assert.Equal(t, map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, response.Body, options) + } + + startTestUpdate(t, workspace, pipelineID, `{"test_only": true, "test_options": {"pytest_args": ["-k", "orders", "-m", "smoke"], "test_selection": ["`+testRootPath+`/tests"]}}`) +} + +func TestPipelineFiniteUpdateIsActiveUntilItEnds(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + selection := `{"test_only": true, "test_options": {"test_selection": ["` + testRootPath + `/tests/test_orders.py::test_ok"]}}` + updateID := startTestUpdate(t, workspace, pipelineID, selection) + + // Like the service, a second update is rejected until the first one ends, even if it will end on its own. + assert.Equal(t, 409, workspace.PipelineStartUpdate(pipelineID, []byte(selection)).StatusCode) + assert.Equal(t, pipelines.UpdateInfoStateWaitingForResources, getUpdateState(workspace, pipelineID, updateID)) + assert.Equal(t, pipelines.UpdateInfoStateRunning, getUpdateState(workspace, pipelineID, updateID)) + require.Equal(t, pipelines.UpdateInfoStateCompleted, getUpdateState(workspace, pipelineID, updateID)) + startTestUpdate(t, workspace, pipelineID, selection) +} + +func TestPipelineUpdateThatIsNeverPolledEndsWithItsFirstState(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + + // A plain update completes at once, so a second one can start without polling the first. + updateID := startTestUpdate(t, workspace, pipelineID, `{}`) + startTestUpdate(t, workspace, pipelineID, `{}`) + + // Stop leaves an update that already completed as it is. + require.Equal(t, 0, workspace.PipelineStop(pipelineID).StatusCode) + assert.Equal(t, pipelines.UpdateInfoStateCompleted, getUpdateState(workspace, pipelineID, updateID)) +} + +func TestPipelineStopHidesEventsOfStatesNotReached(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + updateID := startTestUpdate(t, workspace, pipelineID, `{"test_only": true, "test_options": {"test_selection": ["`+testRootPath+`/tests/test_orders.py"]}}`) + query := url.Values{"filter": {"update_id='" + updateID + "'"}} + + assert.Equal(t, pipelines.UpdateInfoStateWaitingForResources, getUpdateState(workspace, pipelineID, updateID)) + require.Equal(t, 0, workspace.PipelineStop(pipelineID).StatusCode) + assert.Equal(t, pipelines.UpdateInfoStateCanceled, getUpdateState(workspace, pipelineID, updateID)) + + // The update was stopped before RUNNING, so its case results and summary never show. + events, _ := listEvents(t, workspace, pipelineID, query) + assert.Empty(t, events) +} + +func TestPipelineStopCancelsAFiniteUpdate(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + selection := `{"test_only": true, "test_options": {"test_selection": ["` + testRootPath + `/tests/test_orders.py::test_ok"]}}` + updateID := startTestUpdate(t, workspace, pipelineID, selection) + + require.Equal(t, 0, workspace.PipelineStop(pipelineID).StatusCode) + // The stopped update ends at once, so a new one can start before anyone reads the old one. + startTestUpdate(t, workspace, pipelineID, selection) + assert.Equal(t, pipelines.UpdateInfoStateCanceled, getUpdateState(workspace, pipelineID, updateID)) + assert.Equal(t, pipelines.UpdateInfoStateCanceled, getUpdateState(workspace, pipelineID, updateID)) +} diff --git a/python/databricks/bundles/pipelines/__init__.py b/python/databricks/bundles/pipelines/__init__.py index 412cbfa59b3..d9bf369bdb3 100644 --- a/python/databricks/bundles/pipelines/__init__.py +++ b/python/databricks/bundles/pipelines/__init__.py @@ -207,6 +207,15 @@ "PipelinePermissionLevel", "PipelinePermissionLevelParam", "PipelinePermissionParam", + "PipelineTestEnvironment", + "PipelineTestEnvironmentDict", + "PipelineTestEnvironmentParam", + "PipelineTestSource", + "PipelineTestSourceDict", + "PipelineTestSourceParam", + "PipelineTesting", + "PipelineTestingDict", + "PipelineTestingParam", "PipelinesEnvironment", "PipelinesEnvironmentDict", "PipelinesEnvironmentParam", @@ -645,6 +654,21 @@ PipelinePermissionLevel, PipelinePermissionLevelParam, ) +from databricks.bundles.pipelines._models.pipeline_test_environment import ( + PipelineTestEnvironment, + PipelineTestEnvironmentDict, + PipelineTestEnvironmentParam, +) +from databricks.bundles.pipelines._models.pipeline_test_source import ( + PipelineTestSource, + PipelineTestSourceDict, + PipelineTestSourceParam, +) +from databricks.bundles.pipelines._models.pipeline_testing import ( + PipelineTesting, + PipelineTestingDict, + PipelineTestingParam, +) from databricks.bundles.pipelines._models.pipelines_environment import ( PipelinesEnvironment, PipelinesEnvironmentDict, diff --git a/python/databricks/bundles/pipelines/_models/pipeline.py b/python/databricks/bundles/pipelines/_models/pipeline.py index 070e813ef34..e0010a55e8f 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/python/databricks/bundles/pipelines/_models/pipeline.py @@ -47,6 +47,10 @@ PipelinePermission, PipelinePermissionParam, ) +from databricks.bundles.pipelines._models.pipeline_testing import ( + PipelineTesting, + PipelineTestingParam, +) from databricks.bundles.pipelines._models.pipelines_environment import ( PipelinesEnvironment, PipelinesEnvironmentParam, @@ -260,6 +264,13 @@ class Pipeline(Resource): [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. """ + testing: VariableOrOptional[PipelineTesting] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. + """ + usage_policy_id: VariableOrOptional[str] = None """ :meta private: [EXPERIMENTAL] @@ -473,6 +484,13 @@ class PipelineDict(TypedDict, total=False): [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. """ + testing: VariableOrOptional[PipelineTestingParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. + """ + usage_policy_id: VariableOrOptional[str] """ :meta private: [EXPERIMENTAL] diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py new file mode 100644 index 00000000000..3386c1d69b1 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py @@ -0,0 +1,42 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTestEnvironment: + """ + :meta private: [EXPERIMENTAL] + """ + + dependencies: VariableOrList[str] = field(default_factory=list) + """ + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and an extra, for example `--editable ${workspace.file_path}[test]`. Package names and URLs are rejected. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestEnvironmentDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestEnvironmentDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestEnvironmentDict(TypedDict, total=False): + """""" + + dependencies: VariableOrList[str] + """ + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and an extra, for example `--editable ${workspace.file_path}[test]`. Package names and URLs are rejected. + """ + + +PipelineTestEnvironmentParam = PipelineTestEnvironmentDict | PipelineTestEnvironment diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_source.py b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py new file mode 100644 index 00000000000..b893d161d69 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py @@ -0,0 +1,46 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.pipelines._models.path_pattern import ( + PathPattern, + PathPatternParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTestSource: + """ + :meta private: [EXPERIMENTAL] + """ + + glob: VariableOrOptional[PathPattern] = None + """ + A test file, or a folder followed by `/**`. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestSourceDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestSourceDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestSourceDict(TypedDict, total=False): + """""" + + glob: VariableOrOptional[PathPatternParam] + """ + A test file, or a folder followed by `/**`. + """ + + +PipelineTestSourceParam = PipelineTestSourceDict | PipelineTestSource diff --git a/python/databricks/bundles/pipelines/_models/pipeline_testing.py b/python/databricks/bundles/pipelines/_models/pipeline_testing.py new file mode 100644 index 00000000000..c8f20835c95 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_testing.py @@ -0,0 +1,60 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.pipelines._models.pipeline_test_environment import ( + PipelineTestEnvironment, + PipelineTestEnvironmentParam, +) +from databricks.bundles.pipelines._models.pipeline_test_source import ( + PipelineTestSource, + PipelineTestSourceParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTesting: + """ + :meta private: [EXPERIMENTAL] + """ + + environment: VariableOrOptional[PipelineTestEnvironment] = None + """ + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + """ + + sources: VariableOrList[PipelineTestSource] = field(default_factory=list) + """ + The workspace files and folders that hold the pipeline's tests. They must not overlap the pipeline's `libraries`. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestingDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestingDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestingDict(TypedDict, total=False): + """""" + + environment: VariableOrOptional[PipelineTestEnvironmentParam] + """ + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + """ + + sources: VariableOrList[PipelineTestSourceParam] + """ + The workspace files and folders that hold the pipeline's tests. They must not overlap the pipeline's `libraries`. + """ + + +PipelineTestingParam = PipelineTestingDict | PipelineTesting