From 8d93a193a4ac3e4614f40dc36da980ba1486a32f Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:00:42 +0000 Subject: [PATCH 01/18] Wait for and print the results of databricks pipelines test --- .nextchanges/cli/pipelines-test-results.md | 1 + .../pipelines/test/arguments/output.txt | 21 + acceptance/pipelines/test/arguments/script | 6 + .../pipelines/test/immutable/output.txt | 4 +- acceptance/pipelines/test/immutable/script | 6 +- .../pipelines/test/many-cases/databricks.yml | 11 + .../pipelines/test/many-cases/out.test.toml | 1 + .../pipelines/test/many-cases/output.txt | 13 + .../test/many-cases/pipeline_file.py | 6 + acceptance/pipelines/test/many-cases/script | 9 + .../test/many-cases/tests/test_many.py | 799 ++++++++++++++++++ .../pipeline-id/orders/tests/test_orders.py | 2 + .../pipelines/test/pipeline-id/output.txt | 24 +- acceptance/pipelines/test/pipeline-id/script | 18 +- .../pipelines/test/results/databricks.yml | 11 + .../pipelines/test/results/out.test.toml | 1 + acceptance/pipelines/test/results/output.txt | 146 ++++ .../pipelines/test/results/pipeline_file.py | 6 + acceptance/pipelines/test/results/script | 22 + .../pipelines/test/results/tests/helpers.py | 2 + .../test/results/tests/test_compute.py | 11 + .../test/results/tests/test_delivery.py | 3 + .../test/results/tests/test_late_failure.py | 7 + .../test/results/tests/test_orders.py | 15 + .../pipelines/test/run-control/databricks.yml | 11 + .../pipelines/test/run-control/out.test.toml | 1 + .../pipelines/test/run-control/output.txt | 70 ++ .../test/run-control/pipeline_file.py | 6 + acceptance/pipelines/test/run-control/script | 32 + .../pipelines/test/run-control/test.toml | 1 + .../test/run-control/tests/test_orders.py | 2 + .../test/run-control/tests/test_slow.py | 3 + .../pipelines/test/selection/output.txt | 6 +- acceptance/pipelines/test/selection/script | 11 +- cmd/pipelines/test.go | 208 ++++- cmd/pipelines/test_events.go | 354 ++++++++ cmd/pipelines/test_events_test.go | 218 +++++ cmd/pipelines/test_render.go | 134 +++ cmd/pipelines/test_render_test.go | 129 +++ cmd/pipelines/test_test.go | 124 ++- libs/testserver/fake_workspace.go | 7 +- libs/testserver/handlers.go | 4 +- libs/testserver/pipelines.go | 78 +- libs/testserver/pipelines_test.go | 4 +- libs/testserver/pipelines_testing.go | 407 +++++++++ libs/testserver/pipelines_testing_test.go | 217 +++++ 46 files changed, 3119 insertions(+), 53 deletions(-) create mode 100644 .nextchanges/cli/pipelines-test-results.md create mode 100644 acceptance/pipelines/test/many-cases/databricks.yml create mode 100644 acceptance/pipelines/test/many-cases/out.test.toml create mode 100644 acceptance/pipelines/test/many-cases/output.txt create mode 100644 acceptance/pipelines/test/many-cases/pipeline_file.py create mode 100644 acceptance/pipelines/test/many-cases/script create mode 100644 acceptance/pipelines/test/many-cases/tests/test_many.py create mode 100644 acceptance/pipelines/test/pipeline-id/orders/tests/test_orders.py create mode 100644 acceptance/pipelines/test/results/databricks.yml create mode 100644 acceptance/pipelines/test/results/out.test.toml create mode 100644 acceptance/pipelines/test/results/output.txt create mode 100644 acceptance/pipelines/test/results/pipeline_file.py create mode 100644 acceptance/pipelines/test/results/script create mode 100644 acceptance/pipelines/test/results/tests/helpers.py create mode 100644 acceptance/pipelines/test/results/tests/test_compute.py create mode 100644 acceptance/pipelines/test/results/tests/test_delivery.py create mode 100644 acceptance/pipelines/test/results/tests/test_late_failure.py create mode 100644 acceptance/pipelines/test/results/tests/test_orders.py create mode 100644 acceptance/pipelines/test/run-control/databricks.yml create mode 100644 acceptance/pipelines/test/run-control/out.test.toml create mode 100644 acceptance/pipelines/test/run-control/output.txt create mode 100644 acceptance/pipelines/test/run-control/pipeline_file.py create mode 100644 acceptance/pipelines/test/run-control/script create mode 100644 acceptance/pipelines/test/run-control/test.toml create mode 100644 acceptance/pipelines/test/run-control/tests/test_orders.py create mode 100644 acceptance/pipelines/test/run-control/tests/test_slow.py create mode 100644 cmd/pipelines/test_events.go create mode 100644 cmd/pipelines/test_events_test.go create mode 100644 cmd/pipelines/test_render.go create mode 100644 cmd/pipelines/test_render_test.go create mode 100644 libs/testserver/pipelines_testing.go create mode 100644 libs/testserver/pipelines_testing_test.go diff --git a/.nextchanges/cli/pipelines-test-results.md b/.nextchanges/cli/pipelines-test-results.md new file mode 100644 index 00000000000..72217847c99 --- /dev/null +++ b/.nextchanges/cli/pipelines-test-results.md @@ -0,0 +1 @@ +* `databricks pipelines test` waits for the test results, prints them, and fails when a test fails or the results are incomplete. ([#7272](https://github.com/databricks/cli/pull/7272)) diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index ed845172e52..92c40b25bf7 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -23,12 +23,20 @@ example: databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' +The command waits for the results and succeeds only if at least one test ran +and none failed or errored. --no-wait returns after the update starts. +--timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume +to read the results later. To stop the update, run 'pipelines stop'. + Usage: databricks pipelines test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...] Pipeline Test Flags: + --no-wait Return after the test update starts, without waiting for results. --pipeline-id string ID of the pipeline to test, instead of a PIPELINE_KEY. --pytest-arg stringArray Argument to pass to pytest (repeatable). Only -k and -m are supported. + --resume string Read the results of an existing test update instead of starting one. + --timeout duration Stop waiting for results after this duration, for example 30m. The update keeps running. Flags: -h, --help help for test @@ -55,6 +63,19 @@ Error: the -- separator is not supported: pass pytest arguments with --pytest-ar >>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x Error: unsupported pytest argument "-x": only -k and -m are supported +=== Incompatible run-control flags +>>> musterr [CLI] pipelines test --pipeline-id 1234 tests --resume 5678 +Error: --resume cannot be used with test selections + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678 --pytest-arg=-m --pytest-arg=smoke +Error: --resume cannot be used with --pytest-arg + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678 --no-wait +Error: --resume cannot be used with --no-wait + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --timeout 10m +Error: --timeout cannot be used with --no-wait + === JSON output is not supported >>> musterr [CLI] pipelines test --pipeline-id 1234 -o json Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script index d027c13dcb8..19e85ad4c40 100644 --- a/acceptance/pipelines/test/arguments/script +++ b/acceptance/pipelines/test/arguments/script @@ -10,5 +10,11 @@ trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders title "Invalid --pytest-arg values fail before any request" trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x +title "Incompatible run-control flags" +trace musterr $CLI pipelines test --pipeline-id 1234 tests --resume 5678 +trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --pytest-arg=-m --pytest-arg=smoke +trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --no-wait +trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --timeout 10m + title "JSON output is not supported" trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/acceptance/pipelines/test/immutable/output.txt b/acceptance/pipelines/test/immutable/output.txt index a162096897f..486b91e4a1e 100644 --- a/acceptance/pipelines/test/immutable/output.txt +++ b/acceptance/pipelines/test/immutable/output.txt @@ -15,7 +15,7 @@ If you've made local changes, run 'databricks pipelines deploy' first to ensure Error: cannot resolve test selection "tests/test_orders.py::test_backfill": local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path === Absolute workspace paths and running all tests still work ->>> [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill +>>> [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -23,7 +23,7 @@ If you've made local changes, run 'databricks pipelines deploy' first to ensure Pipeline: my_pipeline ([UUID]) Update: [UUID] ->>> [CLI] pipelines test my_pipeline +>>> [CLI] pipelines test my_pipeline --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. diff --git a/acceptance/pipelines/test/immutable/script b/acceptance/pipelines/test/immutable/script index 972ac6e3e45..a7bca1bfb07 100644 --- a/acceptance/pipelines/test/immutable/script +++ b/acceptance/pipelines/test/immutable/script @@ -5,6 +5,8 @@ title "Local test paths are not supported with workspace.sync_mode immutable" trace musterr $CLI pipelines test my_pipeline tests/test_orders.py::test_backfill title "Absolute workspace paths and running all tests still work" -trace $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill -trace $CLI pipelines test my_pipeline +trace $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait +# Like the service, the test server rejects a new update while one is active. +$CLI pipelines stop my_pipeline &>> LOG.stop +trace $CLI pipelines test my_pipeline --no-wait trace print_requests.py //updates diff --git a/acceptance/pipelines/test/many-cases/databricks.yml b/acceptance/pipelines/test/many-cases/databricks.yml new file mode 100644 index 00000000000..56b2facefa4 --- /dev/null +++ b/acceptance/pipelines/test/many-cases/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-pipeline-test-many-cases + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-many-cases + root_path: . + libraries: + - file: + path: pipeline_file.py diff --git a/acceptance/pipelines/test/many-cases/out.test.toml b/acceptance/pipelines/test/many-cases/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/many-cases/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/many-cases/output.txt b/acceptance/pipelines/test/many-cases/output.txt new file mode 100644 index 00000000000..3171229e84b --- /dev/null +++ b/acceptance/pipelines/test/many-cases/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-many-cases/default/files... +Created pipelines.my_pipeline +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== Every case is counted once across pages, polls, and the final read +>>> tail -n 1 LOG.many +200 tests: 200 passed, 0 failed, 0 skipped, 0 errors +Event pages read: 8 +Pages read with page_token: 4 diff --git a/acceptance/pipelines/test/many-cases/pipeline_file.py b/acceptance/pipelines/test/many-cases/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/many-cases/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/many-cases/script b/acceptance/pipelines/test/many-cases/script new file mode 100644 index 00000000000..ddd3e37328f --- /dev/null +++ b/acceptance/pipelines/test/many-cases/script @@ -0,0 +1,9 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "Every case is counted once across pages, polls, and the final read" +$CLI pipelines test my_pipeline &> LOG.many +trace tail -n 1 LOG.many +print_requests.py --get //events --oneline > LOG.events +echo "Event pages read: $(wc -l < LOG.events | tr -d ' ')" +echo "Pages read with page_token: $(grep -c page_token LOG.events)" diff --git a/acceptance/pipelines/test/many-cases/tests/test_many.py b/acceptance/pipelines/test/many-cases/tests/test_many.py new file mode 100644 index 00000000000..c4a7d594fc8 --- /dev/null +++ b/acceptance/pipelines/test/many-cases/tests/test_many.py @@ -0,0 +1,799 @@ +# 200 cases emit 601 events, more than two pages of List Pipeline Events. +def test_case_000(): + assert True + + +def test_case_001(): + assert True + + +def test_case_002(): + assert True + + +def test_case_003(): + assert True + + +def test_case_004(): + assert True + + +def test_case_005(): + assert True + + +def test_case_006(): + assert True + + +def test_case_007(): + assert True + + +def test_case_008(): + assert True + + +def test_case_009(): + assert True + + +def test_case_010(): + assert True + + +def test_case_011(): + assert True + + +def test_case_012(): + assert True + + +def test_case_013(): + assert True + + +def test_case_014(): + assert True + + +def test_case_015(): + assert True + + +def test_case_016(): + assert True + + +def test_case_017(): + assert True + + +def test_case_018(): + assert True + + +def test_case_019(): + assert True + + +def test_case_020(): + assert True + + +def test_case_021(): + assert True + + +def test_case_022(): + assert True + + +def test_case_023(): + assert True + + +def test_case_024(): + assert True + + +def test_case_025(): + assert True + + +def test_case_026(): + assert True + + +def test_case_027(): + assert True + + +def test_case_028(): + assert True + + +def test_case_029(): + assert True + + +def test_case_030(): + assert True + + +def test_case_031(): + assert True + + +def test_case_032(): + assert True + + +def test_case_033(): + assert True + + +def test_case_034(): + assert True + + +def test_case_035(): + assert True + + +def test_case_036(): + assert True + + +def test_case_037(): + assert True + + +def test_case_038(): + assert True + + +def test_case_039(): + assert True + + +def test_case_040(): + assert True + + +def test_case_041(): + assert True + + +def test_case_042(): + assert True + + +def test_case_043(): + assert True + + +def test_case_044(): + assert True + + +def test_case_045(): + assert True + + +def test_case_046(): + assert True + + +def test_case_047(): + assert True + + +def test_case_048(): + assert True + + +def test_case_049(): + assert True + + +def test_case_050(): + assert True + + +def test_case_051(): + assert True + + +def test_case_052(): + assert True + + +def test_case_053(): + assert True + + +def test_case_054(): + assert True + + +def test_case_055(): + assert True + + +def test_case_056(): + assert True + + +def test_case_057(): + assert True + + +def test_case_058(): + assert True + + +def test_case_059(): + assert True + + +def test_case_060(): + assert True + + +def test_case_061(): + assert True + + +def test_case_062(): + assert True + + +def test_case_063(): + assert True + + +def test_case_064(): + assert True + + +def test_case_065(): + assert True + + +def test_case_066(): + assert True + + +def test_case_067(): + assert True + + +def test_case_068(): + assert True + + +def test_case_069(): + assert True + + +def test_case_070(): + assert True + + +def test_case_071(): + assert True + + +def test_case_072(): + assert True + + +def test_case_073(): + assert True + + +def test_case_074(): + assert True + + +def test_case_075(): + assert True + + +def test_case_076(): + assert True + + +def test_case_077(): + assert True + + +def test_case_078(): + assert True + + +def test_case_079(): + assert True + + +def test_case_080(): + assert True + + +def test_case_081(): + assert True + + +def test_case_082(): + assert True + + +def test_case_083(): + assert True + + +def test_case_084(): + assert True + + +def test_case_085(): + assert True + + +def test_case_086(): + assert True + + +def test_case_087(): + assert True + + +def test_case_088(): + assert True + + +def test_case_089(): + assert True + + +def test_case_090(): + assert True + + +def test_case_091(): + assert True + + +def test_case_092(): + assert True + + +def test_case_093(): + assert True + + +def test_case_094(): + assert True + + +def test_case_095(): + assert True + + +def test_case_096(): + assert True + + +def test_case_097(): + assert True + + +def test_case_098(): + assert True + + +def test_case_099(): + assert True + + +def test_case_100(): + assert True + + +def test_case_101(): + assert True + + +def test_case_102(): + assert True + + +def test_case_103(): + assert True + + +def test_case_104(): + assert True + + +def test_case_105(): + assert True + + +def test_case_106(): + assert True + + +def test_case_107(): + assert True + + +def test_case_108(): + assert True + + +def test_case_109(): + assert True + + +def test_case_110(): + assert True + + +def test_case_111(): + assert True + + +def test_case_112(): + assert True + + +def test_case_113(): + assert True + + +def test_case_114(): + assert True + + +def test_case_115(): + assert True + + +def test_case_116(): + assert True + + +def test_case_117(): + assert True + + +def test_case_118(): + assert True + + +def test_case_119(): + assert True + + +def test_case_120(): + assert True + + +def test_case_121(): + assert True + + +def test_case_122(): + assert True + + +def test_case_123(): + assert True + + +def test_case_124(): + assert True + + +def test_case_125(): + assert True + + +def test_case_126(): + assert True + + +def test_case_127(): + assert True + + +def test_case_128(): + assert True + + +def test_case_129(): + assert True + + +def test_case_130(): + assert True + + +def test_case_131(): + assert True + + +def test_case_132(): + assert True + + +def test_case_133(): + assert True + + +def test_case_134(): + assert True + + +def test_case_135(): + assert True + + +def test_case_136(): + assert True + + +def test_case_137(): + assert True + + +def test_case_138(): + assert True + + +def test_case_139(): + assert True + + +def test_case_140(): + assert True + + +def test_case_141(): + assert True + + +def test_case_142(): + assert True + + +def test_case_143(): + assert True + + +def test_case_144(): + assert True + + +def test_case_145(): + assert True + + +def test_case_146(): + assert True + + +def test_case_147(): + assert True + + +def test_case_148(): + assert True + + +def test_case_149(): + assert True + + +def test_case_150(): + assert True + + +def test_case_151(): + assert True + + +def test_case_152(): + assert True + + +def test_case_153(): + assert True + + +def test_case_154(): + assert True + + +def test_case_155(): + assert True + + +def test_case_156(): + assert True + + +def test_case_157(): + assert True + + +def test_case_158(): + assert True + + +def test_case_159(): + assert True + + +def test_case_160(): + assert True + + +def test_case_161(): + assert True + + +def test_case_162(): + assert True + + +def test_case_163(): + assert True + + +def test_case_164(): + assert True + + +def test_case_165(): + assert True + + +def test_case_166(): + assert True + + +def test_case_167(): + assert True + + +def test_case_168(): + assert True + + +def test_case_169(): + assert True + + +def test_case_170(): + assert True + + +def test_case_171(): + assert True + + +def test_case_172(): + assert True + + +def test_case_173(): + assert True + + +def test_case_174(): + assert True + + +def test_case_175(): + assert True + + +def test_case_176(): + assert True + + +def test_case_177(): + assert True + + +def test_case_178(): + assert True + + +def test_case_179(): + assert True + + +def test_case_180(): + assert True + + +def test_case_181(): + assert True + + +def test_case_182(): + assert True + + +def test_case_183(): + assert True + + +def test_case_184(): + assert True + + +def test_case_185(): + assert True + + +def test_case_186(): + assert True + + +def test_case_187(): + assert True + + +def test_case_188(): + assert True + + +def test_case_189(): + assert True + + +def test_case_190(): + assert True + + +def test_case_191(): + assert True + + +def test_case_192(): + assert True + + +def test_case_193(): + assert True + + +def test_case_194(): + assert True + + +def test_case_195(): + assert True + + +def test_case_196(): + assert True + + +def test_case_197(): + assert True + + +def test_case_198(): + assert True + + +def test_case_199(): + assert True diff --git a/acceptance/pipelines/test/pipeline-id/orders/tests/test_orders.py b/acceptance/pipelines/test/pipeline-id/orders/tests/test_orders.py new file mode 100644 index 00000000000..60ebe0db44c --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/orders/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_valid_order(): + pass diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt index 8c72ffd34f7..6c3611ec2f9 100644 --- a/acceptance/pipelines/test/pipeline-id/output.txt +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -1,6 +1,6 @@ === Relative test selections resolve against the pipeline root_path ->>> [CLI] pipelines test --pipeline-id [UUID] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests +>>> [CLI] pipelines test --pipeline-id [UUID] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests --no-wait Pipeline: [UUID] Update: [UUID] @@ -19,11 +19,21 @@ Update: [UUID] } } +=== Test case IDs are shown relative to the pipeline root_path +>>> [CLI] pipelines test --pipeline-id [UUID] tests +Pipeline: [UUID] +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_orders.py::test_valid_order PASSED (1.00s) + +1 test: 1 passed, 0 failed, 0 skipped, 0 errors + === Without a root_path, test selections must be workspace paths >>> musterr [CLI] pipelines test --pipeline-id [UUID] tests Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests ->>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Users/someone@example.com/tests/test_other.py::test_x +>>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Users/someone@example.com/tests/test_other.py::test_x --no-wait Pipeline: [UUID] Update: [UUID] @@ -41,6 +51,16 @@ Update: [UUID] } } +=== Without a root_path, test case IDs are absolute workspace paths +>>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Users/someone@example.com/orders/tests/test_orders.py +Pipeline: [UUID] +Update: [UUID] +Waiting for compute... +Running tests... +/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_valid_order PASSED (1.00s) + +1 test: 1 passed, 0 failed, 0 skipped, 0 errors + === Outside a project, --pipeline-id is required >>> musterr [CLI] pipelines test tests Error: outside a project, identify the pipeline with --pipeline-id diff --git a/acceptance/pipelines/test/pipeline-id/script b/acceptance/pipelines/test/pipeline-id/script index 5f07186c723..abb1a0ae3fe 100644 --- a/acceptance/pipelines/test/pipeline-id/script +++ b/acceptance/pipelines/test/pipeline-id/script @@ -1,15 +1,29 @@ pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) no_root_pipeline_id=$($CLI pipelines create --json '{"name": "no-root"}' | jq -r .pipeline_id) +$CLI workspace import-dir orders /Workspace/Users/someone@example.com/orders &> LOG.import rm "$OUT_REQUESTS" title "Relative test selections resolve against the pipeline root_path" -trace $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests +trace $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests --no-wait trace print_requests.py //updates +# Like the service, the test server rejects a new update while one is active. +$CLI api post /api/2.0/pipelines/$pipeline_id/stop &>> LOG.stop +rm "$OUT_REQUESTS" + +title "Test case IDs are shown relative to the pipeline root_path" +trace $CLI pipelines test --pipeline-id $pipeline_id tests +rm "$OUT_REQUESTS" title "Without a root_path, test selections must be workspace paths" trace musterr $CLI pipelines test --pipeline-id $no_root_pipeline_id tests -trace $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/tests/test_other.py::test_x +trace $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/tests/test_other.py::test_x --no-wait trace print_requests.py //updates +$CLI api post /api/2.0/pipelines/$no_root_pipeline_id/stop &>> LOG.stop +rm "$OUT_REQUESTS" + +title "Without a root_path, test case IDs are absolute workspace paths" +trace $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/orders/tests/test_orders.py +rm "$OUT_REQUESTS" title "Outside a project, --pipeline-id is required" trace musterr $CLI pipelines test tests diff --git a/acceptance/pipelines/test/results/databricks.yml b/acceptance/pipelines/test/results/databricks.yml new file mode 100644 index 00000000000..ab5a7d66a1a --- /dev/null +++ b/acceptance/pipelines/test/results/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-pipeline-test-results + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-results + root_path: . + libraries: + - file: + path: pipeline_file.py diff --git a/acceptance/pipelines/test/results/out.test.toml b/acceptance/pipelines/test/results/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/results/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/results/output.txt b/acceptance/pipelines/test/results/output.txt new file mode 100644 index 00000000000..6f3ccfb2842 --- /dev/null +++ b/acceptance/pipelines/test/results/output.txt @@ -0,0 +1,146 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-results/default/files... +Created pipelines.my_pipeline +Files: 7 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== All selected tests pass +>>> [CLI] pipelines test my_pipeline tests/test_orders.py::test_valid_order +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_orders.py::test_valid_order PASSED (1.00s) + +1 test: 1 passed, 0 failed, 0 skipped, 0 errors + +>>> print_requests.py --get //events +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID]/events", + "q": { + "filter": "update_id='[UUID]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID]/events", + "q": { + "filter": "update_id='[UUID]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID]/events", + "q": { + "filter": "update_id='[UUID]' AND timestamp >= '[TIMESTAMP]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID]/events", + "q": { + "filter": "update_id='[UUID]'", + "max_results": "250", + "order_by": "timestamp asc" + } +} + +=== Failed, errored, and skipped tests +>>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_orders.py::test_valid_order PASSED (1.00s) +tests/test_orders.py::test_backfill_fails FAILED (1.00s) +tests/test_orders.py::test_legacy_format_skipped SKIPPED (1.00s) +tests/test_orders.py::test_fixture_error ERROR (1.00s) + +FAILURES +tests/test_orders.py::test_backfill_fails + AssertionError: assert 100 == 150 +tests/test_orders.py::test_fixture_error + RuntimeError: fixture 'orders' setup failed + +4 tests: 1 passed, 1 failed, 1 skipped, 1 error + +=== No tests collected +>>> musterr [CLI] pipelines test my_pipeline tests/helpers.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... + +No tests were collected. + +=== The update fails after all tests passed +>>> musterr [CLI] pipelines test my_pipeline tests/test_delivery.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_delivery.py::test_breaks_update_after_summary PASSED (1.00s) + +1 test: 1 passed, 0 failed, 0 skipped, 0 errors +Error: update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. + +=== The update fails with an error of its own after a test failed +>>> musterr [CLI] pipelines test my_pipeline tests/test_late_failure.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_late_failure.py::test_backfill_fails FAILED (1.00s) +tests/test_late_failure.py::test_breaks_update_after_summary PASSED (1.00s) + +FAILURES +tests/test_late_failure.py::test_backfill_fails + AssertionError: assert 100 == 150 + +2 tests: 1 passed, 1 failed, 0 skipped, 0 errors +Error: update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. + +=== Compute terminates before all results are received +>>> musterr [CLI] pipelines test my_pipeline tests/test_compute.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_compute.py::test_small_order PASSED (1.00s) + +RUN INCOMPLETE: the update failed before all test results were received. +Results received: 1 passed. +Error: update [UUID] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly. diff --git a/acceptance/pipelines/test/results/pipeline_file.py b/acceptance/pipelines/test/results/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/results/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/results/script b/acceptance/pipelines/test/results/script new file mode 100644 index 00000000000..e19daac850c --- /dev/null +++ b/acceptance/pipelines/test/results/script @@ -0,0 +1,22 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "All selected tests pass" +trace $CLI pipelines test my_pipeline tests/test_orders.py::test_valid_order +trace print_requests.py --get //events + +title "Failed, errored, and skipped tests" +trace musterr $CLI pipelines test my_pipeline tests/test_orders.py + +title "No tests collected" +trace musterr $CLI pipelines test my_pipeline tests/helpers.py + +title "The update fails after all tests passed" +trace musterr $CLI pipelines test my_pipeline tests/test_delivery.py + +title "The update fails with an error of its own after a test failed" +trace musterr $CLI pipelines test my_pipeline tests/test_late_failure.py + +title "Compute terminates before all results are received" +trace musterr $CLI pipelines test my_pipeline tests/test_compute.py +rm "$OUT_REQUESTS" diff --git a/acceptance/pipelines/test/results/tests/helpers.py b/acceptance/pipelines/test/results/tests/helpers.py new file mode 100644 index 00000000000..5205aa61063 --- /dev/null +++ b/acceptance/pipelines/test/results/tests/helpers.py @@ -0,0 +1,2 @@ +def make_order(): + return {"id": 1} diff --git a/acceptance/pipelines/test/results/tests/test_compute.py b/acceptance/pipelines/test/results/tests/test_compute.py new file mode 100644 index 00000000000..32954a03b8e --- /dev/null +++ b/acceptance/pipelines/test/results/tests/test_compute.py @@ -0,0 +1,11 @@ +# The test server terminates compute when a test name contains "crash". +def test_small_order(): + assert True + + +def test_crash_on_large_order(): + pass + + +def test_last_order(): + assert True diff --git a/acceptance/pipelines/test/results/tests/test_delivery.py b/acceptance/pipelines/test/results/tests/test_delivery.py new file mode 100644 index 00000000000..63022e21729 --- /dev/null +++ b/acceptance/pipelines/test/results/tests/test_delivery.py @@ -0,0 +1,3 @@ +# The test server fails the update after the summary when a test name contains "breaks_update". +def test_breaks_update_after_summary(): + assert True diff --git a/acceptance/pipelines/test/results/tests/test_late_failure.py b/acceptance/pipelines/test/results/tests/test_late_failure.py new file mode 100644 index 00000000000..4880a0bca88 --- /dev/null +++ b/acceptance/pipelines/test/results/tests/test_late_failure.py @@ -0,0 +1,7 @@ +# One test fails, and the test server then fails the update with an error of its own. +def test_backfill_fails(): + assert 100 == 150 + + +def test_breaks_update_after_summary(): + assert True diff --git a/acceptance/pipelines/test/results/tests/test_orders.py b/acceptance/pipelines/test/results/tests/test_orders.py new file mode 100644 index 00000000000..96be34e78a0 --- /dev/null +++ b/acceptance/pipelines/test/results/tests/test_orders.py @@ -0,0 +1,15 @@ +# The test server picks each outcome from the test name. +def test_valid_order(): + assert True + + +def test_backfill_fails(): + assert 100 == 150 + + +def test_legacy_format_skipped(): + pass + + +def test_fixture_error(): + pass diff --git a/acceptance/pipelines/test/run-control/databricks.yml b/acceptance/pipelines/test/run-control/databricks.yml new file mode 100644 index 00000000000..3c55712d9e8 --- /dev/null +++ b/acceptance/pipelines/test/run-control/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-pipeline-test-run-control + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-run-control + root_path: . + libraries: + - file: + path: pipeline_file.py diff --git a/acceptance/pipelines/test/run-control/out.test.toml b/acceptance/pipelines/test/run-control/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/run-control/output.txt b/acceptance/pipelines/test/run-control/output.txt new file mode 100644 index 00000000000..ac6eb9b2bbc --- /dev/null +++ b/acceptance/pipelines/test/run-control/output.txt @@ -0,0 +1,70 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-run-control/default/files... +Created pipelines.my_pipeline +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== --no-wait returns after the update starts +>>> [CLI] pipelines test my_pipeline tests/test_orders.py --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +=== --resume reads the results of an existing update +>>> [CLI] pipelines test my_pipeline --resume [UUID] +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... +tests/test_orders.py::test_valid_order PASSED (1.00s) + +1 test: 1 passed, 0 failed, 0 skipped, 0 errors + +=== --timeout stops waiting but not the update +>>> [CLI] pipelines test my_pipeline tests/test_slow.py --no-wait +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Waiting for compute... +Running tests... +Error: stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --resume [UUID] + +=== The command to read the results keeps the profile +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s -p run-control +Pipeline: my_pipeline ([UUID]) +Update: [UUID] +Running tests... +Error: stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --profile run-control --resume [UUID] + +=== Another update cannot start while the test update is active +>>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: An active update '[UUID]' already exists for pipeline '[UUID]'. + +=== Stopping the pipeline cancels the test update +>>> [CLI] pipelines stop my_pipeline +Stopping my_pipeline... +my_pipeline has been stopped. + +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +RUN INCOMPLETE: the update was canceled before all test results were received. +Results received: none. +Error: update [UUID] is CANCELED diff --git a/acceptance/pipelines/test/run-control/pipeline_file.py b/acceptance/pipelines/test/run-control/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/run-control/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/run-control/script b/acceptance/pipelines/test/run-control/script new file mode 100644 index 00000000000..41e047e621a --- /dev/null +++ b/acceptance/pipelines/test/run-control/script @@ -0,0 +1,32 @@ +trace $CLI pipelines deploy + +title "--no-wait returns after the update starts" +trace $CLI pipelines test my_pipeline tests/test_orders.py --no-wait | tee submit.txt +update_id=$(sed -n 's/^Update: //p' submit.txt) + +title "--resume reads the results of an existing update" +trace $CLI pipelines test my_pipeline --resume $update_id + +title "--timeout stops waiting but not the update" +trace $CLI pipelines test my_pipeline tests/test_slow.py --no-wait | tee submit.txt +hang_update_id=$(sed -n 's/^Update: //p' submit.txt) +rm submit.txt +trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id --timeout 3s + +title "The command to read the results keeps the profile" +cat > .databrickscfg <>> [CLI] pipelines test +>>> [CLI] pipelines test --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -25,7 +25,7 @@ Update: [UUID] } === Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms ->>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast +>>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -56,7 +56,7 @@ Update: [UUID] } === Relative paths resolve from the working directory ->>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order +>>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order --no-wait Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script index 1134c783f15..d39180763cf 100644 --- a/acceptance/pipelines/test/selection/script +++ b/acceptance/pipelines/test/selection/script @@ -2,16 +2,21 @@ trace $CLI pipelines deploy rm "$OUT_REQUESTS" title "Run all tests of the only pipeline" -trace $CLI pipelines test +trace $CLI pipelines test --no-wait trace print_requests.py //updates +# Like the service, the test server rejects a new update while one is active. +$CLI pipelines stop my_pipeline &>> LOG.stop +rm "$OUT_REQUESTS" title "Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms" -trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg -m --pytest-arg smoke,fast +trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg -m --pytest-arg smoke,fast --no-wait trace print_requests.py //updates +$CLI pipelines stop my_pipeline &>> LOG.stop +rm "$OUT_REQUESTS" title "Relative paths resolve from the working directory" cd tests -trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order +trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order --no-wait cd .. trace print_requests.py //updates diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 96295a1d1b2..5245ccc52fa 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -8,9 +8,12 @@ import ( "io/fs" "net/http" "os" + "os/signal" "path" "path/filepath" "strings" + "syscall" + "time" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" @@ -268,7 +271,7 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) } // resolveTestTargetInBundle identifies the pipeline by its bundle KEY, which must precede any selections. -func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, error) { +func resolveTestTargetInBundle(cmd *cobra.Command, args []string, resume bool) (testTarget, error) { var key string b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ PostInitFunc: func(ctx context.Context, b *bundle.Bundle) error { @@ -291,7 +294,9 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e if err != nil { return testTarget{}, err } - suggestPipelineDeploy(ctx, cmd) + if !resume { + suggestPipelineDeploy(ctx, cmd) + } cwd, err := os.Getwd() if err != nil { @@ -311,6 +316,117 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx)}, nil } +// resumeCommand returns the command that reads the result of the update later. +func (t testTarget) resumeCommand(cmd *cobra.Command, updateID string) string { + parts := []string{cmd.CommandPath()} + if t.key != "" { + parts = append(parts, shellquote.BashArg(t.key)) + } else { + parts = append(parts, "--pipeline-id", shellquote.BashArg(t.pipelineID)) + } + // Quoted so that the command can be pasted into a shell as is. + if target := cmd.Flag("target").Value.String(); target != "" { + parts = append(parts, "--target", shellquote.BashArg(target)) + } + if profile := cmd.Flag("profile").Value.String(); profile != "" { + parts = append(parts, "--profile", shellquote.BashArg(profile)) + } + return strings.Join(append(parts, "--resume", updateID), " ") +} + +const testPollInterval = time.Second + +// testWatcher follows a test update and prints its cases as they finish. +type testWatcher struct { + w *databricks.WorkspaceClient + pipelineID string + updateID string + events *testEventReader + printer *testProgressPrinter + run *testRun +} + +func (tw *testWatcher) addEvents(events []testPipelineEvent) { + for _, e := range events { + if c := tw.run.add(e); c != nil { + tw.printer.caseFinished(c) + } + } +} + +// watch polls the update until it ends and returns its final state. +func (tw *testWatcher) watch(ctx context.Context) (pipelines.UpdateInfoState, error) { + lastTimestamp := "" + for { + response, err := tw.w.Pipelines.GetUpdate(ctx, pipelines.GetUpdateRequest{ + PipelineId: tw.pipelineID, + UpdateId: tw.updateID, + }) + if err != nil { + return "", err + } + if response.Update == nil { + return "", fmt.Errorf("no update found with id %s for pipeline %s", tw.updateID, tw.pipelineID) + } + state := response.Update.State + tw.printer.updateState(state) + + // Reading from the last timestamp returns its events again; the run ignores duplicates. + events, err := tw.events.read(ctx, lastTimestamp) + if err != nil { + return "", err + } + if len(events) > 0 { + lastTimestamp = events[len(events)-1].Timestamp + } + tw.addEvents(events) + + switch state { + case pipelines.UpdateInfoStateCompleted, pipelines.UpdateInfoStateFailed, pipelines.UpdateInfoStateCanceled: + // Events can become readable after later ones, so read all of them once more. + events, err := tw.events.read(ctx, "") + if err != nil { + return "", err + } + tw.addEvents(events) + return state, nil + default: + } + + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(testPollInterval): + } + } +} + +// testRunError returns the error the command exits with after the update ended, or nil +// if every collected test passed or was skipped. +func testRunError(updateID string, state pipelines.UpdateInfoState, run *testRun, incompleteReason string) error { + if incompleteReason == "" { + summary := run.summaries[0] + testsFailed := summary.Total == 0 || summary.Failed > 0 || summary.Errors > 0 + if !testsFailed && state == pipelines.UpdateInfoStateCompleted { + return nil + } + // A test update fails when its tests fail; the printed result explains it, unless the + // update ended in another way or reports an error of its own. The service's own verdict, + // such as "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed", is not. + if testsFailed && state == pipelines.UpdateInfoStateFailed && run.fatalError == "" { + return root.ErrAlreadyPrinted + } + } else if state == pipelines.UpdateInfoStateCompleted { + return root.ErrAlreadyPrinted + } + if updateError := run.updateError(); updateError != "" { + return fmt.Errorf("update %s is %s: %s", updateID, state, updateError) + } + return fmt.Errorf("update %s is %s", updateID, state) +} + +var errTestWaitTimeout = errors.New("test wait timed out") + func testCommand() *cobra.Command { cmd := &cobra.Command{ Use: "test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...]", @@ -336,15 +452,26 @@ Each --pytest-arg passes one argument to pytest, in the order given. Only -k and -m are supported, each followed by its value in the next --pytest-arg, for example: - databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow'`, + databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' + +The command waits for the results and succeeds only if at least one test ran +and none failed or errored. --no-wait returns after the update starts. +--timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume +to read the results later. To stop the update, run 'pipelines stop'.`, } var pipelineID string var pytestArgs []string + var noWait bool + var timeout time.Duration + var resumeUpdateID string testGroup := cmdgroup.NewFlagGroup("Pipeline Test") testGroup.FlagSet().StringVar(&pipelineID, "pipeline-id", "", "ID of the pipeline to test, instead of a PIPELINE_KEY.") testGroup.FlagSet().StringArrayVar(&pytestArgs, "pytest-arg", nil, "Argument to pass to pytest (repeatable). Only -k and -m are supported.") + testGroup.FlagSet().BoolVar(&noWait, "no-wait", false, "Return after the test update starts, without waiting for results.") + testGroup.FlagSet().DurationVar(&timeout, "timeout", 0, "Stop waiting for results after this duration, for example 30m. The update keeps running.") + testGroup.FlagSet().StringVar(&resumeUpdateID, "resume", "", "Read the results of an existing test update instead of starting one.") wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) wrappedCmd.AddFlagGroup(testGroup) @@ -364,6 +491,26 @@ example: if err := validatePytestArgs(pytestArgs); err != nil { return err } + if timeout < 0 { + return errors.New("--timeout must not be negative") + } + if noWait && timeout > 0 { + return errors.New("--timeout cannot be used with --no-wait") + } + if resumeUpdateID != "" { + selectionCount := len(args) + if pipelineID == "" { + selectionCount = max(len(args)-1, 0) + } + switch { + case noWait: + return errors.New("--resume cannot be used with --no-wait") + case len(pytestArgs) > 0: + return errors.New("--resume cannot be used with --pytest-arg") + case selectionCount > 0: + return errors.New("--resume cannot be used with test selections") + } + } var target testTarget var err error @@ -375,20 +522,63 @@ example: if !bundle.HasRoot(ctx) { return errors.New("outside a project, identify the pipeline with --pipeline-id") } - target, err = resolveTestTargetInBundle(cmd, args) + target, err = resolveTestTargetInBundle(cmd, args, resumeUpdateID != "") } if err != nil { return err } - ctx := cmd.Context() + // The CLI root doesn't cancel the context on signals, so stop waiting here. + ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer stop() + out := cmd.OutOrStdout() + + updateID := resumeUpdateID + if updateID == "" { + req := newStartTestUpdateRequest(target.selections, pytestArgs) + updateID, err = startTestUpdate(ctx, target.w, target.pipelineID, req) + if err != nil { + return err + } + } + target.printHeader(out, updateID) + if noWait { + return nil + } - req := newStartTestUpdateRequest(target.selections, pytestArgs) - updateID, err := startTestUpdate(ctx, target.w, target.pipelineID, req) + events, err := newTestEventReader(target.w, target.pipelineID, updateID) if err != nil { return err } - target.printHeader(cmd.OutOrStdout(), updateID) - return nil + watcher := &testWatcher{ + w: target.w, + pipelineID: target.pipelineID, + updateID: updateID, + events: events, + printer: &testProgressPrinter{out: out}, + run: newTestRun(), + } + + waitCtx := ctx + if timeout > 0 { + var cancel context.CancelFunc + waitCtx, cancel = context.WithTimeoutCause(ctx, timeout, errTestWaitTimeout) + defer cancel() + } + state, err := watcher.watch(waitCtx) + if err != nil { + if waitCtx.Err() == nil { + return err + } + resume := target.resumeCommand(cmd, updateID) + if errors.Is(context.Cause(waitCtx), errTestWaitTimeout) { + return fmt.Errorf("stopped waiting for update %s after %s; the update keeps running, to get its results run: %s", updateID, timeout, resume) + } + return fmt.Errorf("stopped waiting for update %s; the update keeps running, to get its results run: %s", updateID, resume) + } + + incompleteReason := watcher.run.incompleteReason(state) + renderTestResult(out, watcher.run, incompleteReason) + return testRunError(updateID, state, watcher.run, incompleteReason) } return cmd diff --git a/cmd/pipelines/test_events.go b/cmd/pipelines/test_events.go new file mode 100644 index 00000000000..517f1ec59c2 --- /dev/null +++ b/cmd/pipelines/test_events.go @@ -0,0 +1,354 @@ +package pipelines + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/databricks/cli/libs/auth" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/pipelines" +) + +const ( + testCaseProgressEventType = "test_case_progress" + testRunSummaryEventType = "test_run_summary" +) + +type testCaseStatus string + +const ( + testCasePending testCaseStatus = "PENDING" + testCaseRunning testCaseStatus = "RUNNING" + testCasePassed testCaseStatus = "PASSED" + testCaseFailed testCaseStatus = "FAILED" + testCaseSkipped testCaseStatus = "SKIPPED" + testCaseError testCaseStatus = "ERROR" +) + +// parseTestCaseStatus accepts both the short status and the proto enum value name. +func parseTestCaseStatus(s string) testCaseStatus { + return testCaseStatus(strings.TrimPrefix(s, "PIPELINE_TEST_CASE_STATUS_")) +} + +func (s testCaseStatus) isFinal() bool { + switch s { + case testCasePassed, testCaseFailed, testCaseSkipped, testCaseError: + return true + default: + return false + } +} + +type testRunSummary struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Skipped int `json:"skipped"` + Errors int `json:"errors"` +} + +// testPipelineEvent is an entry of List Pipeline Events. The Go SDK PipelineEvent has no +// details or origin.test_case_id, so test events are decoded from the raw response. +// Once the SDK has those fields, remove this type and testEventReader and switch to +// w.Pipelines.ListPipelineEvents. +type testPipelineEvent struct { + Id string `json:"id"` + EventType string `json:"event_type"` + Timestamp string `json:"timestamp"` + Level pipelines.EventLevel `json:"level"` + Message string `json:"message"` + Origin struct { + TestCaseId string `json:"test_case_id"` + } `json:"origin"` + Error *pipelines.ErrorDetail `json:"error"` + Truncation *pipelines.Truncation `json:"truncation"` + Details struct { + TestCaseProgress *struct { + Status string `json:"status"` + } `json:"test_case_progress"` + TestRunSummary *testRunSummary `json:"test_run_summary"` + } `json:"details"` +} + +func (e testPipelineEvent) isTruncated(field string) bool { + if e.Truncation == nil { + return false + } + for _, f := range e.Truncation.TruncatedFields { + if f.FieldName == field { + return true + } + } + return false +} + +// testEventsPageSize is the largest page List Pipeline Events returns. +const testEventsPageSize = 250 + +// testEventReader reads the events of one update. fetchAllPipelineEvents in utils.go is not reused: +// it reads one page into SDK events, which drop the test fields, and a test run needs every page. +type testEventReader struct { + apiClient *client.DatabricksClient + headers map[string]string + pipelineID string + updateID string +} + +func newTestEventReader(w *databricks.WorkspaceClient, pipelineID, updateID string) (*testEventReader, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return nil, fmt.Errorf("failed to create API client: %w", err) + } + return &testEventReader{ + apiClient: apiClient, + headers: auth.WorkspaceIDHeaders(w.Config), + pipelineID: pipelineID, + updateID: updateID, + }, nil +} + +// read returns the update's events in timestamp order, following every page. With +// sinceTimestamp it returns only the events at or after that timestamp, which must be a +// timestamp string exactly as the API returned it. +func (r *testEventReader) read(ctx context.Context, sinceTimestamp string) ([]testPipelineEvent, error) { + path := fmt.Sprintf("/api/2.0/pipelines/%s/events", r.pipelineID) + filter := fmt.Sprintf("update_id='%s'", r.updateID) + if sinceTimestamp != "" { + filter += fmt.Sprintf(" AND timestamp >= '%s'", sinceTimestamp) + } + request := pipelines.ListPipelineEventsRequest{ + Filter: filter, + OrderBy: []string{"timestamp asc"}, + MaxResults: testEventsPageSize, + } + + var events []testPipelineEvent + for { + var response struct { + Events []testPipelineEvent `json:"events"` + NextPageToken string `json:"next_page_token"` + } + err := r.apiClient.Do(ctx, http.MethodGet, path, r.headers, nil, request, &response) + if err != nil { + return nil, fmt.Errorf("failed to read events of update %s: %w", r.updateID, err) + } + events = append(events, response.Events...) + if response.NextPageToken == "" { + return events, nil + } + // A request with page_token must not repeat filter or order_by; the token carries + // them. The SDK iterator repeats them, which is another reason not to use it here. + request = pipelines.ListPipelineEventsRequest{ + PageToken: response.NextPageToken, + MaxResults: testEventsPageSize, + } + } +} + +// testCaseResult is what the events report about one test case. +type testCaseResult struct { + // id is origin.test_case_id: the pytest test ID, relative to pytest's rootdir, such as + // tests/test_orders.py::test_backfill. + id string + status testCaseStatus + // finals counts final outcomes; a complete result has exactly one per case. + finals int + started time.Time + finished time.Time + errorDetail *pipelines.ErrorDetail + detailsTruncated bool +} + +// duration returns the time between the RUNNING and the final event, if both are known. +func (c *testCaseResult) duration() (time.Duration, bool) { + if c.started.IsZero() || c.finished.IsZero() || c.finished.Before(c.started) { + return 0, false + } + return c.finished.Sub(c.started), true +} + +// testRun accumulates the events of a test update, ignoring events it has already seen. +type testRun struct { + seen map[string]bool + cases []*testCaseResult + byID map[string]*testCaseResult + summaries []testRunSummary + // unreadable counts test events whose outcome is missing. + unreadable int + // fatalError is the first fatal error that the update reported, other than the test verdict. + fatalError string + // verdictError is the service's verdict that tests failed, if the update reported it. + verdictError string +} + +// testVerdictErrorPrefix starts the message of the error by which the service reports that tests +// failed, such as "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed (1 passed, 0 skipped).". +const testVerdictErrorPrefix = "[PIPELINE_TEST_FAILED]" + +// isTestVerdict reports whether an update error is the service's verdict that tests failed. +func isTestVerdict(e *pipelines.ErrorDetail) bool { + for _, exception := range e.Exceptions { + if strings.HasPrefix(exception.Message, testVerdictErrorPrefix) { + return true + } + } + return false +} + +func newTestRun() *testRun { + return &testRun{seen: map[string]bool{}, byID: map[string]*testCaseResult{}} +} + +// add records an event and returns the case it gave a final outcome to, if any. +func (r *testRun) add(e testPipelineEvent) *testCaseResult { + if r.seen[e.Id] { + return nil + } + r.seen[e.Id] = true + + if e.Error != nil && e.Error.Fatal { + switch { + case isTestVerdict(e.Error): + r.verdictError = errorDetailText(e.Error) + case r.fatalError == "": + r.fatalError = errorDetailText(e.Error) + } + } + + switch e.EventType { + case testRunSummaryEventType: + if e.Details.TestRunSummary == nil { + r.unreadable++ + return nil + } + r.summaries = append(r.summaries, *e.Details.TestRunSummary) + case testCaseProgressEventType: + progress := e.Details.TestCaseProgress + if progress == nil || e.Origin.TestCaseId == "" { + r.unreadable++ + return nil + } + status := parseTestCaseStatus(progress.Status) + if !status.isFinal() && status != testCasePending && status != testCaseRunning { + r.unreadable++ + return nil + } + + c, ok := r.byID[e.Origin.TestCaseId] + if !ok { + c = &testCaseResult{id: e.Origin.TestCaseId, status: testCasePending} + r.byID[c.id] = c + r.cases = append(r.cases, c) + } + timestamp, _ := time.Parse(time.RFC3339Nano, e.Timestamp) + switch { + case status == testCaseRunning: + c.started = timestamp + if !c.status.isFinal() { + c.status = testCaseRunning + } + case status.isFinal(): + c.finals++ + c.status = status + c.finished = timestamp + c.errorDetail = e.Error + c.detailsTruncated = e.isTruncated("error") + if c.finals == 1 { + return c + } + } + } + return nil +} + +// counts returns the number of cases with each final outcome. +func (r *testRun) counts() testRunSummary { + var s testRunSummary + for _, c := range r.cases { + if !c.status.isFinal() { + continue + } + s.Total++ + switch c.status { + case testCasePassed: + s.Passed++ + case testCaseFailed: + s.Failed++ + case testCaseSkipped: + s.Skipped++ + case testCaseError: + s.Errors++ + default: + } + } + return s +} + +// incompleteReason explains why the events of an ended update are not a complete result, +// or returns "" if they are: one summary and one final outcome per case, matching its counts. +func (r *testRun) incompleteReason(state pipelines.UpdateInfoState) string { + if r.unreadable > 0 { + return "some test results could not be read" + } + switch len(r.summaries) { + case 0: + switch state { + case pipelines.UpdateInfoStateFailed: + return "the update failed before all test results were received" + case pipelines.UpdateInfoStateCanceled: + return "the update was canceled before all test results were received" + default: + return "the update ended without a test summary" + } + case 1: + default: + return fmt.Sprintf("received %d test summaries instead of one", len(r.summaries)) + } + + for _, c := range r.cases { + if c.finals > 1 { + return fmt.Sprintf("test %s has more than one result", c.id) + } + } + summary := r.summaries[0] + received := r.counts() + if received.Total != summary.Total { + return fmt.Sprintf("received %d of %d test results", received.Total, summary.Total) + } + if received != summary { + return "the test results do not match the test summary" + } + for _, c := range r.cases { + if c.finals == 0 { + return fmt.Sprintf("test %s has no result", c.id) + } + } + return "" +} + +// updateError returns the error that the update reported: its own, or else the test verdict. +func (r *testRun) updateError() string { + if r.fatalError != "" { + return r.fatalError + } + return r.verdictError +} + +func errorDetailText(detail *pipelines.ErrorDetail) string { + var parts []string + for _, e := range detail.Exceptions { + switch { + case e.ClassName != "" && e.Message != "": + parts = append(parts, e.ClassName+": "+e.Message) + case e.Message != "": + parts = append(parts, e.Message) + case e.ClassName != "": + parts = append(parts, e.ClassName) + } + } + return strings.Join(parts, "\n") +} diff --git a/cmd/pipelines/test_events_test.go b/cmd/pipelines/test_events_test.go new file mode 100644 index 00000000000..36a2c6a5fbb --- /dev/null +++ b/cmd/pipelines/test_events_test.go @@ -0,0 +1,218 @@ +package pipelines + +import ( + "encoding/json" + "fmt" + "net/url" + "strings" + "testing" + "time" + + "github.com/databricks/cli/libs/testserver" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var testEventSeq int + +func caseEvent(t *testing.T, id, status, timestamp string, extra ...string) testPipelineEvent { + testEventSeq++ + return decodeTestEvent(t, fmt.Sprintf(`{ + "id": "e%d", "event_type": "test_case_progress", "timestamp": %q, + "origin": {"test_case_id": %q}, + "details": {"test_case_progress": {"status": %q}}%s + }`, testEventSeq, timestamp, id, status, strings.Join(append([]string{""}, extra...), ","))) +} + +func summaryEvent(t *testing.T, total, passed, failed, skipped, errors int) testPipelineEvent { + testEventSeq++ + return decodeTestEvent(t, fmt.Sprintf(`{ + "id": "e%d", "event_type": "test_run_summary", + "details": {"test_run_summary": {"total": %d, "passed": %d, "failed": %d, "skipped": %d, "errors": %d}} + }`, testEventSeq, total, passed, failed, skipped, errors)) +} + +func decodeTestEvent(t *testing.T, data string) testPipelineEvent { + var e testPipelineEvent + require.NoError(t, json.Unmarshal([]byte(data), &e)) + return e +} + +func TestTestRunAdd(t *testing.T) { + run := newTestRun() + events := []testPipelineEvent{ + caseEvent(t, "tests/test_a.py::test_ok", "PENDING", "2026-01-01T00:00:00.000Z"), + caseEvent(t, "tests/test_a.py::test_ok", "RUNNING", "2026-01-01T00:00:01.000Z"), + caseEvent(t, "tests/test_a.py::test_ok", "PASSED", "2026-01-01T00:00:02.840Z"), + caseEvent(t, "tests/test_a.py::test_bad", "PIPELINE_TEST_CASE_STATUS_FAILED", "2026-01-01T00:00:03.000Z", + `"error": {"exceptions": [{"class_name": "AssertionError", "message": "assert 1 == 2"}]}`, + `"truncation": {"truncated_fields": [{"field_name": "error"}]}`), + // A late RUNNING event does not undo the final outcome. + caseEvent(t, "tests/test_a.py::test_bad", "RUNNING", "2026-01-01T00:00:02.000Z"), + summaryEvent(t, 2, 1, 1, 0, 0), + } + + var finished []string + for _, e := range events { + if c := run.add(e); c != nil { + finished = append(finished, c.id) + } + } + // Reading every page again adds nothing. + for _, e := range events { + assert.Nil(t, run.add(e)) + } + + assert.Equal(t, []string{"tests/test_a.py::test_ok", "tests/test_a.py::test_bad"}, finished) + require.Len(t, run.cases, 2) + + ok := run.cases[0] + assert.Equal(t, testCasePassed, ok.status) + d, known := ok.duration() + assert.True(t, known) + assert.Equal(t, 1840*time.Millisecond, d) + + bad := run.cases[1] + assert.Equal(t, testCaseFailed, bad.status) + assert.True(t, bad.detailsTruncated) + assert.Equal(t, "AssertionError: assert 1 == 2", errorDetailText(bad.errorDetail)) + d, known = bad.duration() + assert.True(t, known) + assert.Equal(t, time.Second, d) + + assert.Empty(t, run.incompleteReason(pipelines.UpdateInfoStateFailed)) +} + +func TestTestRunIncompleteReason(t *testing.T) { + passed := func() testPipelineEvent { + return caseEvent(t, "t::a", "PASSED", "") + } + tests := []struct { + name string + events []testPipelineEvent + state pipelines.UpdateInfoState + want string + }{ + { + name: "no summary after failure", + events: []testPipelineEvent{passed()}, + state: pipelines.UpdateInfoStateFailed, + want: "the update failed before all test results were received", + }, + { + name: "no summary after cancel", + events: nil, + state: pipelines.UpdateInfoStateCanceled, + want: "the update was canceled before all test results were received", + }, + { + name: "no summary after completion", + events: []testPipelineEvent{passed()}, + state: pipelines.UpdateInfoStateCompleted, + want: "the update ended without a test summary", + }, + { + name: "two summaries", + events: []testPipelineEvent{passed(), summaryEvent(t, 1, 1, 0, 0, 0), summaryEvent(t, 1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "received 2 test summaries instead of one", + }, + { + name: "missing outcome", + events: []testPipelineEvent{passed(), caseEvent(t, "t::b", "RUNNING", ""), summaryEvent(t, 2, 2, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "received 1 of 2 test results", + }, + { + name: "case without outcome that the summary leaves out", + events: []testPipelineEvent{passed(), caseEvent(t, "t::b", "RUNNING", ""), summaryEvent(t, 1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "test t::b has no result", + }, + { + name: "two outcomes for one case", + events: []testPipelineEvent{passed(), caseEvent(t, "t::a", "FAILED", ""), summaryEvent(t, 1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "test t::a has more than one result", + }, + { + name: "outcomes differ from summary", + events: []testPipelineEvent{caseEvent(t, "t::a", "FAILED", ""), summaryEvent(t, 1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "the test results do not match the test summary", + }, + { + name: "outcome without details", + events: []testPipelineEvent{ + passed(), + decodeTestEvent(t, `{"id": "truncated", "event_type": "test_case_progress", "origin": {"test_case_id": "t::b"}}`), + summaryEvent(t, 1, 1, 0, 0, 0), + }, + state: pipelines.UpdateInfoStateCompleted, + want: "some test results could not be read", + }, + { + name: "unknown status", + events: []testPipelineEvent{passed(), caseEvent(t, "t::b", "PIPELINE_TEST_CASE_STATUS_UNSPECIFIED", ""), summaryEvent(t, 1, 1, 0, 0, 0)}, + state: pipelines.UpdateInfoStateCompleted, + want: "some test results could not be read", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + run := newTestRun() + for _, e := range tc.events { + run.add(e) + } + assert.Equal(t, tc.want, run.incompleteReason(tc.state)) + }) + } +} + +func TestTestRunFatalError(t *testing.T) { + run := newTestRun() + run.add(decodeTestEvent(t, `{ + "id": "fatal", "event_type": "update_progress", "level": "ERROR", + "error": {"fatal": true, "exceptions": [{"class_name": "ComputeTerminated", "message": "Compute stopped."}]} + }`)) + assert.Equal(t, "ComputeTerminated: Compute stopped.", run.fatalError) +} + +func TestTestEventReaderFollowsPages(t *testing.T) { + server := testserver.New(t) + var queries []url.Values + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/events", func(req testserver.Request) any { + queries = append(queries, req.URL.Query()) + if req.URL.Query().Get("page_token") == "" { + return map[string]any{"events": []map[string]string{{"id": "e1"}}, "next_page_token": "page2"} + } + return map[string]any{"events": []map[string]string{{"id": "e2"}}} + }) + + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + reader, err := newTestEventReader(w, "p1", "u1") + require.NoError(t, err) + + events, err := reader.read(t.Context(), "2026-01-01T00:00:01.000Z") + require.NoError(t, err) + + var ids []string + for _, e := range events { + ids = append(ids, e.Id) + } + assert.Equal(t, []string{"e1", "e2"}, ids) + assert.Equal(t, []url.Values{ + { + "filter": {"update_id='u1' AND timestamp >= '2026-01-01T00:00:01.000Z'"}, + "order_by": {"timestamp asc"}, + "max_results": {"250"}, + }, + { + "page_token": {"page2"}, + "max_results": {"250"}, + }, + }, queries) +} diff --git a/cmd/pipelines/test_render.go b/cmd/pipelines/test_render.go new file mode 100644 index 00000000000..d92f9cff701 --- /dev/null +++ b/cmd/pipelines/test_render.go @@ -0,0 +1,134 @@ +package pipelines + +import ( + "fmt" + "io" + "strings" + + "github.com/databricks/databricks-sdk-go/service/pipelines" +) + +// testProgressPrinter prints the phase of a test update and each case as it finishes. +type testProgressPrinter struct { + out io.Writer + waiting bool + running bool +} + +func (p *testProgressPrinter) updateState(state pipelines.UpdateInfoState) { + switch state { + case pipelines.UpdateInfoStateQueued, + pipelines.UpdateInfoStateCreated, + pipelines.UpdateInfoStateWaitingForResources, + pipelines.UpdateInfoStateInitializing, + pipelines.UpdateInfoStateResetting, + pipelines.UpdateInfoStateSettingUpTables: + if !p.waiting && !p.running { + fmt.Fprintln(p.out, "Waiting for compute...") + p.waiting = true + } + case pipelines.UpdateInfoStateRunning: + p.startRunning() + default: + } +} + +func (p *testProgressPrinter) startRunning() { + if !p.running { + fmt.Fprintln(p.out, "Running tests...") + p.running = true + } +} + +func (p *testProgressPrinter) caseFinished(c *testCaseResult) { + p.startRunning() + if d, ok := c.duration(); ok { + fmt.Fprintf(p.out, "%s %s (%.2fs)\n", c.id, c.status, d.Seconds()) + } else { + fmt.Fprintf(p.out, "%s %s\n", c.id, c.status) + } +} + +// renderTestFailures prints the details of every failed or errored case. +func renderTestFailures(out io.Writer, run *testRun) { + var failed []*testCaseResult + for _, c := range run.cases { + if c.status == testCaseFailed || c.status == testCaseError { + failed = append(failed, c) + } + } + if len(failed) == 0 { + return + } + + fmt.Fprintln(out) + fmt.Fprintln(out, "FAILURES") + for _, c := range failed { + fmt.Fprintln(out, c.id) + details := "" + if c.errorDetail != nil { + details = errorDetailText(c.errorDetail) + } + if details == "" { + details = "No details were reported." + } + for line := range strings.SplitSeq(details, "\n") { + fmt.Fprintf(out, " %s\n", line) + } + if c.detailsTruncated { + fmt.Fprintln(out, " (details truncated)") + } + } +} + +func errorsLabel(n int) string { + if n == 1 { + return "error" + } + return "errors" +} + +func formatTestCounts(s testRunSummary) string { + return fmt.Sprintf("%d passed, %d failed, %d skipped, %d %s", s.Passed, s.Failed, s.Skipped, s.Errors, errorsLabel(s.Errors)) +} + +// renderTestResult prints the end of a test run: failure details, then the summary or why +// the result is incomplete. +func renderTestResult(out io.Writer, run *testRun, incompleteReason string) { + renderTestFailures(out, run) + fmt.Fprintln(out) + + if incompleteReason != "" { + fmt.Fprintf(out, "RUN INCOMPLETE: %s.\n", incompleteReason) + received := run.counts() + var parts []string + for _, count := range []struct { + n int + label string + }{ + {received.Passed, "passed"}, + {received.Failed, "failed"}, + {received.Skipped, "skipped"}, + {received.Errors, errorsLabel(received.Errors)}, + } { + if count.n > 0 { + parts = append(parts, fmt.Sprintf("%d %s", count.n, count.label)) + } + } + if len(parts) == 0 { + parts = []string{"none"} + } + fmt.Fprintf(out, "Results received: %s.\n", strings.Join(parts, ", ")) + return + } + + summary := run.summaries[0] + switch summary.Total { + case 0: + fmt.Fprintln(out, "No tests were collected.") + case 1: + fmt.Fprintf(out, "1 test: %s\n", formatTestCounts(summary)) + default: + fmt.Fprintf(out, "%d tests: %s\n", summary.Total, formatTestCounts(summary)) + } +} diff --git a/cmd/pipelines/test_render_test.go b/cmd/pipelines/test_render_test.go new file mode 100644 index 00000000000..5129b001cdb --- /dev/null +++ b/cmd/pipelines/test_render_test.go @@ -0,0 +1,129 @@ +package pipelines + +import ( + "bytes" + "testing" + + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" +) + +func TestTestProgressPrinter(t *testing.T) { + var out bytes.Buffer + p := &testProgressPrinter{out: &out} + run := newTestRun() + + p.updateState(pipelines.UpdateInfoStateQueued) + p.updateState(pipelines.UpdateInfoStateWaitingForResources) + run.add(caseEvent(t, "t::timed", "RUNNING", "2026-01-01T00:00:00.000Z")) + p.caseFinished(run.add(caseEvent(t, "t::timed", "PASSED", "2026-01-01T00:01:15.300Z"))) + p.updateState(pipelines.UpdateInfoStateRunning) + p.caseFinished(run.add(caseEvent(t, "t::untimed", "SKIPPED", "2026-01-01T00:01:16.000Z"))) + + assert.Equal(t, `Waiting for compute... +Running tests... +t::timed PASSED (75.30s) +t::untimed SKIPPED +`, out.String()) +} + +func TestRenderTestResult(t *testing.T) { + tests := []struct { + name string + events func() []testPipelineEvent + state pipelines.UpdateInfoState + want string + }{ + { + name: "failures", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + caseEvent(t, "t::ok", "PASSED", ""), + caseEvent(t, "t::bad", "FAILED", "", + `"error": {"exceptions": [{"class_name": "AssertionError", "message": "assert 1 == 2\nwhere 1 = f()"}]}`, + `"truncation": {"truncated_fields": [{"field_name": "error"}]}`), + caseEvent(t, "t::setup", "ERROR", ""), + summaryEvent(t, 3, 1, 1, 0, 1), + } + }, + state: pipelines.UpdateInfoStateFailed, + want: ` +FAILURES +t::bad + AssertionError: assert 1 == 2 + where 1 = f() + (details truncated) +t::setup + No details were reported. + +3 tests: 1 passed, 1 failed, 0 skipped, 1 error +`, + }, + { + name: "several errors", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + caseEvent(t, "t::setup", "ERROR", ""), + caseEvent(t, "t::teardown", "ERROR", ""), + summaryEvent(t, 2, 0, 0, 0, 2), + } + }, + state: pipelines.UpdateInfoStateFailed, + want: ` +FAILURES +t::setup + No details were reported. +t::teardown + No details were reported. + +2 tests: 0 passed, 0 failed, 0 skipped, 2 errors +`, + }, + { + name: "incomplete", + events: func() []testPipelineEvent { + return []testPipelineEvent{caseEvent(t, "t::ok", "PASSED", ""), caseEvent(t, "t::skip", "SKIPPED", "")} + }, + state: pipelines.UpdateInfoStateFailed, + want: ` +RUN INCOMPLETE: the update failed before all test results were received. +Results received: 1 passed, 1 skipped. +`, + }, + { + name: "incomplete with one error", + events: func() []testPipelineEvent { + return []testPipelineEvent{caseEvent(t, "t::setup", "ERROR", "")} + }, + state: pipelines.UpdateInfoStateFailed, + want: ` +FAILURES +t::setup + No details were reported. + +RUN INCOMPLETE: the update failed before all test results were received. +Results received: 1 error. +`, + }, + { + name: "incomplete without results", + events: func() []testPipelineEvent { return nil }, + state: pipelines.UpdateInfoStateCanceled, + want: ` +RUN INCOMPLETE: the update was canceled before all test results were received. +Results received: none. +`, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + run := newTestRun() + for _, e := range tc.events() { + run.add(e) + } + var out bytes.Buffer + renderTestResult(&out, run, run.incompleteReason(tc.state)) + assert.Equal(t, tc.want, out.String()) + }) + } +} diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index dc4de92ed0f..d36f999d491 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -8,7 +8,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - + "github.com/databricks/cli/cmd/root" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -93,3 +95,123 @@ func TestBundleTestSelectionPath(t *testing.T) { _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, filepath.Join("..", "tests")) assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+filepath.ToSlash(syncRoot)) } + +func TestTestTargetResumeCommand(t *testing.T) { + tests := []struct { + name string + target testTarget + flags map[string]string + want string + }{ + { + name: "pipeline key", + target: testTarget{key: "my_pipeline", pipelineID: "p1"}, + want: "databricks pipelines test my_pipeline --resume u1", + }, + { + name: "pipeline key with target and profile", + target: testTarget{key: "my_pipeline", pipelineID: "p1"}, + flags: map[string]string{"target": "dev", "profile": "staging"}, + want: "databricks pipelines test my_pipeline --target dev --profile staging --resume u1", + }, + { + name: "names that need quoting", + target: testTarget{key: "my_pipeline", pipelineID: "p1"}, + flags: map[string]string{"target": "dev west", "profile": "CI West"}, + want: "databricks pipelines test my_pipeline --target 'dev west' --profile 'CI West' --resume u1", + }, + { + name: "pipeline ID with profile", + target: testTarget{pipelineID: "p1"}, + flags: map[string]string{"profile": "staging"}, + want: "databricks pipelines test --pipeline-id p1 --profile staging --resume u1", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + rootCmd := &cobra.Command{Use: "databricks"} + rootCmd.PersistentFlags().StringP("profile", "p", "", "") + rootCmd.PersistentFlags().StringP("target", "t", "", "") + pipelinesCmd := &cobra.Command{Use: "pipelines"} + cmd := &cobra.Command{Use: "test"} + rootCmd.AddCommand(pipelinesCmd) + pipelinesCmd.AddCommand(cmd) + for name, value := range tc.flags { + require.NoError(t, cmd.Flag(name).Value.Set(value)) + } + assert.Equal(t, tc.want, tc.target.resumeCommand(cmd, "u1")) + }) + } +} + +func TestTestRunError(t *testing.T) { + tests := []struct { + name string + events func() []testPipelineEvent + state pipelines.UpdateInfoState + wantErr string + }{ + { + name: "passed", + events: func() []testPipelineEvent { + return []testPipelineEvent{caseEvent(t, "t::a", "PASSED", ""), summaryEvent(t, 1, 1, 0, 0, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + }, + { + name: "incomplete after completion", + events: func() []testPipelineEvent { + return []testPipelineEvent{caseEvent(t, "t::a", "PASSED", "")} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "update failed with an error of its own and then the service's verdict", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + caseEvent(t, "t::a", "FAILED", ""), + summaryEvent(t, 1, 0, 1, 0, 0), + decodeTestEvent(t, `{"id": "own", "error": {"fatal": true, "exceptions": [{"message": "Commit failed."}]}}`), + decodeTestEvent(t, `{"id": "verdict", "error": {"fatal": true, "exceptions": [{"class_name": "TestFailure", "message": "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed."}]}}`), + } + }, + state: pipelines.UpdateInfoStateFailed, + wantErr: "update u1 is FAILED: Commit failed.", + }, + { + name: "update failed with the service's verdict although no test failed", + events: func() []testPipelineEvent { + return []testPipelineEvent{ + caseEvent(t, "t::a", "PASSED", ""), + summaryEvent(t, 1, 1, 0, 0, 0), + decodeTestEvent(t, `{"id": "fatal", "error": {"fatal": true, "exceptions": [{"message": "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed."}]}}`), + } + }, + state: pipelines.UpdateInfoStateFailed, + wantErr: "update u1 is FAILED: [PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed.", + }, + { + name: "canceled after tests failed", + events: func() []testPipelineEvent { + return []testPipelineEvent{caseEvent(t, "t::a", "FAILED", ""), summaryEvent(t, 1, 0, 1, 0, 0)} + }, + state: pipelines.UpdateInfoStateCanceled, + wantErr: "update u1 is CANCELED", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + run := newTestRun() + for _, e := range tc.events() { + run.add(e) + } + err := testRunError("u1", tc.state, run, run.incompleteReason(tc.state)) + if tc.wantErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.wantErr) + } + }) + } +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index dd5050f6f49..1e2fe6dc536 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -216,7 +216,9 @@ type FakeWorkspace struct { JobRunOutputs map[int64]jobs.RunOutput JobRunIdempotency map[string]int64 Pipelines map[string]pipelines.GetPipelineResponse - PipelineUpdates map[string]bool + PipelineUpdates map[string]*fakePipelineUpdate + pipelineEvents []fakePipelineEvent + pipelineEventTime time.Time Monitors map[string]catalog.MonitorInfo Apps map[string]apps.App Schemas map[string]catalog.SchemaInfo @@ -516,7 +518,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { JobRunIdempotency: map[string]int64{}, Grants: map[string][]catalog.PrivilegeAssignment{}, Pipelines: map[string]pipelines.GetPipelineResponse{}, - PipelineUpdates: map[string]bool{}, + PipelineUpdates: map[string]*fakePipelineUpdate{}, + pipelineEventTime: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), Monitors: map[string]catalog.MonitorInfo{}, Apps: map[string]apps.App{}, Catalogs: map[string]catalog.CatalogInfo{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 50ff904a573..f42a74636e6 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -648,11 +648,11 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("POST", "/api/2.0/pipelines/{pipeline_id}/updates", func(req Request) any { - return req.Workspace.PipelineStartUpdate(req.Vars["pipeline_id"]) + return req.Workspace.PipelineStartUpdate(req.Vars["pipeline_id"], req.Body) }) server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/events", func(req Request) any { - return req.Workspace.PipelineEvents(req.Vars["pipeline_id"]) + return req.Workspace.PipelineEvents(req.Vars["pipeline_id"], req.URL.Query()) }) server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}/updates/{update_id}", func(req Request) any { diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 0aedc44c5d3..216ef06fc57 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -143,10 +143,10 @@ func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response return Response{} } -func (s *FakeWorkspace) PipelineStartUpdate(pipelineId string) Response { +func (s *FakeWorkspace) PipelineStartUpdate(pipelineId string, body []byte) Response { defer s.LockUnlock()() - _, exists := s.Pipelines[pipelineId] + pipeline, exists := s.Pipelines[pipelineId] if !exists { return Response{ StatusCode: 404, @@ -154,30 +154,55 @@ func (s *FakeWorkspace) PipelineStartUpdate(pipelineId string) Response { } } - updateId := nextUUID() - s.PipelineUpdates[updateId] = true - - return Response{ - Body: pipelines.StartUpdateResponse{ - UpdateId: updateId, - }, + for id, update := range s.PipelineUpdates { + if update.pipelineID == pipelineId && update.active() { + return Response{ + StatusCode: 409, + Body: map[string]string{ + "error_code": "INVALID_STATE_TRANSITION", + "message": fmt.Sprintf("An active update '%s' already exists for pipeline '%s'.", id, pipelineId), + }, + } + } } -} - -func (s *FakeWorkspace) PipelineEvents(pipelineId string) Response { - defer s.LockUnlock()() - _, exists := s.Pipelines[pipelineId] - if !exists { + var test pipelineTestRequest + if len(body) > 0 { + if err := json.Unmarshal(body, &test); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("cannot unmarshal request body: %s", err), + } + } + } + if test.TestOptions != nil && !test.TestOnly { return Response{ - StatusCode: 404, - Body: map[string]string{"message": fmt.Sprintf("The specified pipeline %s was not found.", pipelineId)}, + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": "test_options requires test_only to be true."}, + } + } + + updateId := nextUUID() + update := &fakePipelineUpdate{ + pipelineID: pipelineId, + states: []pipelines.UpdateInfoState{pipelines.UpdateInfoStateCompleted}, + } + if test.TestOnly { + var selection []string + if test.TestOptions != nil { + selection = test.TestOptions.TestSelection + } + var rootPath string + if pipeline.Spec != nil { + rootPath = pipeline.Spec.RootPath } + update = s.startPipelineTestRun(pipelineId, updateId, s.collectPipelineTests(rootPath, selection)) } + s.PipelineUpdates[updateId] = update return Response{ - Body: map[string]any{ - "events": []pipelines.PipelineEvent{}, + Body: pipelines.StartUpdateResponse{ + UpdateId: updateId, }, } } @@ -193,20 +218,20 @@ func (s *FakeWorkspace) PipelineGetUpdate(pipelineId, updateId string) Response } } - // Check if the update exists - _, updateExists := s.PipelineUpdates[updateId] - if !updateExists { + update, updateExists := s.PipelineUpdates[updateId] + if !updateExists || update.pipelineID != pipelineId { return Response{ StatusCode: 404, Body: map[string]string{"message": fmt.Sprintf("The specified update %s was not found.", updateId)}, } } + update.polls++ return Response{ Body: pipelines.GetUpdateResponse{ Update: &pipelines.UpdateInfo{ UpdateId: updateId, - State: pipelines.UpdateInfoStateCompleted, + State: update.states[update.stateIndex()], }, }, } @@ -223,6 +248,13 @@ func (s *FakeWorkspace) PipelineStop(pipelineId string) Response { } } + // Stop is pipeline-wide: it cancels whichever update is active. + for _, update := range s.PipelineUpdates { + if update.pipelineID == pipelineId && update.active() { + update.cancel() + } + } + return Response{ Body: pipelines.GetPipelineResponse{ PipelineId: pipelineId, diff --git a/libs/testserver/pipelines_test.go b/libs/testserver/pipelines_test.go index 39114714082..d8a9391c6d5 100644 --- a/libs/testserver/pipelines_test.go +++ b/libs/testserver/pipelines_test.go @@ -51,7 +51,7 @@ func TestPipelineCreate_AllowsSingleSegmentTargetSchemaName(t *testing.T) { func TestPipelineStartUpdate_HandlesNonExistentPipeline(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") - response := workspace.PipelineStartUpdate("non-existent-pipeline") + response := workspace.PipelineStartUpdate("non-existent-pipeline", nil) assert.Equal(t, 404, response.StatusCode) assert.Contains(t, response.Body.(map[string]string)["message"], "The specified pipeline non-existent-pipeline was not found") } @@ -74,7 +74,7 @@ func TestPipelineStop_AfterUpdate(t *testing.T) { pipelineId := createTestPipeline(t, workspace) - startResponse := workspace.PipelineStartUpdate(pipelineId) + startResponse := workspace.PipelineStartUpdate(pipelineId, nil) assert.Equal(t, 0, startResponse.StatusCode) stopResponse := workspace.PipelineStop(pipelineId) diff --git a/libs/testserver/pipelines_testing.go b/libs/testserver/pipelines_testing.go new file mode 100644 index 00000000000..d1e6ee20ab3 --- /dev/null +++ b/libs/testserver/pipelines_testing.go @@ -0,0 +1,407 @@ +package testserver + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/url" + "path" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "github.com/databricks/databricks-sdk-go/service/pipelines" +) + +// Pipeline event types, which also name the event's details field. +const ( + pipelineEventUpdateProgress = "update_progress" + pipelineEventTestCaseProgress = "test_case_progress" + pipelineEventTestRunSummary = "test_run_summary" +) + +// fakePipelineUpdate is a pipeline update whose state advances by one step on each Get Update call. +type fakePipelineUpdate struct { + pipelineID string + states []pipelines.UpdateInfoState + polls int + + // canceled is set by cancel. CANCELED then holds the index of a state the update never reached. + canceled bool +} + +// stateIndex returns the index of the state returned by the last Get Update call, or -1 before the +// first call. +func (u *fakePipelineUpdate) stateIndex() int { + return min(u.polls, len(u.states)) - 1 +} + +// eventIndex returns the index of the last state whose events are visible. Events emitted for a +// later state are not visible yet, and never are for a state that a canceled update did not reach. +func (u *fakePipelineUpdate) eventIndex() int { + if u.canceled { + return len(u.states) - 2 + } + return u.stateIndex() +} + +// active reports whether the update has not ended yet: the state of the last Get Update call, or +// the first state before that call, is not terminal. Like the service, an update that will end on +// its own is active until it does. +func (u *fakePipelineUpdate) active() bool { + return !isTerminalUpdateState(u.states[max(u.stateIndex(), 0)]) +} + +// cancel ends the update at its current state, as Stop does. CANCELED is the update's state from +// now on, even before the next Get Update call. +func (u *fakePipelineUpdate) cancel() { + u.states = append(u.states[:u.stateIndex()+1], pipelines.UpdateInfoStateCanceled) + u.polls = len(u.states) + u.canceled = true +} + +func isTerminalUpdateState(state pipelines.UpdateInfoState) bool { + switch state { + case pipelines.UpdateInfoStateCompleted, pipelines.UpdateInfoStateFailed, pipelines.UpdateInfoStateCanceled: + return true + default: + return false + } +} + +type fakePipelineEventOrigin struct { + PipelineId string `json:"pipeline_id"` + UpdateId string `json:"update_id"` + TestCaseId string `json:"test_case_id,omitempty"` +} + +// fakePipelineEvent is a pipeline event as returned by List Pipeline Events. The Go SDK +// PipelineEvent has no details or origin.test_case_id, so the fake defines its own shape. +type fakePipelineEvent struct { + Id string `json:"id"` + EventType string `json:"event_type"` + Timestamp string `json:"timestamp"` + Level pipelines.EventLevel `json:"level"` + Message string `json:"message"` + Origin fakePipelineEventOrigin `json:"origin"` + Error *pipelines.ErrorDetail `json:"error,omitempty"` + Details map[string]any `json:"details"` + + // visibleAt is the index of the update state from which the event can be read. + visibleAt int +} + +// pipelineTestRequest holds the Start Update fields of a test run, which are not in the Go SDK yet. +type pipelineTestRequest struct { + TestOnly bool `json:"test_only"` + TestOptions *struct { + TestSelection []string `json:"test_selection"` + PytestArgs []string `json:"pytest_args"` + } `json:"test_options"` +} + +var pipelineTestFunctionRegexp = regexp.MustCompile(`(?m)^def (test_\w+)\(`) + +// pipelineTestRootDir returns pytest's rootdir for a test run, as the service sets it: the deepest +// folder that holds rootPath and every test source or, without rootPath, the folder of every test +// source. It is "" when there is neither. The CLI applies the same rule (testIDBase in +// cmd/pipelines), so acceptance tests cannot catch a wrong rule; only a run on a real workspace checks it. +func pipelineTestRootDir(rootPath string, sources []string) string { + var folders [][]string + if rootPath != "" { + folders = append(folders, strings.Split(strings.Trim(rootPath, "/"), "/")) + } + for _, source := range sources { + segments := strings.Split(strings.Trim(strings.TrimSuffix(source, "/**"), "/"), "/") + if rootPath == "" { + segments = segments[:len(segments)-1] + } + folders = append(folders, segments) + } + if len(folders) == 0 { + return "" + } + common := folders[0] + for _, folder := range folders[1:] { + n := 0 + for n < len(common) && n < len(folder) && common[n] == folder[n] { + n++ + } + common = common[:n] + } + return "/" + strings.Join(common, "/") +} + +// collectPipelineTests stands in for pytest collection: it finds the top-level "def test_*" +// functions in the workspace test_*.py files under each selection, or under rootPath without one. +// Like origin.test_case_id, test IDs are relative to pytest's rootdir, such as +// tests/test_orders.py::test_backfill. +func (s *FakeWorkspace) collectPipelineTests(rootPath string, selection []string) []string { + rootDir := pipelineTestRootDir(rootPath, nil) + if len(selection) == 0 && rootPath != "" { + selection = []string{rootPath} + } + + var ids []string + for _, target := range selection { + targetPath, testName, _ := strings.Cut(target, "::") + var files []string + if _, ok := s.files[targetPath]; ok { + files = []string{targetPath} + } else { + for filePath := range s.files { + base := path.Base(filePath) + if strings.HasPrefix(filePath, targetPath+"/") && strings.HasPrefix(base, "test_") && strings.HasSuffix(base, ".py") { + files = append(files, filePath) + } + } + slices.Sort(files) + } + + for _, filePath := range files { + for _, match := range pipelineTestFunctionRegexp.FindAllSubmatch(s.files[filePath].Data, -1) { + name := string(match[1]) + id := filePath + "::" + name + if rootDir != "" { + id = strings.TrimPrefix(id, rootDir+"/") + } + if (testName == "" || testName == name) && !slices.Contains(ids, id) { + ids = append(ids, id) + } + } + } + } + return ids +} + +// startPipelineTestRun records the events of a test-only update. Tests do not run here; each +// test name selects its outcome: +// - "fail": FAILED, "error": ERROR (a fixture failed), "skip": SKIPPED, otherwise PASSED. +// - "crash": compute terminates when the case starts and the update fails without a summary. +// - "hang": the case never finishes and the update stays RUNNING until the pipeline is stopped. +// - "breaks_update": the case passes, but the update fails after the summary. +func (s *FakeWorkspace) startPipelineTestRun(pipelineID, updateID string, caseIDs []string) *fakePipelineUpdate { + const ( + running = 1 + ended = 2 + ) + update := &fakePipelineUpdate{ + pipelineID: pipelineID, + states: []pipelines.UpdateInfoState{pipelines.UpdateInfoStateWaitingForResources, pipelines.UpdateInfoStateRunning}, + } + + emit := func(visibleAt int, event fakePipelineEvent) { + s.pipelineEventTime = s.pipelineEventTime.Add(time.Second) + event.Id = nextUUID() + event.Timestamp = s.pipelineEventTime.UTC().Format("2006-01-02T15:04:05.000Z") + event.Origin.PipelineId = pipelineID + event.Origin.UpdateId = updateID + if event.Level == "" { + event.Level = pipelines.EventLevelInfo + } + event.visibleAt = visibleAt + s.pipelineEvents = append(s.pipelineEvents, event) + } + emitCase := func(id, status string, errorDetail *pipelines.ErrorDetail) { + event := fakePipelineEvent{ + EventType: pipelineEventTestCaseProgress, + Message: fmt.Sprintf("Test case %s is %s.", id, status), + Origin: fakePipelineEventOrigin{TestCaseId: id}, + Error: errorDetail, + Details: map[string]any{pipelineEventTestCaseProgress: map[string]any{"status": status}}, + } + if errorDetail != nil { + event.Level = pipelines.EventLevelError + } + emit(running, event) + } + exception := func(className, message string) *pipelines.ErrorDetail { + return &pipelines.ErrorDetail{Exceptions: []pipelines.SerializedException{{ClassName: className, Message: message}}} + } + + for _, id := range caseIDs { + emitCase(id, "PENDING", nil) + } + + counts := map[string]int{} + breaksUpdate := false + for _, id := range caseIDs { + emitCase(id, "RUNNING", nil) + _, name, _ := strings.Cut(id, "::") + switch { + case strings.Contains(name, "hang"): + return update + case strings.Contains(name, "crash"): + emit(ended, fakePipelineEvent{ + EventType: pipelineEventUpdateProgress, + Level: pipelines.EventLevelError, + Message: fmt.Sprintf("Update %s is FAILED.", updateID[:6]), + Error: &pipelines.ErrorDetail{Fatal: true, Exceptions: []pipelines.SerializedException{{ClassName: "ComputeTerminatedException", Message: "Compute terminated unexpectedly."}}}, + Details: map[string]any{pipelineEventUpdateProgress: map[string]any{"state": "FAILED"}}, + }) + update.states = append(update.states, pipelines.UpdateInfoStateFailed) + return update + case strings.Contains(name, "breaks_update"): + emitCase(id, "PASSED", nil) + counts["passed"]++ + breaksUpdate = true + case strings.Contains(name, "fail"): + emitCase(id, "FAILED", exception("AssertionError", "assert 100 == 150")) + counts["failed"]++ + case strings.Contains(name, "error"): + emitCase(id, "ERROR", exception("RuntimeError", "fixture 'orders' setup failed")) + counts["errors"]++ + case strings.Contains(name, "skip"): + emitCase(id, "SKIPPED", nil) + counts["skipped"]++ + default: + emitCase(id, "PASSED", nil) + counts["passed"]++ + } + } + + emit(ended, fakePipelineEvent{ + EventType: pipelineEventTestRunSummary, + Message: fmt.Sprintf("Collected %d tests.", len(caseIDs)), + Details: map[string]any{pipelineEventTestRunSummary: map[string]any{ + "total": len(caseIDs), + "passed": counts["passed"], + "failed": counts["failed"], + "skipped": counts["skipped"], + "errors": counts["errors"], + }}, + }) + // Like the service, a run whose tests failed or errored ends with the test verdict as the + // update's error. + if !breaksUpdate && (counts["failed"] > 0 || counts["errors"] > 0) { + failed := counts["failed"] + counts["errors"] + emit(ended, fakePipelineEvent{ + EventType: pipelineEventUpdateProgress, + Level: pipelines.EventLevelError, + Message: fmt.Sprintf("%d pipeline tests failed.", failed), + Error: &pipelines.ErrorDetail{Fatal: true, Exceptions: []pipelines.SerializedException{{ + ClassName: "TestFailure", + Message: fmt.Sprintf("[PIPELINE_TEST_FAILED] %d of the pipeline's tests failed (%d passed, %d skipped).", failed, counts["passed"], counts["skipped"]), + }}}, + Details: map[string]any{pipelineEventUpdateProgress: map[string]any{"state": "FAILED"}}, + }) + } + if breaksUpdate { + emit(ended, fakePipelineEvent{ + EventType: pipelineEventUpdateProgress, + Level: pipelines.EventLevelError, + Message: fmt.Sprintf("Update %s is FAILED.", updateID[:6]), + Error: &pipelines.ErrorDetail{Fatal: true, Exceptions: []pipelines.SerializedException{{ClassName: "ResultDeliveryException", Message: "Test results could not be verified."}}}, + Details: map[string]any{pipelineEventUpdateProgress: map[string]any{"state": "FAILED"}}, + }) + } + if len(caseIDs) == 0 || counts["failed"] > 0 || counts["errors"] > 0 || breaksUpdate { + update.states = append(update.states, pipelines.UpdateInfoStateFailed) + } else { + update.states = append(update.states, pipelines.UpdateInfoStateCompleted) + } + return update +} + +var ( + pipelineEventsUpdateIDFilter = regexp.MustCompile(`update_id\s*=\s*'([^']*)'`) + pipelineEventsTimestampFilter = regexp.MustCompile(`timestamp\s*>=\s*'([^']*)'`) +) + +// pipelineEventsPageToken is the query of the first page plus the offset of the next one. +type pipelineEventsPageToken struct { + Query url.Values `json:"query"` + Offset int `json:"offset"` +} + +// PipelineEvents lists the visible events of a pipeline. Only the update_id and timestamp >= +// conditions of the filter are applied. +func (s *FakeWorkspace) PipelineEvents(pipelineId string, query url.Values) Response { + defer s.LockUnlock()() + + _, exists := s.Pipelines[pipelineId] + if !exists { + return Response{ + StatusCode: 404, + Body: map[string]string{"message": fmt.Sprintf("The specified pipeline %s was not found.", pipelineId)}, + } + } + + invalid := func(message string) Response { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, + } + } + + maxResults := 25 + if value := query.Get("max_results"); value != "" { + n, err := strconv.Atoi(value) + if err != nil || n <= 0 { + return invalid("max_results must be a positive integer.") + } + maxResults = min(n, 250) + } + + offset := 0 + if token := query.Get("page_token"); token != "" { + for key := range query { + if key != "page_token" && key != "max_results" { + return invalid(fmt.Sprintf("page_token cannot be used with %s.", key)) + } + } + data, err := base64.URLEncoding.DecodeString(token) + var decoded pipelineEventsPageToken + if err == nil { + err = json.Unmarshal(data, &decoded) + } + if err != nil { + return invalid("Invalid page_token.") + } + query = decoded.Query + offset = decoded.Offset + } + + updateID := "" + if match := pipelineEventsUpdateIDFilter.FindStringSubmatch(query.Get("filter")); match != nil { + updateID = match[1] + } + var since time.Time + if match := pipelineEventsTimestampFilter.FindStringSubmatch(query.Get("filter")); match != nil { + var err error + since, err = time.Parse(time.RFC3339Nano, match[1]) + if err != nil { + return invalid(fmt.Sprintf("Invalid timestamp in filter: %s.", match[1])) + } + } + + var events []fakePipelineEvent + for _, event := range s.pipelineEvents { + update := s.PipelineUpdates[event.Origin.UpdateId] + if update.pipelineID != pipelineId || event.visibleAt > update.eventIndex() { + continue + } + if updateID != "" && event.Origin.UpdateId != updateID { + continue + } + if timestamp, _ := time.Parse(time.RFC3339Nano, event.Timestamp); timestamp.Before(since) { + continue + } + events = append(events, event) + } + + // Events are returned newest first unless ascending order is requested. + if query.Get("order_by") != "timestamp asc" { + slices.Reverse(events) + } + + page := events[min(offset, len(events)):] + body := map[string]any{"events": append([]fakePipelineEvent{}, page[:min(maxResults, len(page))]...)} + if len(page) > maxResults { + data, _ := json.Marshal(pipelineEventsPageToken{Query: query, Offset: offset + maxResults}) + body["next_page_token"] = base64.URLEncoding.EncodeToString(data) + } + return Response{Body: body} +} diff --git a/libs/testserver/pipelines_testing_test.go b/libs/testserver/pipelines_testing_test.go new file mode 100644 index 00000000000..691fe23dbd3 --- /dev/null +++ b/libs/testserver/pipelines_testing_test.go @@ -0,0 +1,217 @@ +package testserver + +import ( + "net/url" + "testing" + + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testRootPath = "/Workspace/Users/someone/orders" + +func createTestPipelineWithTests(t *testing.T, workspace *FakeWorkspace) string { + response := workspace.PipelineCreate(Request{Body: []byte(`{"name": "orders", "root_path": "` + testRootPath + `"}`)}) + require.Equal(t, 0, response.StatusCode) + workspace.files[testRootPath+"/tests/test_orders.py"] = FileEntry{Data: []byte("def test_ok():\n pass\n\ndef test_fails():\n pass\n\n def test_nested():\n pass\n")} + workspace.files[testRootPath+"/tests/test_slow.py"] = FileEntry{Data: []byte("def test_hang():\n pass\n")} + workspace.files[testRootPath+"/tests/helpers.py"] = FileEntry{Data: []byte("def test_helper():\n pass\n")} + return response.Body.(pipelines.CreatePipelineResponse).PipelineId +} + +func startTestUpdate(t *testing.T, workspace *FakeWorkspace, pipelineID, body string) string { + response := workspace.PipelineStartUpdate(pipelineID, []byte(body)) + require.Equal(t, 0, response.StatusCode, response.Body) + return response.Body.(pipelines.StartUpdateResponse).UpdateId +} + +func getUpdateState(workspace *FakeWorkspace, pipelineID, updateID string) pipelines.UpdateInfoState { + return workspace.PipelineGetUpdate(pipelineID, updateID).Body.(pipelines.GetUpdateResponse).Update.State +} + +func listEvents(t *testing.T, workspace *FakeWorkspace, pipelineID string, query url.Values) ([]fakePipelineEvent, string) { + response := workspace.PipelineEvents(pipelineID, query) + require.Equal(t, 0, response.StatusCode, response.Body) + body := response.Body.(map[string]any) + token, _ := body["next_page_token"].(string) + return body["events"].([]fakePipelineEvent), token +} + +func TestCollectPipelineTests(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + createTestPipelineWithTests(t, workspace) + + // Like the service, IDs are relative to pytest's rootdir, which is the root path here. + assert.Equal(t, []string{"tests/test_orders.py::test_ok", "tests/test_orders.py::test_fails", "tests/test_slow.py::test_hang"}, + workspace.collectPipelineTests(testRootPath, nil)) + assert.Equal(t, []string{"tests/test_orders.py::test_fails"}, + workspace.collectPipelineTests(testRootPath, []string{testRootPath + "/tests/test_orders.py::test_fails", testRootPath + "/tests/test_orders.py::test_fails"})) + assert.Equal(t, []string{"tests/helpers.py::test_helper"}, + workspace.collectPipelineTests(testRootPath, []string{testRootPath + "/tests/helpers.py"})) + assert.Equal(t, []string{testRootPath + "/tests/test_slow.py::test_hang"}, + workspace.collectPipelineTests("", []string{testRootPath + "/tests/test_slow.py"})) + assert.Empty(t, workspace.collectPipelineTests("", nil)) +} + +func TestPipelineTestUpdate(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + updateID := startTestUpdate(t, workspace, pipelineID, `{"test_only": true, "test_options": {"test_selection": ["`+testRootPath+`/tests/test_orders.py"]}}`) + query := url.Values{"filter": {"update_id='" + updateID + "'"}, "order_by": {"timestamp asc"}} + + events, _ := listEvents(t, workspace, pipelineID, query) + assert.Empty(t, events) + + assert.Equal(t, pipelines.UpdateInfoStateWaitingForResources, getUpdateState(workspace, pipelineID, updateID)) + assert.Equal(t, pipelines.UpdateInfoStateRunning, getUpdateState(workspace, pipelineID, updateID)) + events, _ = listEvents(t, workspace, pipelineID, query) + var statuses []string + for _, e := range events { + statuses = append(statuses, e.Origin.TestCaseId+" "+e.Details["test_case_progress"].(map[string]any)["status"].(string)) + } + file := "tests/test_orders.py" + assert.Equal(t, []string{ + file + "::test_ok PENDING", + file + "::test_fails PENDING", + file + "::test_ok RUNNING", + file + "::test_ok PASSED", + file + "::test_fails RUNNING", + file + "::test_fails FAILED", + }, statuses) + + assert.Equal(t, pipelines.UpdateInfoStateFailed, getUpdateState(workspace, pipelineID, updateID)) + last := events[len(events)-1] + query.Set("filter", "update_id='"+updateID+"' AND timestamp >= '"+last.Timestamp+"'") + events, _ = listEvents(t, workspace, pipelineID, query) + require.Len(t, events, 3) + assert.Equal(t, last.Id, events[0].Id) + assert.Equal(t, map[string]any{"total": 2, "passed": 1, "failed": 1, "skipped": 0, "errors": 0}, events[1].Details["test_run_summary"]) + // Like the service, the update's error is the test verdict. + require.NotNil(t, events[2].Error) + assert.True(t, events[2].Error.Fatal) + assert.Equal(t, "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed (1 passed, 0 skipped).", events[2].Error.Exceptions[0].Message) + + query.Set("filter", "timestamp >= 'yesterday'") + assert.Equal(t, 400, workspace.PipelineEvents(pipelineID, query).StatusCode) +} + +func TestPipelineEventsPages(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + updateID := startTestUpdate(t, workspace, pipelineID, `{"test_only": true}`) + for range 3 { + getUpdateState(workspace, pipelineID, updateID) + } + + all, token := listEvents(t, workspace, pipelineID, url.Values{"order_by": {"timestamp asc"}, "max_results": {"100"}}) + assert.Empty(t, token) + require.Len(t, all, 8) + + var paged []fakePipelineEvent + query := url.Values{"filter": {"update_id='" + updateID + "'"}, "order_by": {"timestamp asc"}, "max_results": {"3"}} + for { + events, token := listEvents(t, workspace, pipelineID, query) + paged = append(paged, events...) + if token == "" { + break + } + query = url.Values{"page_token": {token}, "max_results": {"3"}} + } + assert.Equal(t, all, paged) + + // Events are newest first by default. + newest, _ := listEvents(t, workspace, pipelineID, url.Values{"max_results": {"1"}}) + assert.Equal(t, all[len(all)-1].Id, newest[0].Id) + + response := workspace.PipelineEvents(pipelineID, url.Values{"page_token": {"x"}, "filter": {"level='INFO'"}}) + assert.Equal(t, 400, response.StatusCode) +} + +func TestPipelineTestUpdateConflictAndStop(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + updateID := startTestUpdate(t, workspace, pipelineID, `{"test_only": true, "test_options": {"test_selection": ["`+testRootPath+`/tests/test_slow.py"]}}`) + + response := workspace.PipelineStartUpdate(pipelineID, nil) + assert.Equal(t, 409, response.StatusCode) + assert.Equal(t, "An active update '"+updateID+"' already exists for pipeline '"+pipelineID+"'.", response.Body.(map[string]string)["message"]) + + workspace.PipelineStop(pipelineID) + for range 2 { + getUpdateState(workspace, pipelineID, updateID) + } + assert.Equal(t, pipelines.UpdateInfoStateCanceled, getUpdateState(workspace, pipelineID, updateID)) + startTestUpdate(t, workspace, pipelineID, `{}`) +} + +func TestPipelineStartUpdateRejectsTestOptionsWithoutTestOnly(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + + response := workspace.PipelineStartUpdate(pipelineID, []byte(`{"test_options": {"pytest_args": ["-k", "x"]}}`)) + assert.Equal(t, 400, response.StatusCode) +} + +func TestPipelineTestRootDir(t *testing.T) { + assert.Equal(t, "/Workspace/Users/u/files", pipelineTestRootDir("/Workspace/Users/u/files/src/orders", []string{"/Workspace/Users/u/files/tests/**"})) + assert.Equal(t, "/Workspace/Users/u/orders", pipelineTestRootDir("/Workspace/Users/u/orders", nil)) + assert.Equal(t, "/Workspace/Users/u", pipelineTestRootDir("", []string{"/Workspace/Users/u/tests/**"})) + assert.Equal(t, "/Workspace/Users/u/tests", pipelineTestRootDir("", []string{"/Workspace/Users/u/tests/test_x.py"})) + assert.Empty(t, pipelineTestRootDir("", nil)) +} + +func TestPipelineFiniteUpdateIsActiveUntilItEnds(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + selection := `{"test_only": true, "test_options": {"test_selection": ["` + testRootPath + `/tests/test_orders.py::test_ok"]}}` + updateID := startTestUpdate(t, workspace, pipelineID, selection) + + // Like the service, a second update is rejected until the first one ends, even if it will end on its own. + assert.Equal(t, 409, workspace.PipelineStartUpdate(pipelineID, []byte(selection)).StatusCode) + assert.Equal(t, pipelines.UpdateInfoStateWaitingForResources, getUpdateState(workspace, pipelineID, updateID)) + assert.Equal(t, pipelines.UpdateInfoStateRunning, getUpdateState(workspace, pipelineID, updateID)) + require.Equal(t, pipelines.UpdateInfoStateCompleted, getUpdateState(workspace, pipelineID, updateID)) + startTestUpdate(t, workspace, pipelineID, selection) +} + +func TestPipelineUpdateThatIsNeverPolledEndsWithItsFirstState(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + + // A plain update completes at once, so a second one can start without polling the first. + updateID := startTestUpdate(t, workspace, pipelineID, `{}`) + startTestUpdate(t, workspace, pipelineID, `{}`) + + // Stop leaves an update that already completed as it is. + require.Equal(t, 0, workspace.PipelineStop(pipelineID).StatusCode) + assert.Equal(t, pipelines.UpdateInfoStateCompleted, getUpdateState(workspace, pipelineID, updateID)) +} + +func TestPipelineStopHidesEventsOfStatesNotReached(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + updateID := startTestUpdate(t, workspace, pipelineID, `{"test_only": true, "test_options": {"test_selection": ["`+testRootPath+`/tests/test_orders.py"]}}`) + query := url.Values{"filter": {"update_id='" + updateID + "'"}} + + assert.Equal(t, pipelines.UpdateInfoStateWaitingForResources, getUpdateState(workspace, pipelineID, updateID)) + require.Equal(t, 0, workspace.PipelineStop(pipelineID).StatusCode) + assert.Equal(t, pipelines.UpdateInfoStateCanceled, getUpdateState(workspace, pipelineID, updateID)) + + // The update was stopped before RUNNING, so its case results and summary never show. + events, _ := listEvents(t, workspace, pipelineID, query) + assert.Empty(t, events) +} + +func TestPipelineStopCancelsAFiniteUpdate(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + selection := `{"test_only": true, "test_options": {"test_selection": ["` + testRootPath + `/tests/test_orders.py::test_ok"]}}` + updateID := startTestUpdate(t, workspace, pipelineID, selection) + + require.Equal(t, 0, workspace.PipelineStop(pipelineID).StatusCode) + // The stopped update ends at once, so a new one can start before anyone reads the old one. + startTestUpdate(t, workspace, pipelineID, selection) + assert.Equal(t, pipelines.UpdateInfoStateCanceled, getUpdateState(workspace, pipelineID, updateID)) + assert.Equal(t, pipelines.UpdateInfoStateCanceled, getUpdateState(workspace, pipelineID, updateID)) +} From 3c83190398be5e5b2d97e46d9142f5f62afc90f2 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:03:44 +0000 Subject: [PATCH 02/18] Reject a pipelines test --resume update that the service reports is not a test update --- .../pipelines/test/run-control/output.txt | 6 +++-- acceptance/pipelines/test/run-control/script | 5 ++++ cmd/pipelines/test.go | 27 +++++++++++++++++++ libs/testserver/pipelines.go | 17 ++++++++++-- libs/testserver/pipelines_testing.go | 2 ++ libs/testserver/pipelines_testing_test.go | 14 +++++++++- 6 files changed, 66 insertions(+), 5 deletions(-) diff --git a/acceptance/pipelines/test/run-control/output.txt b/acceptance/pipelines/test/run-control/output.txt index ac6eb9b2bbc..75481749060 100644 --- a/acceptance/pipelines/test/run-control/output.txt +++ b/acceptance/pipelines/test/run-control/output.txt @@ -19,7 +19,6 @@ Update: [UUID] >>> [CLI] pipelines test my_pipeline --resume [UUID] Pipeline: my_pipeline ([UUID]) Update: [UUID] -Waiting for compute... Running tests... tests/test_orders.py::test_valid_order PASSED (1.00s) @@ -37,7 +36,6 @@ Update: [UUID] >>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s Pipeline: my_pipeline ([UUID]) Update: [UUID] -Waiting for compute... Running tests... Error: stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --resume [UUID] @@ -68,3 +66,7 @@ Update: [UUID] RUN INCOMPLETE: the update was canceled before all test results were received. Results received: none. Error: update [UUID] is CANCELED + +=== --resume rejects an update that is not a test update +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] +Error: update [UUID] is not a test update; pass the update ID that pipelines test printed diff --git a/acceptance/pipelines/test/run-control/script b/acceptance/pipelines/test/run-control/script index 41e047e621a..cf969240498 100644 --- a/acceptance/pipelines/test/run-control/script +++ b/acceptance/pipelines/test/run-control/script @@ -30,3 +30,8 @@ trace musterr $CLI pipelines test my_pipeline tests/test_orders.py title "Stopping the pipeline cancels the test update" trace $CLI pipelines stop my_pipeline trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id + +title "--resume rejects an update that is not a test update" +pipeline_id=$($CLI bundle summary -o json | jq -r '.resources.pipelines.my_pipeline.id') +regular_update_id=$($CLI pipelines start-update $pipeline_id | jq -r .update_id) +trace musterr $CLI pipelines test my_pipeline --resume $regular_update_id diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 5245ccc52fa..5da0f7bfd05 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -202,6 +202,31 @@ func getRootPath(ctx context.Context, w *databricks.WorkspaceClient, pipelineID return pipeline.Spec.RootPath, nil } +// updateTestOnly holds UpdateInfo.test_only from a Get Update response; the Go SDK does not have it yet. +// The service leaves it out where it cannot report it, such as before pipeline testing is enabled. +type updateTestOnly struct { + Update struct { + TestOnly *bool `json:"test_only"` + } `json:"update"` +} + +// checkTestUpdate returns an error if the service reports that the update is not a test-only update. +func checkTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipelineID, updateID string) error { + apiClient, err := client.New(w.Config) + if err != nil { + return fmt.Errorf("failed to create API client: %w", err) + } + var response updateTestOnly + err = apiClient.Do(ctx, http.MethodGet, fmt.Sprintf("/api/2.0/pipelines/%s/updates/%s", pipelineID, updateID), auth.WorkspaceIDHeaders(w.Config), nil, nil, &response) + if err != nil { + return err + } + if testOnly := response.Update.TestOnly; testOnly != nil && !*testOnly { + return fmt.Errorf("update %s is not a test update; pass the update ID that pipelines test printed", updateID) + } + return nil +} + // startTestUpdate starts a test-only update and returns its ID. func startTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string, req startTestUpdateRequest) (string, error) { apiClient, err := client.New(w.Config) @@ -539,6 +564,8 @@ to read the results later. To stop the update, run 'pipelines stop'.`, if err != nil { return err } + } else if err := checkTestUpdate(ctx, target.w, target.pipelineID, updateID); err != nil { + return err } target.printHeader(out, updateID) if noWait { diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 216ef06fc57..b45272ff04d 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -228,15 +228,28 @@ func (s *FakeWorkspace) PipelineGetUpdate(pipelineId, updateId string) Response update.polls++ return Response{ - Body: pipelines.GetUpdateResponse{ - Update: &pipelines.UpdateInfo{ + Body: fakeGetUpdateResponse{ + Update: fakeUpdateInfo{ UpdateId: updateId, State: update.states[update.stateIndex()], + TestOnly: update.testOnly, }, }, } } +// fakeGetUpdateResponse is a Get Update response with UpdateInfo.test_only, which the Go SDK does not have +// yet. Like the service where pipeline testing is enabled, it reports test_only for every update. +type fakeGetUpdateResponse struct { + Update fakeUpdateInfo `json:"update"` +} + +type fakeUpdateInfo struct { + UpdateId string `json:"update_id"` + State pipelines.UpdateInfoState `json:"state"` + TestOnly bool `json:"test_only"` +} + func (s *FakeWorkspace) PipelineStop(pipelineId string) Response { defer s.LockUnlock()() diff --git a/libs/testserver/pipelines_testing.go b/libs/testserver/pipelines_testing.go index d1e6ee20ab3..5c683861354 100644 --- a/libs/testserver/pipelines_testing.go +++ b/libs/testserver/pipelines_testing.go @@ -25,6 +25,7 @@ const ( // fakePipelineUpdate is a pipeline update whose state advances by one step on each Get Update call. type fakePipelineUpdate struct { pipelineID string + testOnly bool states []pipelines.UpdateInfoState polls int @@ -189,6 +190,7 @@ func (s *FakeWorkspace) startPipelineTestRun(pipelineID, updateID string, caseID ) update := &fakePipelineUpdate{ pipelineID: pipelineID, + testOnly: true, states: []pipelines.UpdateInfoState{pipelines.UpdateInfoStateWaitingForResources, pipelines.UpdateInfoStateRunning}, } diff --git a/libs/testserver/pipelines_testing_test.go b/libs/testserver/pipelines_testing_test.go index 691fe23dbd3..2636ce5ab8d 100644 --- a/libs/testserver/pipelines_testing_test.go +++ b/libs/testserver/pipelines_testing_test.go @@ -27,7 +27,19 @@ func startTestUpdate(t *testing.T, workspace *FakeWorkspace, pipelineID, body st } func getUpdateState(workspace *FakeWorkspace, pipelineID, updateID string) pipelines.UpdateInfoState { - return workspace.PipelineGetUpdate(pipelineID, updateID).Body.(pipelines.GetUpdateResponse).Update.State + return workspace.PipelineGetUpdate(pipelineID, updateID).Body.(fakeGetUpdateResponse).Update.State +} + +func TestPipelineGetUpdateReportsTestOnly(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + + testUpdateID := startTestUpdate(t, workspace, pipelineID, `{"test_only": true}`) + assert.True(t, workspace.PipelineGetUpdate(pipelineID, testUpdateID).Body.(fakeGetUpdateResponse).Update.TestOnly) + + workspace.PipelineStop(pipelineID) + updateID := startTestUpdate(t, workspace, pipelineID, `{}`) + assert.False(t, workspace.PipelineGetUpdate(pipelineID, updateID).Body.(fakeGetUpdateResponse).Update.TestOnly) } func listEvents(t *testing.T, workspace *FakeWorkspace, pipelineID string, query url.Values) ([]fakePipelineEvent, string) { From 7546147376ee83993edd11fe58bd484799e94879 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:14:17 +0000 Subject: [PATCH 03/18] Say why the pipelines test event reader cannot use the SDK iterator yet --- cmd/pipelines/test_events.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/pipelines/test_events.go b/cmd/pipelines/test_events.go index 517f1ec59c2..a272d440bbe 100644 --- a/cmd/pipelines/test_events.go +++ b/cmd/pipelines/test_events.go @@ -53,8 +53,8 @@ type testRunSummary struct { // testPipelineEvent is an entry of List Pipeline Events. The Go SDK PipelineEvent has no // details or origin.test_case_id, so test events are decoded from the raw response. -// Once the SDK has those fields, remove this type and testEventReader and switch to -// w.Pipelines.ListPipelineEvents. +// w.Pipelines.ListPipelineEvents can replace testEventReader only once the SDK has those +// fields and its iterator stops repeating order_by with page_token, which the service rejects. type testPipelineEvent struct { Id string `json:"id"` EventType string `json:"event_type"` From 2aab7bbcf92d6b16d6286931d64ac26d9daede3a Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:14:18 +0000 Subject: [PATCH 04/18] Reject a pipelines test --resume value that is not an update ID --- acceptance/pipelines/test/arguments/output.txt | 3 +++ acceptance/pipelines/test/arguments/script | 1 + cmd/pipelines/test.go | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 92c40b25bf7..d185af18226 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -76,6 +76,9 @@ Error: --resume cannot be used with --no-wait >>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --timeout 10m Error: --timeout cannot be used with --no-wait +>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678' OR '1'='1 +Error: --resume "5678' OR '1'='1" is not an update ID; pass the update ID that pipelines test printed + === JSON output is not supported >>> musterr [CLI] pipelines test --pipeline-id 1234 -o json Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script index 19e85ad4c40..ac3d1a30925 100644 --- a/acceptance/pipelines/test/arguments/script +++ b/acceptance/pipelines/test/arguments/script @@ -15,6 +15,7 @@ trace musterr $CLI pipelines test --pipeline-id 1234 tests --resume 5678 trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --pytest-arg=-m --pytest-arg=smoke trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --no-wait trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --timeout 10m +trace musterr $CLI pipelines test --pipeline-id 1234 --resume "5678' OR '1'='1" title "JSON output is not supported" trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 5da0f7bfd05..ffc37c5190f 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -30,6 +30,7 @@ import ( databricks "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/google/uuid" "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -535,6 +536,9 @@ to read the results later. To stop the update, run 'pipelines stop'.`, case selectionCount > 0: return errors.New("--resume cannot be used with test selections") } + if _, err := uuid.Parse(resumeUpdateID); err != nil { + return fmt.Errorf("--resume %q is not an update ID; pass the update ID that pipelines test printed", resumeUpdateID) + } } var target testTarget From 8fbd23bc8423d9964b4f463e8b5d14aa173f5eee Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:24:52 +0000 Subject: [PATCH 05/18] Show pipelines test case IDs in the shortest form that selects them again --- .../pipelines/test/submit-error/output.txt | 9 +-- cmd/pipelines/test.go | 76 ++++++++++++++----- cmd/pipelines/test_render.go | 13 ++-- cmd/pipelines/test_render_test.go | 24 +++++- cmd/pipelines/test_test.go | 38 ++++++++++ libs/testserver/pipelines_testing.go | 38 +--------- libs/testserver/pipelines_testing_test.go | 19 ++--- 7 files changed, 134 insertions(+), 83 deletions(-) diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt index 524700892b2..6ef5d8b4de4 100644 --- a/acceptance/pipelines/test/submit-error/output.txt +++ b/acceptance/pipelines/test/submit-error/output.txt @@ -1,13 +1,6 @@ === A submit error from the service is shown as is, and no update ID is printed >>> musterr [CLI] pipelines test --pipeline-id 1234 -Error: An active update '[UUID]' already exists for pipeline '1234'. +Error: The specified pipeline 1234 was not found. >>> print_requests.py //updates -{ - "method": "POST", - "path": "/api/2.0/pipelines/1234/updates", - "body": { - "test_only": true - } -} diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index ffc37c5190f..7e874ce02ae 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -173,14 +173,54 @@ func bundleTestSelectionPath(cwd, syncRoot, filesRoot, localPath string) (string return workspacePath, nil } -// hasRelativeSelection reports whether a selection is a relative path. -func hasRelativeSelection(selections []string) bool { - for _, selection := range selections { - if p, _, _ := strings.Cut(selection, "::"); !strings.HasPrefix(p, "/") { - return true - } +// bundleShortTestID shows a test ID whose path is under filesRoot, the deployed path of the bundle's +// synchronized files, as the local path relative to the working directory, which selects the test again. +// Other test IDs are returned unchanged. +func bundleShortTestID(cwd, syncRoot, filesRoot, id string) string { + rel, testID, ok := cutTestIDFolder(filesRoot, id) + if !ok { + return id + } + local, err := filepath.Rel(cwd, filepath.Join(syncRoot, filepath.FromSlash(rel))) + if err != nil { + return id + } + return filepath.ToSlash(local) + testID +} + +// rootPathShortTestID shows a test ID whose path is under rootPath relative to it, which --pipeline-id +// resolves against root_path to select the test again. Other test IDs are returned unchanged. +func rootPathShortTestID(rootPath, id string) string { + rel, testID, ok := cutTestIDFolder(rootPath, id) + if !ok { + return id } - return false + return rel + testID +} + +// cutTestIDFolder splits a test ID whose path is under folder into the path relative to folder and the +// "::" suffix that names the test, if any. +func cutTestIDFolder(folder, id string) (rel, testID string, ok bool) { + if folder == "" { + return "", "", false + } + p, suffix, hasSuffix := strings.Cut(id, "::") + rel, ok = strings.CutPrefix(withWorkspacePrefix(p), strings.TrimSuffix(withWorkspacePrefix(folder), "/")+"/") + if !ok { + return "", "", false + } + if hasSuffix { + testID = "::" + suffix + } + return rel, testID, true +} + +// withWorkspacePrefix adds /Workspace to an absolute workspace path without it, as the service does. +func withWorkspacePrefix(p string) string { + if p == "/Workspace" || strings.HasPrefix(p, "/Workspace/") || !strings.HasPrefix(p, "/") { + return p + } + return "/Workspace" + p } // rootPathSelection resolves a relative path against the pipeline's root_path. @@ -257,6 +297,8 @@ type testTarget struct { pipelineID string selections []string w *databricks.WorkspaceClient + // shortID shows a test ID in the shortest form that selects the test again. + shortID func(id string) string } func (t testTarget) printHeader(out io.Writer, updateID string) { @@ -279,13 +321,9 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - var rootPath string - if hasRelativeSelection(args) { - var err error - rootPath, err = getRootPath(ctx, w, pipelineID) - if err != nil { - return testTarget{}, err - } + rootPath, err := getRootPath(ctx, w, pipelineID) + if err != nil { + return testTarget{}, err } selections, err := resolveTestSelections(args, func(p string) (string, error) { return rootPathSelection(rootPath, p) @@ -293,7 +331,8 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) if err != nil { return testTarget{}, err } - return testTarget{pipelineID: pipelineID, selections: selections, w: w}, nil + shortID := func(id string) string { return rootPathShortTestID(rootPath, id) } + return testTarget{pipelineID: pipelineID, selections: selections, w: w, shortID: shortID}, nil } // resolveTestTargetInBundle identifies the pipeline by its bundle KEY, which must precede any selections. @@ -339,7 +378,8 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string, resume bool) ( if err != nil { return testTarget{}, err } - return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx)}, nil + shortID := func(id string) string { return bundleShortTestID(cwd, b.SyncRootPath, filesRoot, id) } + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx), shortID: shortID}, nil } // resumeCommand returns the command that reads the result of the update later. @@ -585,7 +625,7 @@ to read the results later. To stop the update, run 'pipelines stop'.`, pipelineID: target.pipelineID, updateID: updateID, events: events, - printer: &testProgressPrinter{out: out}, + printer: &testProgressPrinter{out: out, shortID: target.shortID}, run: newTestRun(), } @@ -608,7 +648,7 @@ to read the results later. To stop the update, run 'pipelines stop'.`, } incompleteReason := watcher.run.incompleteReason(state) - renderTestResult(out, watcher.run, incompleteReason) + renderTestResult(out, watcher.run, incompleteReason, target.shortID) return testRunError(updateID, state, watcher.run, incompleteReason) } diff --git a/cmd/pipelines/test_render.go b/cmd/pipelines/test_render.go index d92f9cff701..c16c83689c9 100644 --- a/cmd/pipelines/test_render.go +++ b/cmd/pipelines/test_render.go @@ -11,6 +11,7 @@ import ( // testProgressPrinter prints the phase of a test update and each case as it finishes. type testProgressPrinter struct { out io.Writer + shortID func(id string) string waiting bool running bool } @@ -43,14 +44,14 @@ func (p *testProgressPrinter) startRunning() { func (p *testProgressPrinter) caseFinished(c *testCaseResult) { p.startRunning() if d, ok := c.duration(); ok { - fmt.Fprintf(p.out, "%s %s (%.2fs)\n", c.id, c.status, d.Seconds()) + fmt.Fprintf(p.out, "%s %s (%.2fs)\n", p.shortID(c.id), c.status, d.Seconds()) } else { - fmt.Fprintf(p.out, "%s %s\n", c.id, c.status) + fmt.Fprintf(p.out, "%s %s\n", p.shortID(c.id), c.status) } } // renderTestFailures prints the details of every failed or errored case. -func renderTestFailures(out io.Writer, run *testRun) { +func renderTestFailures(out io.Writer, run *testRun, shortID func(id string) string) { var failed []*testCaseResult for _, c := range run.cases { if c.status == testCaseFailed || c.status == testCaseError { @@ -64,7 +65,7 @@ func renderTestFailures(out io.Writer, run *testRun) { fmt.Fprintln(out) fmt.Fprintln(out, "FAILURES") for _, c := range failed { - fmt.Fprintln(out, c.id) + fmt.Fprintln(out, shortID(c.id)) details := "" if c.errorDetail != nil { details = errorDetailText(c.errorDetail) @@ -94,8 +95,8 @@ func formatTestCounts(s testRunSummary) string { // renderTestResult prints the end of a test run: failure details, then the summary or why // the result is incomplete. -func renderTestResult(out io.Writer, run *testRun, incompleteReason string) { - renderTestFailures(out, run) +func renderTestResult(out io.Writer, run *testRun, incompleteReason string, shortID func(id string) string) { + renderTestFailures(out, run, shortID) fmt.Fprintln(out) if incompleteReason != "" { diff --git a/cmd/pipelines/test_render_test.go b/cmd/pipelines/test_render_test.go index 5129b001cdb..7ca6aab7bfc 100644 --- a/cmd/pipelines/test_render_test.go +++ b/cmd/pipelines/test_render_test.go @@ -2,15 +2,35 @@ package pipelines import ( "bytes" + "strings" "testing" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" ) +func sameTestID(id string) string { return id } + +func TestRenderShortTestIDs(t *testing.T) { + shortID := func(id string) string { return strings.TrimPrefix(id, "/Workspace/root/") } + var out bytes.Buffer + p := &testProgressPrinter{out: &out, shortID: shortID} + run := newTestRun() + p.caseFinished(run.add(caseEvent(t, "/Workspace/root/t.py::test_x", "FAILED", "2026-01-01T00:00:00.000Z"))) + renderTestFailures(&out, run, shortID) + + assert.Equal(t, `Running tests... +t.py::test_x FAILED + +FAILURES +t.py::test_x + No details were reported. +`, out.String()) +} + func TestTestProgressPrinter(t *testing.T) { var out bytes.Buffer - p := &testProgressPrinter{out: &out} + p := &testProgressPrinter{out: &out, shortID: sameTestID} run := newTestRun() p.updateState(pipelines.UpdateInfoStateQueued) @@ -122,7 +142,7 @@ Results received: none. run.add(e) } var out bytes.Buffer - renderTestResult(&out, run, run.incompleteReason(tc.state)) + renderTestResult(&out, run, run.incompleteReason(tc.state), sameTestID) assert.Equal(t, tc.want, out.String()) }) } diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index d36f999d491..775a9727d28 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -68,6 +68,44 @@ func TestResolveTestSelections(t *testing.T) { assert.EqualError(t, err, `cannot resolve test selection "missing::test_x": not found`) } +func TestBundleShortTestID(t *testing.T) { + syncRoot := t.TempDir() + filesRoot := "/Workspace/Users/someone/.bundle/orders/dev/files" + id := filesRoot + "/tests/test_a.py::TestA::test_x[a::b]" + + // The local path, relative to where the user stands, selects the test again. + assert.Equal(t, "tests/test_a.py::TestA::test_x[a::b]", bundleShortTestID(syncRoot, syncRoot, filesRoot, id)) + assert.Equal(t, "test_a.py::TestA::test_x[a::b]", bundleShortTestID(filepath.Join(syncRoot, "tests"), syncRoot, filesRoot, id)) + assert.Equal(t, "../tests/test_a.py::TestA::test_x[a::b]", bundleShortTestID(filepath.Join(syncRoot, "src"), syncRoot, filesRoot, id)) + assert.Equal(t, "tests", bundleShortTestID(syncRoot, syncRoot, filesRoot, filesRoot+"/tests")) + + // Other IDs are shown as the service sends them. + for _, other := range []string{ + "/Workspace/Users/someone/tests/test_s.py::test_s", + filesRoot + "2/tests/test_a.py::test_x", + "tests/test_a.py::test_x", + } { + assert.Equal(t, other, bundleShortTestID(syncRoot, syncRoot, filesRoot, other)) + } + assert.Equal(t, id, bundleShortTestID(syncRoot, syncRoot, "", id)) +} + +func TestRootPathShortTestID(t *testing.T) { + rootPath := "/Workspace/Users/someone/orders" + assert.Equal(t, "tests/test_a.py::test_x", rootPathShortTestID(rootPath, rootPath+"/tests/test_a.py::test_x")) + assert.Equal(t, "tests/test_a.py::test_x", rootPathShortTestID(rootPath+"/", rootPath+"/tests/test_a.py::test_x")) + // The service accepts workspace paths with or without the /Workspace prefix. + assert.Equal(t, "tests/test_a.py::test_x", rootPathShortTestID("/Users/someone/orders", rootPath+"/tests/test_a.py::test_x")) + for _, other := range []string{ + "/Workspace/Users/someone/shared/test_s.py::test_s", + rootPath + "2/test_a.py::test_x", + "tests/test_a.py::test_x", + } { + assert.Equal(t, other, rootPathShortTestID(rootPath, other)) + } + assert.Equal(t, rootPath+"/tests/test_a.py::test_x", rootPathShortTestID("", rootPath+"/tests/test_a.py::test_x")) +} + // Source-linked deployments run the source files, as TranslatePaths maps them. func TestBundleFilesRootSourceLinked(t *testing.T) { b := &bundle.Bundle{SyncRootPath: "/Workspace/Users/someone/orders"} diff --git a/libs/testserver/pipelines_testing.go b/libs/testserver/pipelines_testing.go index 5c683861354..9d2106f2995 100644 --- a/libs/testserver/pipelines_testing.go +++ b/libs/testserver/pipelines_testing.go @@ -105,42 +105,11 @@ type pipelineTestRequest struct { var pipelineTestFunctionRegexp = regexp.MustCompile(`(?m)^def (test_\w+)\(`) -// pipelineTestRootDir returns pytest's rootdir for a test run, as the service sets it: the deepest -// folder that holds rootPath and every test source or, without rootPath, the folder of every test -// source. It is "" when there is neither. The CLI applies the same rule (testIDBase in -// cmd/pipelines), so acceptance tests cannot catch a wrong rule; only a run on a real workspace checks it. -func pipelineTestRootDir(rootPath string, sources []string) string { - var folders [][]string - if rootPath != "" { - folders = append(folders, strings.Split(strings.Trim(rootPath, "/"), "/")) - } - for _, source := range sources { - segments := strings.Split(strings.Trim(strings.TrimSuffix(source, "/**"), "/"), "/") - if rootPath == "" { - segments = segments[:len(segments)-1] - } - folders = append(folders, segments) - } - if len(folders) == 0 { - return "" - } - common := folders[0] - for _, folder := range folders[1:] { - n := 0 - for n < len(common) && n < len(folder) && common[n] == folder[n] { - n++ - } - common = common[:n] - } - return "/" + strings.Join(common, "/") -} - // collectPipelineTests stands in for pytest collection: it finds the top-level "def test_*" // functions in the workspace test_*.py files under each selection, or under rootPath without one. -// Like origin.test_case_id, test IDs are relative to pytest's rootdir, such as -// tests/test_orders.py::test_backfill. +// Like origin.test_case_id, a test ID is the absolute workspace path of its file followed by the +// pytest test ID, such as /Workspace/Users/someone/orders/tests/test_orders.py::test_backfill. func (s *FakeWorkspace) collectPipelineTests(rootPath string, selection []string) []string { - rootDir := pipelineTestRootDir(rootPath, nil) if len(selection) == 0 && rootPath != "" { selection = []string{rootPath} } @@ -165,9 +134,6 @@ func (s *FakeWorkspace) collectPipelineTests(rootPath string, selection []string for _, match := range pipelineTestFunctionRegexp.FindAllSubmatch(s.files[filePath].Data, -1) { name := string(match[1]) id := filePath + "::" + name - if rootDir != "" { - id = strings.TrimPrefix(id, rootDir+"/") - } if (testName == "" || testName == name) && !slices.Contains(ids, id) { ids = append(ids, id) } diff --git a/libs/testserver/pipelines_testing_test.go b/libs/testserver/pipelines_testing_test.go index 2636ce5ab8d..f786b18aed1 100644 --- a/libs/testserver/pipelines_testing_test.go +++ b/libs/testserver/pipelines_testing_test.go @@ -54,12 +54,13 @@ func TestCollectPipelineTests(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") createTestPipelineWithTests(t, workspace) - // Like the service, IDs are relative to pytest's rootdir, which is the root path here. - assert.Equal(t, []string{"tests/test_orders.py::test_ok", "tests/test_orders.py::test_fails", "tests/test_slow.py::test_hang"}, + // Like origin.test_case_id, an ID is the absolute workspace path of its file and the pytest test ID. + tests := testRootPath + "/tests/" + assert.Equal(t, []string{tests + "test_orders.py::test_ok", tests + "test_orders.py::test_fails", tests + "test_slow.py::test_hang"}, workspace.collectPipelineTests(testRootPath, nil)) - assert.Equal(t, []string{"tests/test_orders.py::test_fails"}, + assert.Equal(t, []string{tests + "test_orders.py::test_fails"}, workspace.collectPipelineTests(testRootPath, []string{testRootPath + "/tests/test_orders.py::test_fails", testRootPath + "/tests/test_orders.py::test_fails"})) - assert.Equal(t, []string{"tests/helpers.py::test_helper"}, + assert.Equal(t, []string{tests + "helpers.py::test_helper"}, workspace.collectPipelineTests(testRootPath, []string{testRootPath + "/tests/helpers.py"})) assert.Equal(t, []string{testRootPath + "/tests/test_slow.py::test_hang"}, workspace.collectPipelineTests("", []string{testRootPath + "/tests/test_slow.py"})) @@ -82,7 +83,7 @@ func TestPipelineTestUpdate(t *testing.T) { for _, e := range events { statuses = append(statuses, e.Origin.TestCaseId+" "+e.Details["test_case_progress"].(map[string]any)["status"].(string)) } - file := "tests/test_orders.py" + file := testRootPath + "/tests/test_orders.py" assert.Equal(t, []string{ file + "::test_ok PENDING", file + "::test_fails PENDING", @@ -165,14 +166,6 @@ func TestPipelineStartUpdateRejectsTestOptionsWithoutTestOnly(t *testing.T) { assert.Equal(t, 400, response.StatusCode) } -func TestPipelineTestRootDir(t *testing.T) { - assert.Equal(t, "/Workspace/Users/u/files", pipelineTestRootDir("/Workspace/Users/u/files/src/orders", []string{"/Workspace/Users/u/files/tests/**"})) - assert.Equal(t, "/Workspace/Users/u/orders", pipelineTestRootDir("/Workspace/Users/u/orders", nil)) - assert.Equal(t, "/Workspace/Users/u", pipelineTestRootDir("", []string{"/Workspace/Users/u/tests/**"})) - assert.Equal(t, "/Workspace/Users/u/tests", pipelineTestRootDir("", []string{"/Workspace/Users/u/tests/test_x.py"})) - assert.Empty(t, pipelineTestRootDir("", nil)) -} - func TestPipelineFiniteUpdateIsActiveUntilItEnds(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") pipelineID := createTestPipelineWithTests(t, workspace) From 64f8f94dad65879e4979c98003676c1dd5a31509 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:30:21 +0000 Subject: [PATCH 06/18] Explain a pipelines test run whose tests failed by its summary alone, also when the update completed --- cmd/pipelines/test.go | 3 ++- cmd/pipelines/test_test.go | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 7e874ce02ae..5e886b1f5ff 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -479,7 +479,8 @@ func testRunError(updateID string, state pipelines.UpdateInfoState, run *testRun // A test update fails when its tests fail; the printed result explains it, unless the // update ended in another way or reports an error of its own. The service's own verdict, // such as "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed", is not. - if testsFailed && state == pipelines.UpdateInfoStateFailed && run.fatalError == "" { + ended := state == pipelines.UpdateInfoStateFailed || state == pipelines.UpdateInfoStateCompleted + if testsFailed && ended && run.fatalError == "" { return root.ErrAlreadyPrinted } } else if state == pipelines.UpdateInfoStateCompleted { diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 775a9727d28..92c7c910418 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -229,6 +229,22 @@ func TestTestRunError(t *testing.T) { state: pipelines.UpdateInfoStateFailed, wantErr: "update u1 is FAILED: [PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed.", }, + { + name: "tests failed although the update completed", + events: func() []testPipelineEvent { + return []testPipelineEvent{caseEvent(t, "t::a", "FAILED", ""), summaryEvent(t, 1, 0, 1, 0, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, + { + name: "no tests collected although the update completed", + events: func() []testPipelineEvent { + return []testPipelineEvent{summaryEvent(t, 0, 0, 0, 0, 0)} + }, + state: pipelines.UpdateInfoStateCompleted, + wantErr: root.ErrAlreadyPrinted.Error(), + }, { name: "canceled after tests failed", events: func() []testPipelineEvent { From 048d6108849913cf60d0b933c2de9b1ce585c853 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:35:12 +0000 Subject: [PATCH 07/18] Test that pipelines test --resume reads results when the service does not report test_only --- cmd/pipelines/test_test.go | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 92c7c910418..c37d696ba3c 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -2,6 +2,8 @@ package pipelines import ( "errors" + "net/http" + "net/http/httptest" "os" "path/filepath" "testing" @@ -9,6 +11,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/cmd/root" + databricks "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" @@ -182,6 +185,46 @@ func TestTestTargetResumeCommand(t *testing.T) { } } +func TestCheckTestUpdate(t *testing.T) { + const updateID = "11111111-1111-1111-1111-111111111111" + tests := []struct { + name string + update string + wantErr string + }{ + // Where the service cannot report test_only, as before pipeline testing is enabled, the + // update may be a test update, so the command reads its results. + {name: "test_only not reported", update: `{"state": "COMPLETED"}`}, + {name: "test update", update: `{"state": "COMPLETED", "test_only": true}`}, + { + name: "regular update", + update: `{"state": "COMPLETED", "test_only": false}`, + wantErr: "update " + updateID + " is not a test update; pass the update ID that pipelines test printed", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.0/pipelines/p1/updates/"+updateID { + _, _ = w.Write([]byte(`{"update": ` + tc.update + `}`)) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: srv.URL, Token: "token"}) + require.NoError(t, err) + + err = checkTestUpdate(t.Context(), w, "p1", updateID) + if tc.wantErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.wantErr) + } + }) + } +} + func TestTestRunError(t *testing.T) { tests := []struct { name string From 7e915ca9928ad3935c985736669d9d1d62692284 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:35:12 +0000 Subject: [PATCH 08/18] Reject test options in the fake pipelines service as the service does --- libs/testserver/pipelines.go | 8 ++++++++ libs/testserver/pipelines_testing.go | 25 +++++++++++++++++++++++ libs/testserver/pipelines_testing_test.go | 21 +++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index b45272ff04d..48e0744acfb 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -181,6 +181,14 @@ func (s *FakeWorkspace) PipelineStartUpdate(pipelineId string, body []byte) Resp Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": "test_options requires test_only to be true."}, } } + if test.TestOptions != nil { + if message := pipelineTestOptionsError(test.TestOptions.PytestArgs, test.TestOptions.TestSelection); message != "" { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, + } + } + } updateId := nextUUID() update := &fakePipelineUpdate{ diff --git a/libs/testserver/pipelines_testing.go b/libs/testserver/pipelines_testing.go index 9d2106f2995..16d91436d9b 100644 --- a/libs/testserver/pipelines_testing.go +++ b/libs/testserver/pipelines_testing.go @@ -105,6 +105,31 @@ type pipelineTestRequest struct { var pipelineTestFunctionRegexp = regexp.MustCompile(`(?m)^def (test_\w+)\(`) +// pipelineTestOptionsError returns the service's message for test options that it rejects, or "". Like +// PipelineTestSelection in the service, pytest_args holds -k and -m, each followed by a value that is not +// empty and does not start with "-", and each test selection is an absolute workspace path. +func pipelineTestOptionsError(pytestArgs, selection []string) string { + for i := 0; i < len(pytestArgs); i += 2 { + option := pytestArgs[i] + switch { + case option != "-k" && option != "-m": + return fmt.Sprintf("pytest_args accepts only -k and -m, each followed by its value. Got '%s'.", option) + case i+1 == len(pytestArgs): + return fmt.Sprintf("The pytest_args option '%s' must be followed by its value.", option) + case strings.TrimSpace(pytestArgs[i+1]) == "": + return fmt.Sprintf("The value of the pytest_args option '%s' must not be empty.", option) + case strings.HasPrefix(pytestArgs[i+1], "-"): + return fmt.Sprintf("The value of the pytest_args option '%s' must not start with '-'. Got '%s'.", option, pytestArgs[i+1]) + } + } + for _, entry := range selection { + if p, _, _ := strings.Cut(strings.TrimSpace(entry), "::"); !strings.HasPrefix(p, "/") { + return fmt.Sprintf("The test selection '%s' must be an absolute workspace path, such as /Workspace/Users/someone@example.com/orders/tests/test_orders.py.", entry) + } + } + return "" +} + // collectPipelineTests stands in for pytest collection: it finds the top-level "def test_*" // functions in the workspace test_*.py files under each selection, or under rootPath without one. // Like origin.test_case_id, a test ID is the absolute workspace path of its file followed by the diff --git a/libs/testserver/pipelines_testing_test.go b/libs/testserver/pipelines_testing_test.go index f786b18aed1..960bc355207 100644 --- a/libs/testserver/pipelines_testing_test.go +++ b/libs/testserver/pipelines_testing_test.go @@ -166,6 +166,27 @@ func TestPipelineStartUpdateRejectsTestOptionsWithoutTestOnly(t *testing.T) { assert.Equal(t, 400, response.StatusCode) } +func TestPipelineStartUpdateRejectsInvalidTestOptions(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + pipelineID := createTestPipelineWithTests(t, workspace) + + // The messages are the service's, from PipelineTestSelection. + for options, message := range map[string]string{ + `{"pytest_args": ["-korders"]}`: "pytest_args accepts only -k and -m, each followed by its value. Got '-korders'.", + `{"pytest_args": ["-x"]}`: "pytest_args accepts only -k and -m, each followed by its value. Got '-x'.", + `{"pytest_args": ["-m"]}`: "The pytest_args option '-m' must be followed by its value.", + `{"pytest_args": ["-k", " "]}`: "The value of the pytest_args option '-k' must not be empty.", + `{"pytest_args": ["-k", "-m"]}`: "The value of the pytest_args option '-k' must not start with '-'. Got '-m'.", + `{"test_selection": ["tests/test_a.py::t"]}`: "The test selection 'tests/test_a.py::t' must be an absolute workspace path, such as /Workspace/Users/someone@example.com/orders/tests/test_orders.py.", + } { + response := workspace.PipelineStartUpdate(pipelineID, []byte(`{"test_only": true, "test_options": `+options+`}`)) + assert.Equal(t, 400, response.StatusCode, options) + assert.Equal(t, map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, response.Body, options) + } + + startTestUpdate(t, workspace, pipelineID, `{"test_only": true, "test_options": {"pytest_args": ["-k", "orders", "-m", "smoke"], "test_selection": ["`+testRootPath+`/tests"]}}`) +} + func TestPipelineFiniteUpdateIsActiveUntilItEnds(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") pipelineID := createTestPipelineWithTests(t, workspace) From d8b8b553db62eab40288776bcf74a162212a5b15 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:49:39 +0000 Subject: [PATCH 09/18] Read the errored count of the pipelines test summary, as the service names it --- cmd/pipelines/test_events.go | 2 +- cmd/pipelines/test_events_test.go | 6 +++--- libs/testserver/pipelines_testing.go | 2 +- libs/testserver/pipelines_testing_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/pipelines/test_events.go b/cmd/pipelines/test_events.go index a272d440bbe..ca08ecd33ab 100644 --- a/cmd/pipelines/test_events.go +++ b/cmd/pipelines/test_events.go @@ -48,7 +48,7 @@ type testRunSummary struct { Passed int `json:"passed"` Failed int `json:"failed"` Skipped int `json:"skipped"` - Errors int `json:"errors"` + Errors int `json:"errored"` } // testPipelineEvent is an entry of List Pipeline Events. The Go SDK PipelineEvent has no diff --git a/cmd/pipelines/test_events_test.go b/cmd/pipelines/test_events_test.go index 36a2c6a5fbb..d76cc1efb4f 100644 --- a/cmd/pipelines/test_events_test.go +++ b/cmd/pipelines/test_events_test.go @@ -26,12 +26,12 @@ func caseEvent(t *testing.T, id, status, timestamp string, extra ...string) test }`, testEventSeq, timestamp, id, status, strings.Join(append([]string{""}, extra...), ","))) } -func summaryEvent(t *testing.T, total, passed, failed, skipped, errors int) testPipelineEvent { +func summaryEvent(t *testing.T, total, passed, failed, skipped, errored int) testPipelineEvent { testEventSeq++ return decodeTestEvent(t, fmt.Sprintf(`{ "id": "e%d", "event_type": "test_run_summary", - "details": {"test_run_summary": {"total": %d, "passed": %d, "failed": %d, "skipped": %d, "errors": %d}} - }`, testEventSeq, total, passed, failed, skipped, errors)) + "details": {"test_run_summary": {"total": %d, "passed": %d, "failed": %d, "skipped": %d, "errored": %d}} + }`, testEventSeq, total, passed, failed, skipped, errored)) } func decodeTestEvent(t *testing.T, data string) testPipelineEvent { diff --git a/libs/testserver/pipelines_testing.go b/libs/testserver/pipelines_testing.go index 16d91436d9b..23b8e498863 100644 --- a/libs/testserver/pipelines_testing.go +++ b/libs/testserver/pipelines_testing.go @@ -263,7 +263,7 @@ func (s *FakeWorkspace) startPipelineTestRun(pipelineID, updateID string, caseID "passed": counts["passed"], "failed": counts["failed"], "skipped": counts["skipped"], - "errors": counts["errors"], + "errored": counts["errors"], }}, }) // Like the service, a run whose tests failed or errored ends with the test verdict as the diff --git a/libs/testserver/pipelines_testing_test.go b/libs/testserver/pipelines_testing_test.go index 960bc355207..c58bc074ba2 100644 --- a/libs/testserver/pipelines_testing_test.go +++ b/libs/testserver/pipelines_testing_test.go @@ -99,7 +99,7 @@ func TestPipelineTestUpdate(t *testing.T) { events, _ = listEvents(t, workspace, pipelineID, query) require.Len(t, events, 3) assert.Equal(t, last.Id, events[0].Id) - assert.Equal(t, map[string]any{"total": 2, "passed": 1, "failed": 1, "skipped": 0, "errors": 0}, events[1].Details["test_run_summary"]) + assert.Equal(t, map[string]any{"total": 2, "passed": 1, "failed": 1, "skipped": 0, "errored": 0}, events[1].Details["test_run_summary"]) // Like the service, the update's error is the test verdict. require.NotNil(t, events[2].Error) assert.True(t, events[2].Error.Fatal) From d2e5029039c129831113d29876058b34eca08897 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:50:59 +0000 Subject: [PATCH 10/18] Reject an empty pipelines test --resume instead of starting a new run --- acceptance/pipelines/test/arguments/output.txt | 3 +++ acceptance/pipelines/test/arguments/script | 1 + cmd/pipelines/test.go | 8 +++++--- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index d185af18226..ec282fab365 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -79,6 +79,9 @@ Error: --timeout cannot be used with --no-wait >>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678' OR '1'='1 Error: --resume "5678' OR '1'='1" is not an update ID; pass the update ID that pipelines test printed +>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume +Error: --resume "" is not an update ID; pass the update ID that pipelines test printed + === JSON output is not supported >>> musterr [CLI] pipelines test --pipeline-id 1234 -o json Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script index ac3d1a30925..ae91c307706 100644 --- a/acceptance/pipelines/test/arguments/script +++ b/acceptance/pipelines/test/arguments/script @@ -16,6 +16,7 @@ trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --pytest-arg= trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --no-wait trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --timeout 10m trace musterr $CLI pipelines test --pipeline-id 1234 --resume "5678' OR '1'='1" +trace musterr $CLI pipelines test --pipeline-id 1234 --resume "" title "JSON output is not supported" trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 5e886b1f5ff..31820800f9b 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -564,7 +564,9 @@ to read the results later. To stop the update, run 'pipelines stop'.`, if noWait && timeout > 0 { return errors.New("--timeout cannot be used with --no-wait") } - if resumeUpdateID != "" { + // An empty --resume, such as from an unset shell variable, must not start a new run. + resume := cmd.Flags().Changed("resume") + if resume { selectionCount := len(args) if pipelineID == "" { selectionCount = max(len(args)-1, 0) @@ -592,7 +594,7 @@ to read the results later. To stop the update, run 'pipelines stop'.`, if !bundle.HasRoot(ctx) { return errors.New("outside a project, identify the pipeline with --pipeline-id") } - target, err = resolveTestTargetInBundle(cmd, args, resumeUpdateID != "") + target, err = resolveTestTargetInBundle(cmd, args, resume) } if err != nil { return err @@ -603,7 +605,7 @@ to read the results later. To stop the update, run 'pipelines stop'.`, out := cmd.OutOrStdout() updateID := resumeUpdateID - if updateID == "" { + if !resume { req := newStartTestUpdateRequest(target.selections, pytestArgs) updateID, err = startTestUpdate(ctx, target.w, target.pipelineID, req) if err != nil { From b6fec7848e02c81bb439b00cc69b900eeefcd45b Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:51:58 +0000 Subject: [PATCH 11/18] Say that pipelines test succeeds when at least one test was collected --- acceptance/pipelines/test/arguments/output.txt | 4 ++-- cmd/pipelines/test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index ec282fab365..c2d17764f80 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -23,8 +23,8 @@ example: databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' -The command waits for the results and succeeds only if at least one test ran -and none failed or errored. --no-wait returns after the update starts. +The command waits for the results and succeeds only if at least one test was +collected and none failed or errored. --no-wait returns after the update starts. --timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume to read the results later. To stop the update, run 'pipelines stop'. diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 31820800f9b..a66b07e73b2 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -521,8 +521,8 @@ example: databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' -The command waits for the results and succeeds only if at least one test ran -and none failed or errored. --no-wait returns after the update starts. +The command waits for the results and succeeds only if at least one test was +collected and none failed or errored. --no-wait returns after the update starts. --timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume to read the results later. To stop the update, run 'pipelines stop'.`, } From e1cab2e7415977520d65a68c5c8a8086a6f1b75e Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:51:59 +0000 Subject: [PATCH 12/18] Drop the unread level and message from pipelines test events --- cmd/pipelines/test_events.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/pipelines/test_events.go b/cmd/pipelines/test_events.go index ca08ecd33ab..4db20555587 100644 --- a/cmd/pipelines/test_events.go +++ b/cmd/pipelines/test_events.go @@ -56,11 +56,9 @@ type testRunSummary struct { // w.Pipelines.ListPipelineEvents can replace testEventReader only once the SDK has those // fields and its iterator stops repeating order_by with page_token, which the service rejects. type testPipelineEvent struct { - Id string `json:"id"` - EventType string `json:"event_type"` - Timestamp string `json:"timestamp"` - Level pipelines.EventLevel `json:"level"` - Message string `json:"message"` + Id string `json:"id"` + EventType string `json:"event_type"` + Timestamp string `json:"timestamp"` Origin struct { TestCaseId string `json:"test_case_id"` } `json:"origin"` From 93b0bc2565cf27661fe9bf72bb5e81df891e90ca Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 2 Nov 2026 19:18:04 +0000 Subject: [PATCH 13/18] Use the standard library uuid package in pipelines test The command validated --resume with github.com/google/uuid, which made it a direct dependency without a license annotation. cmd/pipelines/history.go already uses the standard library package. --- cmd/pipelines/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index a66b07e73b2..dda6361c4e9 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -14,6 +14,7 @@ import ( "strings" "syscall" "time" + "uuid" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" @@ -30,7 +31,6 @@ import ( databricks "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/pipelines" - "github.com/google/uuid" "github.com/spf13/cobra" "github.com/spf13/pflag" ) From 08b877b58cec1e664797de189dc0e24ab945faeb Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 2 Nov 2026 19:46:34 +0000 Subject: [PATCH 14/18] Link the pipelines test changelog entries to #7298 --- .nextchanges/cli/pipelines-test-results.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/cli/pipelines-test-results.md b/.nextchanges/cli/pipelines-test-results.md index 72217847c99..c8973965460 100644 --- a/.nextchanges/cli/pipelines-test-results.md +++ b/.nextchanges/cli/pipelines-test-results.md @@ -1 +1 @@ -* `databricks pipelines test` waits for the test results, prints them, and fails when a test fails or the results are incomplete. ([#7272](https://github.com/databricks/cli/pull/7272)) +* `databricks pipelines test` waits for the test results, prints them, and fails when a test fails or the results are incomplete. ([#7298](https://github.com/databricks/cli/pull/7298)) From 4936091305bae19757c9629b2cfeea2f59834174 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 2 Nov 2026 23:08:11 +0000 Subject: [PATCH 15/18] Document origin.test_case_id as an absolute workspace path in pipelines test --- cmd/pipelines/test_events.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/pipelines/test_events.go b/cmd/pipelines/test_events.go index 4db20555587..b6996c437e3 100644 --- a/cmd/pipelines/test_events.go +++ b/cmd/pipelines/test_events.go @@ -149,8 +149,8 @@ func (r *testEventReader) read(ctx context.Context, sinceTimestamp string) ([]te // testCaseResult is what the events report about one test case. type testCaseResult struct { - // id is origin.test_case_id: the pytest test ID, relative to pytest's rootdir, such as - // tests/test_orders.py::test_backfill. + // id is origin.test_case_id: the absolute workspace path of the test file followed by the pytest + // test ID, such as /Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill. id string status testCaseStatus // finals counts final outcomes; a complete result has exactly one per case. From 92fb665ebb823c8c3d144bdb3c6f15305cfdd227 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 2 Nov 2026 23:08:17 +0000 Subject: [PATCH 16/18] Test each pipelines test result behavior once Unit tests keep the edge cases that the fake service cannot produce; the acceptance scenarios own the outcomes, output, resume and conflict paths. --- .../pipelines/test/run-control/output.txt | 8 -- acceptance/pipelines/test/run-control/script | 3 - cmd/pipelines/test_events_test.go | 21 ----- cmd/pipelines/test_render_test.go | 73 ----------------- cmd/pipelines/test_test.go | 81 ++++--------------- 5 files changed, 14 insertions(+), 172 deletions(-) diff --git a/acceptance/pipelines/test/run-control/output.txt b/acceptance/pipelines/test/run-control/output.txt index 75481749060..e1a1895d5f3 100644 --- a/acceptance/pipelines/test/run-control/output.txt +++ b/acceptance/pipelines/test/run-control/output.txt @@ -46,14 +46,6 @@ Update: [UUID] Running tests... Error: stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --profile run-control --resume [UUID] -=== Another update cannot start while the test update is active ->>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Error: An active update '[UUID]' already exists for pipeline '[UUID]'. - === Stopping the pipeline cancels the test update >>> [CLI] pipelines stop my_pipeline Stopping my_pipeline... diff --git a/acceptance/pipelines/test/run-control/script b/acceptance/pipelines/test/run-control/script index cf969240498..c640f6174a7 100644 --- a/acceptance/pipelines/test/run-control/script +++ b/acceptance/pipelines/test/run-control/script @@ -24,9 +24,6 @@ trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id --timeout unset DATABRICKS_CONFIG_FILE rm .databrickscfg -title "Another update cannot start while the test update is active" -trace musterr $CLI pipelines test my_pipeline tests/test_orders.py - title "Stopping the pipeline cancels the test update" trace $CLI pipelines stop my_pipeline trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id diff --git a/cmd/pipelines/test_events_test.go b/cmd/pipelines/test_events_test.go index d76cc1efb4f..eedffcc4a9f 100644 --- a/cmd/pipelines/test_events_test.go +++ b/cmd/pipelines/test_events_test.go @@ -95,18 +95,6 @@ func TestTestRunIncompleteReason(t *testing.T) { state pipelines.UpdateInfoState want string }{ - { - name: "no summary after failure", - events: []testPipelineEvent{passed()}, - state: pipelines.UpdateInfoStateFailed, - want: "the update failed before all test results were received", - }, - { - name: "no summary after cancel", - events: nil, - state: pipelines.UpdateInfoStateCanceled, - want: "the update was canceled before all test results were received", - }, { name: "no summary after completion", events: []testPipelineEvent{passed()}, @@ -171,15 +159,6 @@ func TestTestRunIncompleteReason(t *testing.T) { } } -func TestTestRunFatalError(t *testing.T) { - run := newTestRun() - run.add(decodeTestEvent(t, `{ - "id": "fatal", "event_type": "update_progress", "level": "ERROR", - "error": {"fatal": true, "exceptions": [{"class_name": "ComputeTerminated", "message": "Compute stopped."}]} - }`)) - assert.Equal(t, "ComputeTerminated: Compute stopped.", run.fatalError) -} - func TestTestEventReaderFollowsPages(t *testing.T) { server := testserver.New(t) var queries []url.Values diff --git a/cmd/pipelines/test_render_test.go b/cmd/pipelines/test_render_test.go index 7ca6aab7bfc..88368f17c9d 100644 --- a/cmd/pipelines/test_render_test.go +++ b/cmd/pipelines/test_render_test.go @@ -2,7 +2,6 @@ package pipelines import ( "bytes" - "strings" "testing" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -11,23 +10,6 @@ import ( func sameTestID(id string) string { return id } -func TestRenderShortTestIDs(t *testing.T) { - shortID := func(id string) string { return strings.TrimPrefix(id, "/Workspace/root/") } - var out bytes.Buffer - p := &testProgressPrinter{out: &out, shortID: shortID} - run := newTestRun() - p.caseFinished(run.add(caseEvent(t, "/Workspace/root/t.py::test_x", "FAILED", "2026-01-01T00:00:00.000Z"))) - renderTestFailures(&out, run, shortID) - - assert.Equal(t, `Running tests... -t.py::test_x FAILED - -FAILURES -t.py::test_x - No details were reported. -`, out.String()) -} - func TestTestProgressPrinter(t *testing.T) { var out bytes.Buffer p := &testProgressPrinter{out: &out, shortID: sameTestID} @@ -77,61 +59,6 @@ t::setup No details were reported. 3 tests: 1 passed, 1 failed, 0 skipped, 1 error -`, - }, - { - name: "several errors", - events: func() []testPipelineEvent { - return []testPipelineEvent{ - caseEvent(t, "t::setup", "ERROR", ""), - caseEvent(t, "t::teardown", "ERROR", ""), - summaryEvent(t, 2, 0, 0, 0, 2), - } - }, - state: pipelines.UpdateInfoStateFailed, - want: ` -FAILURES -t::setup - No details were reported. -t::teardown - No details were reported. - -2 tests: 0 passed, 0 failed, 0 skipped, 2 errors -`, - }, - { - name: "incomplete", - events: func() []testPipelineEvent { - return []testPipelineEvent{caseEvent(t, "t::ok", "PASSED", ""), caseEvent(t, "t::skip", "SKIPPED", "")} - }, - state: pipelines.UpdateInfoStateFailed, - want: ` -RUN INCOMPLETE: the update failed before all test results were received. -Results received: 1 passed, 1 skipped. -`, - }, - { - name: "incomplete with one error", - events: func() []testPipelineEvent { - return []testPipelineEvent{caseEvent(t, "t::setup", "ERROR", "")} - }, - state: pipelines.UpdateInfoStateFailed, - want: ` -FAILURES -t::setup - No details were reported. - -RUN INCOMPLETE: the update failed before all test results were received. -Results received: 1 error. -`, - }, - { - name: "incomplete without results", - events: func() []testPipelineEvent { return nil }, - state: pipelines.UpdateInfoStateCanceled, - want: ` -RUN INCOMPLETE: the update was canceled before all test results were received. -Results received: none. `, }, } diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index c37d696ba3c..ad4844213ad 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -144,17 +144,6 @@ func TestTestTargetResumeCommand(t *testing.T) { flags map[string]string want string }{ - { - name: "pipeline key", - target: testTarget{key: "my_pipeline", pipelineID: "p1"}, - want: "databricks pipelines test my_pipeline --resume u1", - }, - { - name: "pipeline key with target and profile", - target: testTarget{key: "my_pipeline", pipelineID: "p1"}, - flags: map[string]string{"target": "dev", "profile": "staging"}, - want: "databricks pipelines test my_pipeline --target dev --profile staging --resume u1", - }, { name: "names that need quoting", target: testTarget{key: "my_pipeline", pipelineID: "p1"}, @@ -185,44 +174,22 @@ func TestTestTargetResumeCommand(t *testing.T) { } } -func TestCheckTestUpdate(t *testing.T) { +// Where the service cannot report test_only, as before pipeline testing is enabled, the update +// may be a test update, so the command reads its results instead of rejecting it. +func TestCheckTestUpdateWithoutTestOnly(t *testing.T) { const updateID = "11111111-1111-1111-1111-111111111111" - tests := []struct { - name string - update string - wantErr string - }{ - // Where the service cannot report test_only, as before pipeline testing is enabled, the - // update may be a test update, so the command reads its results. - {name: "test_only not reported", update: `{"state": "COMPLETED"}`}, - {name: "test update", update: `{"state": "COMPLETED", "test_only": true}`}, - { - name: "regular update", - update: `{"state": "COMPLETED", "test_only": false}`, - wantErr: "update " + updateID + " is not a test update; pass the update ID that pipelines test printed", - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/api/2.0/pipelines/p1/updates/"+updateID { - _, _ = w.Write([]byte(`{"update": ` + tc.update + `}`)) - return - } - _, _ = w.Write([]byte(`{}`)) - })) - t.Cleanup(srv.Close) - w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: srv.URL, Token: "token"}) - require.NoError(t, err) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.0/pipelines/p1/updates/"+updateID { + _, _ = w.Write([]byte(`{"update": {"state": "COMPLETED"}}`)) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: srv.URL, Token: "token"}) + require.NoError(t, err) - err = checkTestUpdate(t.Context(), w, "p1", updateID) - if tc.wantErr == "" { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, tc.wantErr) - } - }) - } + assert.NoError(t, checkTestUpdate(t.Context(), w, "p1", updateID)) } func TestTestRunError(t *testing.T) { @@ -232,13 +199,6 @@ func TestTestRunError(t *testing.T) { state pipelines.UpdateInfoState wantErr string }{ - { - name: "passed", - events: func() []testPipelineEvent { - return []testPipelineEvent{caseEvent(t, "t::a", "PASSED", ""), summaryEvent(t, 1, 1, 0, 0, 0)} - }, - state: pipelines.UpdateInfoStateCompleted, - }, { name: "incomplete after completion", events: func() []testPipelineEvent { @@ -247,19 +207,6 @@ func TestTestRunError(t *testing.T) { state: pipelines.UpdateInfoStateCompleted, wantErr: root.ErrAlreadyPrinted.Error(), }, - { - name: "update failed with an error of its own and then the service's verdict", - events: func() []testPipelineEvent { - return []testPipelineEvent{ - caseEvent(t, "t::a", "FAILED", ""), - summaryEvent(t, 1, 0, 1, 0, 0), - decodeTestEvent(t, `{"id": "own", "error": {"fatal": true, "exceptions": [{"message": "Commit failed."}]}}`), - decodeTestEvent(t, `{"id": "verdict", "error": {"fatal": true, "exceptions": [{"class_name": "TestFailure", "message": "[PIPELINE_TEST_FAILED] 1 of the pipeline's tests failed."}]}}`), - } - }, - state: pipelines.UpdateInfoStateFailed, - wantErr: "update u1 is FAILED: Commit failed.", - }, { name: "update failed with the service's verdict although no test failed", events: func() []testPipelineEvent { From e129f043bcbafe0a01978990f36c2faff31900b6 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 2 Nov 2026 23:22:25 +0000 Subject: [PATCH 17/18] Read the pipeline in pipelines test --pipeline-id only when a result needs root_path With --no-wait and only workspace paths, nothing uses root_path, so a failed read should not stop the run. --- .../pipelines/test/pipeline-id/output.txt | 3 ++- acceptance/pipelines/test/pipeline-id/script | 5 +++- cmd/pipelines/test.go | 25 +++++++++++++++---- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt index 6c3611ec2f9..1d33c4efa8e 100644 --- a/acceptance/pipelines/test/pipeline-id/output.txt +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -33,11 +33,12 @@ tests/test_orders.py::test_valid_order PASSED (1.00s) >>> musterr [CLI] pipelines test --pipeline-id [UUID] tests Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests +=== With --no-wait and only workspace paths, the pipeline is not read >>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Users/someone@example.com/tests/test_other.py::test_x --no-wait Pipeline: [UUID] Update: [UUID] ->>> print_requests.py //updates +>>> print_requests.py --get //pipelines { "method": "POST", "path": "/api/2.0/pipelines/[UUID]/updates", diff --git a/acceptance/pipelines/test/pipeline-id/script b/acceptance/pipelines/test/pipeline-id/script index abb1a0ae3fe..3c18fd0e8fb 100644 --- a/acceptance/pipelines/test/pipeline-id/script +++ b/acceptance/pipelines/test/pipeline-id/script @@ -16,8 +16,11 @@ rm "$OUT_REQUESTS" title "Without a root_path, test selections must be workspace paths" trace musterr $CLI pipelines test --pipeline-id $no_root_pipeline_id tests +rm "$OUT_REQUESTS" + +title "With --no-wait and only workspace paths, the pipeline is not read" trace $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Users/someone@example.com/tests/test_other.py::test_x --no-wait -trace print_requests.py //updates +trace print_requests.py --get //pipelines $CLI api post /api/2.0/pipelines/$no_root_pipeline_id/stop &>> LOG.stop rm "$OUT_REQUESTS" diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index dda6361c4e9..1f1c4cc1b14 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -223,6 +223,16 @@ func withWorkspacePrefix(p string) string { return "/Workspace" + p } +// hasRelativeSelection reports whether a selection is a relative path. +func hasRelativeSelection(selections []string) bool { + for _, selection := range selections { + if p, _, _ := strings.Cut(selection, "::"); !strings.HasPrefix(p, "/") { + return true + } + } + return false +} + // rootPathSelection resolves a relative path against the pipeline's root_path. func rootPathSelection(rootPath, relPath string) (string, error) { if rootPath == "" { @@ -311,7 +321,7 @@ func (t testTarget) printHeader(out io.Writer, updateID string) { } // resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. -func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) (testTarget, error) { +func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string, wait bool) (testTarget, error) { if cmd.Flag("target").Value.String() == "" { cmd.SetContext(root.SkipLoadBundle(cmd.Context())) } @@ -321,9 +331,14 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - rootPath, err := getRootPath(ctx, w, pipelineID) - if err != nil { - return testTarget{}, err + // root_path resolves relative selections and shortens the test IDs printed while waiting. + var rootPath string + if wait || hasRelativeSelection(args) { + var err error + rootPath, err = getRootPath(ctx, w, pipelineID) + if err != nil { + return testTarget{}, err + } } selections, err := resolveTestSelections(args, func(p string) (string, error) { return rootPathSelection(rootPath, p) @@ -587,7 +602,7 @@ to read the results later. To stop the update, run 'pipelines stop'.`, var target testTarget var err error if pipelineID != "" { - target, err = resolveTestTargetByID(cmd, args, pipelineID) + target, err = resolveTestTargetByID(cmd, args, pipelineID, !noWait) } else { ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) From 025649f4fa576f37ffe17fa78c0cd9c546bbcaee Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 3 Nov 2026 00:20:43 +0000 Subject: [PATCH 18/18] Reach the submit in the pipelines test submit-error case, which now reads the pipeline first --- acceptance/pipelines/test/submit-error/output.txt | 11 +++++++++-- acceptance/pipelines/test/submit-error/script | 6 +++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt index 6ef5d8b4de4..7f0fcedf532 100644 --- a/acceptance/pipelines/test/submit-error/output.txt +++ b/acceptance/pipelines/test/submit-error/output.txt @@ -1,6 +1,13 @@ === A submit error from the service is shown as is, and no update ID is printed ->>> musterr [CLI] pipelines test --pipeline-id 1234 -Error: The specified pipeline 1234 was not found. +>>> musterr [CLI] pipelines test --pipeline-id [UUID] +Error: An active update '[UUID]' already exists for pipeline '1234'. >>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true + } +} diff --git a/acceptance/pipelines/test/submit-error/script b/acceptance/pipelines/test/submit-error/script index fe472e04fb5..29d720cc633 100644 --- a/acceptance/pipelines/test/submit-error/script +++ b/acceptance/pipelines/test/submit-error/script @@ -1,3 +1,7 @@ +# The pipeline exists, so that the command reaches the submit, which the test server rejects. +pipeline_id=$($CLI pipelines create --json '{"name": "orders"}' | jq -r .pipeline_id) +rm "$OUT_REQUESTS" + title "A submit error from the service is shown as is, and no update ID is printed" -trace musterr $CLI pipelines test --pipeline-id 1234 +trace musterr $CLI pipelines test --pipeline-id $pipeline_id trace print_requests.py //updates