diff --git a/acceptance/bin/find.py b/acceptance/bin/find.py index 61ad752fdc7..f8a4bb94dc6 100755 --- a/acceptance/bin/find.py +++ b/acceptance/bin/find.py @@ -4,6 +4,8 @@ Finds all files within current directory matching regex. The output is sorted and slashes are always forward. If --expect N is provided, the number of matches must be N or error is printed. +If --include-dirs is provided, directories are matched and printed too (default: files only). +If --prune REGEX is provided, directories whose path matches REGEX are not descended into or printed. """ import argparse @@ -14,15 +16,25 @@ parser = argparse.ArgumentParser() parser.add_argument("regex") parser.add_argument("--expect", type=int) +parser.add_argument("--include-dirs", action="store_true") +parser.add_argument("--prune") args = parser.parse_args() regex = re.compile(args.regex) +prune = re.compile(args.prune) if args.prune else None result = [] -for root, _dirs, files in os.walk("."): - for filename in files: - path = os.path.join(root, filename).replace("\\", "/") - path = path.removeprefix("./") + +def relpath(root, name): + return os.path.join(root, name).replace("\\", "/").removeprefix("./") + + +for root, dirs, files in os.walk("."): + if prune is not None: + dirs[:] = [d for d in dirs if not prune.search(relpath(root, d))] + names = files + dirs if args.include_dirs else files + for name in names: + path = relpath(root, name) if regex.search(path): result.append(path) diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt index 03d526bf6a5..11d51837441 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt +++ b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt @@ -74,7 +74,7 @@ Exit code: 1 === Assert destroy leaves the bundle state dir clean: ->>> find .databricks -name terraform -prune -o -print +>>> find.py --include-dirs --prune terraform$ ^\.databricks(/|$) .databricks .databricks/.gitignore .databricks/bundle diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/script b/acceptance/bundle/destroy/jobs-and-pipeline/script index 95b093e851e..c87568640da 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/script +++ b/acceptance/bundle/destroy/jobs-and-pipeline/script @@ -20,7 +20,7 @@ cleanup() { # which would make the golden fragile. The bundle scaffolding we care about lives # directly under the target dir, so it is still covered. title "Assert destroy leaves the bundle state dir clean:\n" - trace find .databricks -name terraform -prune -o -print | sort + trace find.py --include-dirs --prune 'terraform$' '^\.databricks(/|$)' title "Assert bundle deployment path is deleted" trace errcode $CLI workspace get-status "${DEPLOYMENT_PATH}" diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt index f6d296e2e6d..22db06cc872 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt @@ -21,7 +21,7 @@ Destroy: 1 deleted === and leaves the bundle state dir clean, with no empty scaffolding directories: ->>> find .databricks -name terraform -prune -o -print +>>> find.py --include-dirs --prune terraform$ ^\.databricks(/|$) .databricks .databricks/.gitignore .databricks/bundle diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script index f8139fab31e..987424f958f 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script @@ -28,7 +28,7 @@ trace find.py '^.databricks/bundle/default/(resources.json|terraform/terraform.t # already asserted gone above). The bundle scaffolding we care about lives directly # under the target dir, so it is still covered. title "and leaves the bundle state dir clean, with no empty scaffolding directories:\n" -trace find .databricks -name terraform -prune -o -print | sort +trace find.py --include-dirs --prune 'terraform$' '^\.databricks(/|$)' # Wipe whatever remains so the next deploy behaves like a fresh machine. rm -rf .databricks diff --git a/acceptance/bundle/generate/job_nested_notebooks/script b/acceptance/bundle/generate/job_nested_notebooks/script index a52154efbb0..e7ce4f18126 100644 --- a/acceptance/bundle/generate/job_nested_notebooks/script +++ b/acceptance/bundle/generate/job_nested_notebooks/script @@ -15,4 +15,4 @@ test ! -f src/my_notebook.py && echo "src/my_notebook.py removed" || echo "src/m test ! -f src/other_notebook.py && echo "src/other_notebook.py removed" || echo "src/other_notebook.py still exists" echo "=== new nested files ===" -find src -type f | sort +find.py '^src/' diff --git a/acceptance/bundle/migrate/auto-migrate-clean/output.txt b/acceptance/bundle/migrate/auto-migrate-clean/output.txt index a358b635116..e4b502e2ae8 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-clean/output.txt @@ -27,10 +27,10 @@ direct_migrated_via_config true === Direct-engine state is now in place; terraform state is backed up ->>> find .databricks/bundle -name resources.json -type f +>>> find.py ^\.databricks/bundle/.*resources\.json$ .databricks/bundle/default/resources.json ->>> find .databricks/bundle -name terraform.tfstate* -type f +>>> find.py ^\.databricks/bundle/.*terraform\.tfstate .databricks/bundle/default/terraform/terraform.tfstate.backup === Remote state also reflects the migration (force-pull surfaces workspace state) diff --git a/acceptance/bundle/migrate/auto-migrate-clean/script b/acceptance/bundle/migrate/auto-migrate-clean/script index eb0b970ddc3..ab14960ed89 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/script +++ b/acceptance/bundle/migrate/auto-migrate-clean/script @@ -14,8 +14,8 @@ trace print_migration_telemetry rm -f "$OUT_REQUESTS" title "Direct-engine state is now in place; terraform state is backed up\n" -trace find .databricks/bundle -name "resources.json" -type f -trace find .databricks/bundle -name "terraform.tfstate*" -type f +trace find.py '^\.databricks/bundle/.*resources\.json$' +trace find.py '^\.databricks/bundle/.*terraform\.tfstate' title "Remote state also reflects the migration (force-pull surfaces workspace state)\n" trace $CLI bundle debug states --force-pull diff --git a/acceptance/bundle/migrate/auto-migrate-default/output.txt b/acceptance/bundle/migrate/auto-migrate-default/output.txt index 33950c186e3..4551a84f29c 100644 --- a/acceptance/bundle/migrate/auto-migrate-default/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-default/output.txt @@ -29,10 +29,10 @@ direct_migrated_via_default true === Direct-engine state is now in place; terraform state is backed up ->>> find .databricks/bundle -name resources.json -type f +>>> find.py ^\.databricks/bundle/.*resources\.json$ .databricks/bundle/default/resources.json ->>> find .databricks/bundle -name terraform.tfstate* -type f +>>> find.py ^\.databricks/bundle/.*terraform\.tfstate .databricks/bundle/default/terraform/terraform.tfstate.backup === Remote state also reflects the migration (force-pull surfaces workspace state) diff --git a/acceptance/bundle/migrate/auto-migrate-default/script b/acceptance/bundle/migrate/auto-migrate-default/script index eb138273aff..fbe71da2776 100644 --- a/acceptance/bundle/migrate/auto-migrate-default/script +++ b/acceptance/bundle/migrate/auto-migrate-default/script @@ -11,8 +11,8 @@ trace print_migration_telemetry rm -f "$OUT_REQUESTS" title "Direct-engine state is now in place; terraform state is backed up\n" -trace find .databricks/bundle -name "resources.json" -type f -trace find .databricks/bundle -name "terraform.tfstate*" -type f +trace find.py '^\.databricks/bundle/.*resources\.json$' +trace find.py '^\.databricks/bundle/.*terraform\.tfstate' title "Remote state also reflects the migration (force-pull surfaces workspace state)\n" trace $CLI bundle debug states --force-pull diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt index 06e544ad5fc..7c1899e50cf 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt @@ -9,9 +9,9 @@ Removing empty terraform state; direct engine will be used on the next deploy (s === Terraform state is renamed to .backup; no resources.json (empty state, nothing to persist) ->>> find .databricks/bundle -name resources.json -type f +>>> find.py ^\.databricks/bundle/.*resources\.json$ ->>> find .databricks/bundle -name terraform.tfstate* -type f +>>> find.py ^\.databricks/bundle/.*terraform\.tfstate .databricks/bundle/default/terraform/terraform.tfstate.backup === Sweep is recorded in telemetry as via-config diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script index e72b1cadab4..8481134b8b0 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script @@ -12,8 +12,8 @@ title "Deploy with bundle.engine: direct and an empty terraform state — sweep, trace $CLI bundle deploy 2>&1 | contains.py 'Removing empty terraform state' '!Migrating state to direct' title "Terraform state is renamed to .backup; no resources.json (empty state, nothing to persist)\n" -trace find .databricks/bundle -name "resources.json" -type f -trace find .databricks/bundle -name "terraform.tfstate*" -type f +trace find.py '^\.databricks/bundle/.*resources\.json$' +trace find.py '^\.databricks/bundle/.*terraform\.tfstate' title "Sweep is recorded in telemetry as via-config\n" trace print_migration_telemetry diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/output.txt b/acceptance/bundle/migrate/auto-migrate-envvar/output.txt index a0f04340420..ea7d0c85060 100644 --- a/acceptance/bundle/migrate/auto-migrate-envvar/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-envvar/output.txt @@ -24,10 +24,10 @@ direct_migrated_via_env true === Direct-engine state is now in place; terraform state is backed up ->>> find .databricks/bundle -name resources.json -type f +>>> find.py ^\.databricks/bundle/.*resources\.json$ .databricks/bundle/default/resources.json ->>> find .databricks/bundle -name terraform.tfstate* -type f +>>> find.py ^\.databricks/bundle/.*terraform\.tfstate .databricks/bundle/default/terraform/terraform.tfstate.backup === Remote state also reflects the migration (force-pull surfaces workspace state) diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/script b/acceptance/bundle/migrate/auto-migrate-envvar/script index 8377687a9b6..abfd740a09d 100644 --- a/acceptance/bundle/migrate/auto-migrate-envvar/script +++ b/acceptance/bundle/migrate/auto-migrate-envvar/script @@ -11,8 +11,8 @@ trace print_migration_telemetry rm -f "$OUT_REQUESTS" title "Direct-engine state is now in place; terraform state is backed up\n" -trace find .databricks/bundle -name "resources.json" -type f -trace find .databricks/bundle -name "terraform.tfstate*" -type f +trace find.py '^\.databricks/bundle/.*resources\.json$' +trace find.py '^\.databricks/bundle/.*terraform\.tfstate' title "Remote state also reflects the migration (force-pull surfaces workspace state)\n" trace $CLI bundle debug states --force-pull diff --git a/acceptance/bundle/migrate/auto-migrate-plan-failure/output.txt b/acceptance/bundle/migrate/auto-migrate-plan-failure/output.txt index 410ec2b580e..270938f3619 100644 --- a/acceptance/bundle/migrate/auto-migrate-plan-failure/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-plan-failure/output.txt @@ -30,9 +30,9 @@ Warn: Direct engine was selected but the migration reported issues; automatic mi direct_migrate_plan_error true === Local state was NOT rewritten (still terraform) ->>> find .databricks/bundle -name resources.json -type f +>>> find.py ^\.databricks/bundle/.*resources\.json$ ->>> find .databricks/bundle -name terraform.tfstate* -type f +>>> find.py ^\.databricks/bundle/.*terraform\.tfstate .databricks/bundle/default/terraform/terraform.tfstate === Retry: plan check passes this time, migration succeeds @@ -48,8 +48,8 @@ Migrated 1 resource to direct deployment engine. direct_migrated_via_env true === Local state is now direct ->>> find .databricks/bundle -name resources.json -type f +>>> find.py ^\.databricks/bundle/.*resources\.json$ .databricks/bundle/default/resources.json ->>> find .databricks/bundle -name terraform.tfstate* -type f +>>> find.py ^\.databricks/bundle/.*terraform\.tfstate .databricks/bundle/default/terraform/terraform.tfstate.backup diff --git a/acceptance/bundle/migrate/auto-migrate-plan-failure/script b/acceptance/bundle/migrate/auto-migrate-plan-failure/script index 0e2e2ead7ed..eaa0f304365 100644 --- a/acceptance/bundle/migrate/auto-migrate-plan-failure/script +++ b/acceptance/bundle/migrate/auto-migrate-plan-failure/script @@ -17,8 +17,8 @@ trace print_migration_telemetry rm -f "$OUT_REQUESTS" title "Local state was NOT rewritten (still terraform)" -trace find .databricks/bundle -name "resources.json" -type f -trace find .databricks/bundle -name "terraform.tfstate*" -type f +trace find.py '^\.databricks/bundle/.*resources\.json$' +trace find.py '^\.databricks/bundle/.*terraform\.tfstate' title "Retry: plan check passes this time, migration succeeds" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy @@ -26,7 +26,7 @@ trace print_migration_telemetry rm -f "$OUT_REQUESTS" title "Local state is now direct" -trace find .databricks/bundle -name "resources.json" -type f -trace find .databricks/bundle -name "terraform.tfstate*" -type f +trace find.py '^\.databricks/bundle/.*resources\.json$' +trace find.py '^\.databricks/bundle/.*terraform\.tfstate' rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/output.txt b/acceptance/bundle/migrate/auto-migrate-push-failure/output.txt index cf125b4bd74..3f2abc7d1ca 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-push-failure/output.txt @@ -24,9 +24,9 @@ direct_migrate_commit_error true === Local state was NOT rewritten (still terraform) ->>> find .databricks/bundle -name resources.json -type f +>>> find.py ^\.databricks/bundle/.*resources\.json$ ->>> find .databricks/bundle -name terraform.tfstate* -type f +>>> find.py ^\.databricks/bundle/.*terraform\.tfstate .databricks/bundle/default/terraform/terraform.tfstate === Retry: auto-migration succeeds this time @@ -43,8 +43,8 @@ direct_migrated_via_env true === Local state is now direct ->>> find .databricks/bundle -name resources.json -type f +>>> find.py ^\.databricks/bundle/.*resources\.json$ .databricks/bundle/default/resources.json ->>> find .databricks/bundle -name terraform.tfstate* -type f +>>> find.py ^\.databricks/bundle/.*terraform\.tfstate .databricks/bundle/default/terraform/terraform.tfstate.backup diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/script b/acceptance/bundle/migrate/auto-migrate-push-failure/script index 283398867c7..f8dea535246 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/script +++ b/acceptance/bundle/migrate/auto-migrate-push-failure/script @@ -15,8 +15,8 @@ trace print_migration_telemetry rm -f "$OUT_REQUESTS" title "Local state was NOT rewritten (still terraform)\n" -trace find .databricks/bundle -name "resources.json" -type f -trace find .databricks/bundle -name "terraform.tfstate*" -type f +trace find.py '^\.databricks/bundle/.*resources\.json$' +trace find.py '^\.databricks/bundle/.*terraform\.tfstate' title "Retry: auto-migration succeeds this time" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy @@ -24,7 +24,7 @@ trace print_migration_telemetry rm -f "$OUT_REQUESTS" title "Local state is now direct\n" -trace find .databricks/bundle -name "resources.json" -type f -trace find .databricks/bundle -name "terraform.tfstate*" -type f +trace find.py '^\.databricks/bundle/.*resources\.json$' +trace find.py '^\.databricks/bundle/.*terraform\.tfstate' rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/output.txt b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/output.txt index ac27fccbf49..a1023b622cd 100644 --- a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/output.txt @@ -24,9 +24,9 @@ direct_migrate_commit_error true === Local state was NOT rewritten (still terraform) ->>> find .databricks/bundle -name resources.json -type f +>>> find.py ^\.databricks/bundle/.*resources\.json$ ->>> find .databricks/bundle -name terraform.tfstate* -type f +>>> find.py ^\.databricks/bundle/.*terraform\.tfstate .databricks/bundle/default/terraform/terraform.tfstate === Retry recovers: subsequent deploy re-does the backup and completes @@ -39,8 +39,8 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === Local state is now direct ->>> find .databricks/bundle -name resources.json -type f +>>> find.py ^\.databricks/bundle/.*resources\.json$ .databricks/bundle/default/resources.json ->>> find .databricks/bundle -name terraform.tfstate* -type f +>>> find.py ^\.databricks/bundle/.*terraform\.tfstate .databricks/bundle/default/terraform/terraform.tfstate diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/script b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/script index 426ee04a73b..cc1864e92ba 100644 --- a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/script +++ b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/script @@ -17,8 +17,8 @@ trace print_migration_telemetry rm -f "$OUT_REQUESTS" title "Local state was NOT rewritten (still terraform)\n" -trace find .databricks/bundle -name "resources.json" -type f -trace find .databricks/bundle -name "terraform.tfstate*" -type f +trace find.py '^\.databricks/bundle/.*resources\.json$' +trace find.py '^\.databricks/bundle/.*terraform\.tfstate' title "Retry recovers: subsequent deploy re-does the backup and completes" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy @@ -26,7 +26,7 @@ trace print_migration_telemetry rm -f "$OUT_REQUESTS" title "Local state is now direct\n" -trace find .databricks/bundle -name "resources.json" -type f -trace find .databricks/bundle -name "terraform.tfstate*" -type f +trace find.py '^\.databricks/bundle/.*resources\.json$' +trace find.py '^\.databricks/bundle/.*terraform\.tfstate' rm -f "$OUT_REQUESTS"