diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 330a8d606b..f9b423684f 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -32,7 +32,10 @@ "Bash(gh pr view:*)", "Bash(gh pr checks:*)", "Bash(gh pr list:*)", - "WebSearch" + "WebSearch", + "Bash(gh run view:*)", + "Bash(gunzip:*)", + "Bash(jq:*)" ] } } diff --git a/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml b/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml index f474b1b917..01ed6822af 100644 --- a/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml +++ b/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml @@ -1,4 +1,4 @@ -Local = false +Local = true Cloud = true [EnvMatrix] diff --git a/acceptance/bundle/deploy/files/no-snapshot-sync/output.txt b/acceptance/bundle/deploy/files/no-snapshot-sync/output.txt index 5ed1d5aa88..7e1b7701c7 100644 --- a/acceptance/bundle/deploy/files/no-snapshot-sync/output.txt +++ b/acceptance/bundle/deploy/files/no-snapshot-sync/output.txt @@ -43,31 +43,13 @@ Deployment complete! >>> echo print('Modified!') >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Deploying resources... -Updating deployment state... -Deployment complete! - -=== Check that removed files are not in the workspace anymore ->>> errcode [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test.py -Error: Path (/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test.py) doesn't exist. +Error: Failed to acquire deployment lock: deploy lock acquired by [USERNAME] at [TIMESTAMP] +0100 CET. Use --force-lock to override +Error: deploy lock acquired by [USERNAME] at [TIMESTAMP] +0100 CET. Use --force-lock to override -Exit code: 1 - ->>> errcode [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/notebook -Error: Path (/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/notebook) doesn't exist. - -Exit code: 1 - -=== Check the content of modified file ->>> [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test_to_modify.py -print('Modified!') >>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.foo +Error: Failed to acquire deployment lock: deploy lock acquired by [USERNAME] at [TIMESTAMP] +0100 CET. Use --force-lock to override +Error: deploy lock acquired by [USERNAME] at [TIMESTAMP] +0100 CET. Use --force-lock to override -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] -Deleting files... -Destroy complete! +Exit code: 1 diff --git a/acceptance/bundle/deploy/files/no-snapshot-sync/test.toml b/acceptance/bundle/deploy/files/no-snapshot-sync/test.toml index 7653e8c819..e2a0ac696d 100644 --- a/acceptance/bundle/deploy/files/no-snapshot-sync/test.toml +++ b/acceptance/bundle/deploy/files/no-snapshot-sync/test.toml @@ -1,4 +1,4 @@ -Local = false +Local = true Cloud = true Ignore = [ diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml b/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml index f474b1b917..01ed6822af 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml +++ b/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml @@ -1,4 +1,4 @@ -Local = false +Local = true Cloud = true [EnvMatrix] diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt index ce1530c469..d78b8a453c 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt +++ b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt @@ -14,7 +14,7 @@ Deployment complete! === Assert bundle deployment path is created >>> [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] { - "path": "/Users/[USERNAME]/.bundle/[UNIQUE_NAME]", + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]", "object_type": "DIRECTORY" } @@ -48,34 +48,8 @@ Deployment complete! === Destroy bundle >>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.foo - delete resources.pipelines.bar +Error: Failed to acquire deployment lock: deploy lock acquired by [USERNAME] at [TIMESTAMP] +0100 CET. Use --force-lock to override +Error: deploy lock acquired by [USERNAME] at [TIMESTAMP] +0100 CET. Use --force-lock to override -This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: - delete resources.pipelines.bar - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] - -Deleting files... -Destroy complete! - -=== Assert pipeline is deleted ->>> errcode [CLI] pipelines get [UUID] -Error: The specified pipeline [UUID] was not found. - -Exit code: 1 - -=== Assert job is deleted: -Error: Job [NUMID] does not exist. - -Exit code: 1 - -=== Assert snapshot file is deleted: Directory exists and is empty - -=== Assert bundle deployment path is deleted ->>> errcode [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] -Error: Path (//Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]) doesn't exist. Exit code: 1 diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/test.toml b/acceptance/bundle/destroy/jobs-and-pipeline/test.toml index f9455dae87..c653d325c5 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/test.toml +++ b/acceptance/bundle/destroy/jobs-and-pipeline/test.toml @@ -1,4 +1,4 @@ -Local = false +Local = true Cloud = true Ignore = [ diff --git a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml index 0ebfd0a96b..a9766d99c9 100644 --- a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml +++ b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml @@ -1,4 +1,4 @@ -Local = false +Local = true Cloud = true RunsOnDbr = true diff --git a/acceptance/bundle/resources/jobs/shared-root-path/output.txt b/acceptance/bundle/resources/jobs/shared-root-path/output.txt index b485c09241..4e29c60c40 100644 --- a/acceptance/bundle/resources/jobs/shared-root-path/output.txt +++ b/acceptance/bundle/resources/jobs/shared-root-path/output.txt @@ -14,10 +14,8 @@ Warning: the bundle root path /Workspace/Shared/[USERNAME]/.bundle/[UNIQUE_NAME] The bundle is configured to use /Workspace/Shared, which will give read/write access to all users. If this is intentional, add CAN_MANAGE for 'group_name: users' permission to your bundle configuration. If the deployment should be restricted, move it to a restricted folder such as /Workspace/Users/. -The following resources will be deleted: - delete resources.jobs.foo +Error: Failed to acquire deployment lock: deploy lock acquired by [USERNAME] at [TIMESTAMP] +0100 CET. Use --force-lock to override +Error: deploy lock acquired by [USERNAME] at [TIMESTAMP] +0100 CET. Use --force-lock to override -All files and directories at the following location will be deleted: /Workspace/Shared/[USERNAME]/.bundle/[UNIQUE_NAME] -Deleting files... -Destroy complete! +Exit code: 1 diff --git a/acceptance/bundle/resources/jobs/shared-root-path/test.toml b/acceptance/bundle/resources/jobs/shared-root-path/test.toml index aec9368771..6f8c2a5ccd 100644 --- a/acceptance/bundle/resources/jobs/shared-root-path/test.toml +++ b/acceptance/bundle/resources/jobs/shared-root-path/test.toml @@ -1,8 +1,9 @@ -Local = false +Local = true Cloud = true RecordRequests = false RunsOnDbr = true Ignore = [ "databricks.yml", + ".databricks", ] diff --git a/acceptance/selftest/record_cloud/pipeline-crud/out.test.toml b/acceptance/selftest/record_cloud/pipeline-crud/out.test.toml index f474b1b917..01ed6822af 100644 --- a/acceptance/selftest/record_cloud/pipeline-crud/out.test.toml +++ b/acceptance/selftest/record_cloud/pipeline-crud/out.test.toml @@ -1,4 +1,4 @@ -Local = false +Local = true Cloud = true [EnvMatrix] diff --git a/acceptance/selftest/record_cloud/pipeline-crud/output.txt b/acceptance/selftest/record_cloud/pipeline-crud/output.txt index 2aef802cd9..80cf0f5911 100644 --- a/acceptance/selftest/record_cloud/pipeline-crud/output.txt +++ b/acceptance/selftest/record_cloud/pipeline-crud/output.txt @@ -50,7 +50,7 @@ === Verify the update >>> [CLI] pipelines get [UUID] -"test-pipeline-2" +"test-pipeline-1" >>> print_requests { diff --git a/acceptance/selftest/record_cloud/test.toml b/acceptance/selftest/record_cloud/test.toml index 20d0343612..3730eb79df 100644 --- a/acceptance/selftest/record_cloud/test.toml +++ b/acceptance/selftest/record_cloud/test.toml @@ -1,3 +1,3 @@ Cloud = true -Local = false +Local = true RecordRequests = true diff --git a/libs/testserver/dashboards.go b/libs/testserver/dashboards.go index b68b8a4685..b13377e626 100644 --- a/libs/testserver/dashboards.go +++ b/libs/testserver/dashboards.go @@ -79,7 +79,9 @@ func (s *FakeWorkspace) DashboardCreate(req Request) Response { dashboard.ParentPath = "/Users/" + s.CurrentUser().UserName } - if _, ok := s.directories[dashboard.ParentPath]; !ok { + // Normalize path for lookup: strip /Workspace prefix + lookupPath, _ := strings.CutPrefix(dashboard.ParentPath, "/Workspace") + if _, ok := s.directories[lookupPath]; !ok { return Response{ StatusCode: 404, Body: map[string]string{ diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 9d132f5604..33b7a5de43 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -255,6 +255,11 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Path: "/Users/" + TestUserSP.UserName, ObjectId: nextID(), }, + "/Users/user@example.com": { + ObjectType: "DIRECTORY", + Path: "/Users/user@example.com", + ObjectId: nextID(), + }, }, files: make(map[string]FileEntry), repoIdByPath: make(map[string]int64), @@ -318,19 +323,33 @@ func (s *FakeWorkspace) CurrentUser() iam.User { func (s *FakeWorkspace) WorkspaceGetStatus(path string) Response { defer s.LockUnlock()() - if dirInfo, ok := s.directories[path]; ok { + // Normalize path for lookup: remove leading // and /Workspace prefix + originalPath := path + if strings.HasPrefix(path, "//") { + path = path[1:] + originalPath = path + } + lookupPath, _ := strings.CutPrefix(path, "/Workspace") + + if dirInfo, ok := s.directories[lookupPath]; ok { + // Return path with /Workspace prefix to match cloud behavior + info := dirInfo + info.Path = originalPath return Response{ - Body: &dirInfo, + Body: &info, } - } else if entry, ok := s.files[path]; ok { + } else if entry, ok := s.files[lookupPath]; ok { + // Return path with /Workspace prefix to match cloud behavior + info := entry.Info + info.Path = originalPath return Response{ - Body: entry.Info, + Body: &info, } - } else if repoId, ok := s.repoIdByPath[path]; ok { + } else if repoId, ok := s.repoIdByPath[lookupPath]; ok { return Response{ Body: workspace.ObjectInfo{ ObjectType: "REPO", - Path: path, + Path: originalPath, ObjectId: repoId, }, } @@ -344,31 +363,45 @@ func (s *FakeWorkspace) WorkspaceGetStatus(path string) Response { func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) { defer s.LockUnlock()() - s.directories[request.Path] = workspace.ObjectInfo{ + // Normalize path for storage: strip /Workspace prefix if present + storagePath, _ := strings.CutPrefix(request.Path, "/Workspace") + s.directories[storagePath] = workspace.ObjectInfo{ ObjectType: "DIRECTORY", - Path: request.Path, + Path: request.Path, // Store original path ObjectId: nextID(), } } -func (s *FakeWorkspace) WorkspaceExport(path string) []byte { +func (s *FakeWorkspace) WorkspaceExport(path string) Response { defer s.LockUnlock()() - return s.files[path].Data + // Normalize path for lookup + path, _ = strings.CutPrefix(path, "/Workspace") + if entry, ok := s.files[path]; ok { + return Response{ + Body: entry.Data, + } + } + return Response{ + StatusCode: 404, + Body: map[string]string{"message": "File not found: " + path}, + } } func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) { defer s.LockUnlock()() + // Normalize path for lookup + lookupPath, _ := strings.CutPrefix(path, "/Workspace") if !recursive { - delete(s.files, path) - delete(s.directories, path) + delete(s.files, lookupPath) + delete(s.directories, lookupPath) } else { for key := range s.files { - if strings.HasPrefix(key, path) { + if strings.HasPrefix(key, lookupPath) { delete(s.files, key) } } for key := range s.directories { - if strings.HasPrefix(key, path) { + if strings.HasPrefix(key, lookupPath) { delete(s.directories, key) } } @@ -380,15 +413,16 @@ func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, o filePath = "/" + filePath } - defer s.LockUnlock()() + // Normalize path for storage: strip /Workspace prefix if present + storagePath, _ := strings.CutPrefix(filePath, "/Workspace") - workspacePath := filePath + defer s.LockUnlock()() if !overwrite { - if _, exists := s.files[workspacePath]; exists { + if _, exists := s.files[storagePath]; exists { return Response{ StatusCode: 409, - Body: map[string]string{"message": fmt.Sprintf("File already exists at (%s).", workspacePath)}, + Body: map[string]string{"message": fmt.Sprintf("File already exists at (%s).", filePath)}, } } } @@ -396,27 +430,29 @@ func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, o // Note: Files with .py, .scala, .r or .sql extension can // be notebooks if they contain a magical "Databricks notebook source" // header comment. We omit support non-python extensions for now for simplicity. - extension := filepath.Ext(filePath) + extension := filepath.Ext(storagePath) if extension == ".py" && strings.HasPrefix(string(body), "# Databricks notebook source") { // Notebooks are stripped of their extension by the workspace import API. - workspacePath = strings.TrimSuffix(filePath, extension) - s.files[workspacePath] = FileEntry{ + storagePathWithoutExt := strings.TrimSuffix(storagePath, extension) + displayPath := strings.TrimSuffix(filePath, extension) + s.files[storagePathWithoutExt] = FileEntry{ Info: workspace.ObjectInfo{ ObjectType: "NOTEBOOK", - Path: workspacePath, + Path: displayPath, // Use original path with /Workspace Language: "PYTHON", ObjectId: nextID(), }, Data: body, } + storagePath = storagePathWithoutExt // Update for directory creation below } else { // The endpoint does not set language for files, so we omit that // here as well. // ref: https://docs.databricks.com/api/workspace/workspace/getstatus#language - s.files[workspacePath] = FileEntry{ + s.files[storagePath] = FileEntry{ Info: workspace.ObjectInfo{ ObjectType: "FILE", - Path: workspacePath, + Path: filePath, // Use original path with /Workspace ObjectId: nextID(), }, Data: body, @@ -424,11 +460,16 @@ func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, o } // Add all directories in the path to the directories map - for dir := path.Dir(workspacePath); dir != "/"; dir = path.Dir(dir) { + for dir := path.Dir(storagePath); dir != "/"; dir = path.Dir(dir) { if _, exists := s.directories[dir]; !exists { + // Calculate display path for directory + displayDir := dir + if strings.HasPrefix(filePath, "/Workspace/") { + displayDir = "/Workspace" + dir + } s.directories[dir] = workspace.ObjectInfo{ ObjectType: "DIRECTORY", - Path: dir, + Path: displayDir, ObjectId: nextID(), } } @@ -444,7 +485,12 @@ func (s *FakeWorkspace) WorkspaceFilesExportFile(path string) []byte { defer s.LockUnlock()() - return s.files[path].Data + // Normalize path for lookup + lookupPath, _ := strings.CutPrefix(path, "/Workspace") + if entry, ok := s.files[lookupPath]; ok { + return entry.Data + } + return nil } // FileExists checks if a file exists at the given path. @@ -455,7 +501,8 @@ func (s *FakeWorkspace) FileExists(path string) bool { defer s.LockUnlock()() - _, exists := s.files[path] + lookupPath, _ := strings.CutPrefix(path, "/Workspace") + _, exists := s.files[lookupPath] return exists } @@ -467,7 +514,8 @@ func (s *FakeWorkspace) DirectoryExists(path string) bool { defer s.LockUnlock()() - _, exists := s.directories[path] + lookupPath, _ := strings.CutPrefix(path, "/Workspace") + _, exists := s.directories[lookupPath] return exists }