[Draft] Support bundle-wide group run identities for jobs

.nextchanges/bundles/run-as-group.md

@@ -0,0 +1 @@
+* Add support for `run_as.group_name` at the bundle and target levels for jobs using the direct deployment engine. ([#6676](https://github.com/databricks/cli/pull/6676))

acceptance/bundle/run_as/empty_run_as_dict/output.txt

@@ -1,4 +1,4 @@
-Error: run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified
+Error: run_as section must specify exactly one non-empty identity: user_name, service_principal_name, or group_name
   in databricks.yml:4:9
 
   "run_as": {},

acceptance/bundle/run_as/empty_sp/output.txt

@@ -1,4 +1,4 @@
-Error: run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified
+Error: run_as section must specify exactly one non-empty identity: user_name, service_principal_name, or group_name
   in databricks.yml:5:3
 
 {

acceptance/bundle/run_as/empty_user/output.txt

@@ -1,4 +1,4 @@
-Error: run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified
+Error: run_as section must specify exactly one non-empty identity: user_name, service_principal_name, or group_name
   in databricks.yml:5:3
 
 {

acceptance/bundle/run_as/empty_user_and_sp/output.txt

@@ -1,4 +1,4 @@
-Error: run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified
+Error: run_as section must specify exactly one non-empty identity: user_name, service_principal_name, or group_name
   in databricks.yml:5:3
 
 {

acceptance/bundle/run_as/group/databricks.yml

@@ -0,0 +1,21 @@
+bundle:
+  name: run_as_group
+
+run_as:
+  group_name: bundle_group
+
+resources:
+  jobs:
+    inherited:
+      name: Inherited identity
+    user_override:
+      name: User override
+      run_as:
+        user_name: user_override
+
+targets:
+  default:
+    default: true
+  group:
+    run_as:
+      group_name: target_group

acceptance/bundle/run_as/group/output.txt

@@ -0,0 +1,17 @@
+
+>>> [CLI] bundle validate -o json
+json.resources.jobs.inherited.run_as.group_name = "bundle_group";
+json.resources.jobs.user_override.run_as.user_name = "user_override";
+json.run_as.group_name = "bundle_group";
+
+>>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle plan
+Error: run_as.group_name is only supported in direct deployment mode
+  at resources.jobs.inherited.run_as.group_name
+
+Set 'bundle.engine: direct' in your databricks.yml or set DATABRICKS_BUNDLE_ENGINE=direct. For an existing Terraform deployment, run 'databricks bundle migrate' first. Alternatively, set run_as.user_name or run_as.service_principal_name on this job
+
+
+>>> [CLI] bundle validate -t group -o json
+json.resources.jobs.inherited.run_as.group_name = "target_group";
+json.resources.jobs.user_override.run_as.user_name = "user_override";
+json.run_as.group_name = "target_group";

acceptance/bundle/run_as/group/script

@@ -0,0 +1,5 @@
+trace $CLI bundle validate -o json | gron.py | grep -F .run_as
+trace DATABRICKS_BUNDLE_ENGINE=terraform musterr $CLI bundle plan
+
+update_file.py databricks.yml "group_name: bundle_group" "user_name: bundle_user"
+trace $CLI bundle validate -t group -o json | gron.py | grep -F .run_as

acceptance/bundle/run_as/group/test.toml

@@ -0,0 +1,2 @@
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+Ignore = [".databricks"]

acceptance/bundle/run_as/invalid_both_sp_and_user/output.txt

@@ -1,4 +1,4 @@
-Error: run_as section cannot specify both user_name and service_principal_name
+Error: run_as section must specify exactly one non-empty identity: user_name, service_principal_name, or group_name
   in databricks.yml:6:3
 
 Name: run_as

bundle/config/mutator/resourcemutator/run_as.go

@@ -3,6 +3,7 @@ package resourcemutator
 import (
 	"context"
 	"fmt"
+	"maps"
 	"slices"
 
 	"github.com/databricks/cli/bundle"
@@ -21,7 +22,7 @@ type setRunAs struct{}
 //  1. Sets the run_as field for jobs to the value of the run_as field in the bundle.
 //
 //  2. Validates that the bundle run_as configuration is valid in the context of the bundle.
-//     If the run_as user is different from the current deployment user, DABs only
+//     If the run_as identity differs from the deployment identity, DABs only
 //     supports a subset of resources.
 func SetRunAs() bundle.Mutator {
 	return &setRunAs{}
@@ -44,31 +45,21 @@ func reportRunAsNotSupported(resourceType string, location dyn.Location, current
 func validateRunAs(b *bundle.Bundle) diag.Diagnostics {
 	diags := diag.Diagnostics{}
 
-	neitherSpecifiedErr := diag.Diagnostics{{
-		Summary:   "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified",
-		Locations: []dyn.Location{b.Config.GetLocation("run_as")},
-		Severity:  diag.Error,
-	}}
-
-	// Fail fast if neither service_principal_name nor user_name are specified, but the
-	// run_as section is present.
-	if b.Config.Value().Get("run_as").Kind() == dyn.KindNil {
-		return neitherSpecifiedErr
-	}
-
-	// Fail fast if one or both of service_principal_name and user_name are specified,
-	// but with empty values.
+	identityCount := 0
 	runAs := b.Config.RunAs
-	if runAs.ServicePrincipalName == "" && runAs.UserName == "" {
-		return neitherSpecifiedErr
+	if runAs != nil {
+		for _, identity := range []string{runAs.UserName, runAs.ServicePrincipalName, runAs.GroupName} {
+			if identity != "" {
+				identityCount++
+			}
+		}
 	}
-
-	if runAs.UserName != "" && runAs.ServicePrincipalName != "" {
-		diags = diags.Extend(diag.Diagnostics{{
-			Summary:   "run_as section cannot specify both user_name and service_principal_name",
+	if identityCount != 1 {
+		return diag.Diagnostics{{
+			Summary:   "run_as section must specify exactly one non-empty identity: user_name, service_principal_name, or group_name",
 			Locations: []dyn.Location{b.Config.GetLocation("run_as")},
 			Severity:  diag.Error,
-		}})
+		}}
 	}
 
 	identity := runAs.ServicePrincipalName
@@ -77,10 +68,32 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics {
 	}
 
 	// All resources are supported if the run_as identity is the same as the current deployment identity.
-	if identity == b.Config.Workspace.CurrentUser.UserName {
+	if runAs.GroupName == "" && identity == b.Config.Workspace.CurrentUser.UserName {
 		return diags
 	}
 
+	if runAs.GroupName != "" {
+		identity = fmt.Sprintf("group %q", runAs.GroupName)
+		for _, key := range slices.Sorted(maps.Keys(b.Config.Resources.Pipelines)) {
+			if b.Config.Resources.Pipelines[key].RunAs == nil {
+				diags = diags.Extend(diag.Diagnostics{{
+					Summary:   "pipelines do not support run_as.group_name; set run_as.user_name or run_as.service_principal_name on this pipeline to override the bundle run_as",
+					Locations: []dyn.Location{b.Config.GetLocation("resources.pipelines." + key)},
+					Severity:  diag.Error,
+				}})
+			}
+		}
+		for _, key := range slices.Sorted(maps.Keys(b.Config.Resources.Alerts)) {
+			if b.Config.Resources.Alerts[key].RunAs == nil {
+				diags = diags.Extend(diag.Diagnostics{{
+					Summary:   "alerts do not support run_as.group_name; set run_as.user_name or run_as.service_principal_name on this alert to override the bundle run_as",
+					Locations: []dyn.Location{b.Config.GetLocation("resources.alerts." + key)},
+					Severity:  diag.Error,
+				}})
+			}
+		}
+	}
+
 	// Model serving endpoints do not support run_as in the API.
 	if len(b.Config.Resources.ModelServingEndpoints) > 0 {
 		diags = diags.Extend(reportRunAsNotSupported(
@@ -141,6 +154,7 @@ func setRunAsForJobs(b *bundle.Bundle) {
 			continue
 		}
 		job.RunAs = &jobs.JobRunAs{
+			GroupName:            runAs.GroupName,
 			ServicePrincipalName: runAs.ServicePrincipalName,
 			UserName:             runAs.UserName,
 		}
@@ -228,6 +242,13 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 	// User has opted to use the legacy behavior of run_as with the
 	// experimental.use_legacy_run_as flag.
 	if b.Config.Experimental != nil && b.Config.Experimental.UseLegacyRunAs {
+		if b.Config.Value().Get("run_as").Get("group_name").Kind() != dyn.KindInvalid {
+			return diag.Diagnostics{{
+				Summary:   "run_as.group_name is not supported with experimental.use_legacy_run_as; disable experimental.use_legacy_run_as to use a group identity",
+				Locations: b.Config.GetLocations("run_as.group_name"),
+				Severity:  diag.Error,
+			}}
+		}
 		setPipelineOwnersToRunAsIdentity(b)
 		setRunAsForJobs(b)
 		return diag.Diagnostics{

bundle/config/mutator/resourcemutator/run_as_test.go

@@ -1,6 +1,7 @@
 package resourcemutator
 
 import (
+	"fmt"
 	"slices"
 	"testing"
 
@@ -316,3 +317,60 @@ func TestRunAsNoErrorForSupportedResources(t *testing.T) {
 		require.NoError(t, diags.Error())
 	}
 }
+
+func TestRunAsGroupInvalidIdentity(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		runAs  string
+		legacy bool
+	}{
+		{name: "empty", runAs: `{group_name: ""}`},
+		{name: "user and group", runAs: `{user_name: user, group_name: group}`},
+		{name: "legacy", runAs: `{group_name: group}`, legacy: true},
+		{name: "legacy empty", runAs: `{group_name: ""}`, legacy: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			yaml := fmt.Sprintf("run_as: %s\nexperimental: {use_legacy_run_as: %t}", tc.runAs, tc.legacy)
+			r, diags := config.LoadFromBytes("databricks.yml", []byte(yaml))
+			require.NoError(t, diags.Error())
+			b := &bundle.Bundle{Config: *r}
+			diags = bundle.Apply(t.Context(), b, SetRunAs())
+			require.Error(t, diags.Error())
+			if tc.legacy {
+				assert.Contains(t, diags.Error().Error(), "run_as.group_name is not supported with experimental.use_legacy_run_as")
+			} else {
+				assert.Contains(t, diags.Error().Error(), "run_as section must specify exactly one non-empty identity")
+			}
+		})
+	}
+}
+
+func TestRunAsGroupResources(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		resource  string
+		wantError string
+	}{
+		{name: "pipeline", resource: `pipelines: {test: {}}`, wantError: "pipelines do not support run_as.group_name"},
+		{name: "alert", resource: `alerts: {test: {}}`, wantError: "alerts do not support run_as.group_name"},
+		{name: "model serving", resource: `model_serving_endpoints: {test: {}}`, wantError: "Run as identity: group \"group\""},
+		{name: "pipeline user override", resource: `pipelines: {test: {run_as: {user_name: user}}}`},
+		{name: "alert sp override", resource: `alerts: {test: {run_as: {service_principal_name: sp}}}`},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			yaml := "run_as: {group_name: group}\nworkspace: {current_user: {userName: group}}\nresources:\n  " + tc.resource
+			r, diags := config.LoadFromBytes("databricks.yml", []byte(yaml))
+			require.NoError(t, diags.Error())
+			b := &bundle.Bundle{Config: *r}
+			before := b.Config.Value().Get("resources")
+			diags = bundle.Apply(t.Context(), b, SetRunAs())
+			if tc.wantError != "" {
+				require.Error(t, diags.Error())
+				assert.Contains(t, diags.Error().Error(), tc.wantError)
+			} else {
+				require.NoError(t, diags.Error())
+				assert.Equal(t, before, b.Config.Value().Get("resources"))
+			}
+		})
+	}
+}

bundle/config/mutator/validate_run_as_group.go

@@ -0,0 +1,51 @@
+package mutator
+
+import (
+	"context"
+	"maps"
+	"slices"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/engine"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+type validateRunAsGroup struct {
+	engine engine.EngineType
+}
+
+// ValidateRunAsGroup rejects group identities unsupported by the Terraform provider.
+// See https://github.com/databricks/terraform-provider-databricks/blob/v1.131.0/jobs/resource_job.go#L629.
+func ValidateRunAsGroup(e engine.EngineType) bundle.Mutator {
+	return &validateRunAsGroup{engine: e}
+}
+
+func (m *validateRunAsGroup) Name() string {
+	return "ValidateRunAsGroup"
+}
+
+func (m *validateRunAsGroup) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
+	if m.engine.IsDirect() {
+		return nil
+	}
+
+	var diags diag.Diagnostics
+	for _, key := range slices.Sorted(maps.Keys(b.Config.Resources.Jobs)) {
+		runAs := b.Config.Resources.Jobs[key].RunAs
+		if runAs == nil || runAs.GroupName == "" {
+			continue
+		}
+		path := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key(key), dyn.Key("run_as"), dyn.Key("group_name"))
+		diags = diags.Append(diag.Diagnostic{
+			Severity: diag.Error,
+			Summary:  "run_as.group_name is only supported in direct deployment mode",
+			Detail: "Set 'bundle.engine: direct' in your databricks.yml or set DATABRICKS_BUNDLE_ENGINE=direct. " +
+				"For an existing Terraform deployment, run 'databricks bundle migrate' first. " +
+				"Alternatively, set run_as.user_name or run_as.service_principal_name on this job",
+			Locations: b.Config.GetLocations(path.String()),
+			Paths:     []dyn.Path{path},
+		})
+	}
+	return diags
+}

bundle/config/mutator/validate_run_as_group_test.go

@@ -0,0 +1,40 @@
+package mutator_test
+
+import (
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/engine"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestValidateRunAsGroup(t *testing.T) {
+	for _, tc := range []struct {
+		engine engine.EngineType
+		paths  []string
+	}{
+		{engine: engine.EngineDirect},
+		{engine: engine.EngineTerraform, paths: []string{"resources.jobs.a.run_as.group_name", "resources.jobs.z.run_as.group_name"}},
+	} {
+		t.Run(string(tc.engine), func(t *testing.T) {
+			b := &bundle.Bundle{Config: config.Root{Resources: config.Resources{Jobs: map[string]*resources.Job{
+				"z":    {JobSettings: jobs.JobSettings{RunAs: &jobs.JobRunAs{GroupName: "group"}}},
+				"a":    {JobSettings: jobs.JobSettings{RunAs: &jobs.JobRunAs{GroupName: "group"}}},
+				"user": {JobSettings: jobs.JobSettings{RunAs: &jobs.JobRunAs{UserName: "user@example.test"}}},
+				"sp":   {JobSettings: jobs.JobSettings{RunAs: &jobs.JobRunAs{ServicePrincipalName: "sp"}}},
+				"none": {},
+			}}}}
+			diags := bundle.Apply(t.Context(), b, mutator.ValidateRunAsGroup(tc.engine))
+			require.Len(t, diags, len(tc.paths))
+			for i, path := range tc.paths {
+				assert.Equal(t, []dyn.Path{dyn.MustPathFromString(path)}, diags[i].Paths)
+			}
+		})
+	}
+}

bundle/phases/plan.go

@@ -27,6 +27,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine
 		deploy.StatePull(),
 		mutator.ValidateGitDetails(),
 		mutator.ValidateDirectOnlyResources(engine),
+		mutator.ValidateRunAsGroup(engine),
 		mutator.ValidateLifecycleStarted(engine),
 		mutator.ValidateCascadeOnDestroy(engine),
 		statemgmt.CheckRunningResource(engine),