diff --git a/.nextchanges/cli/apps-init-auth-mode.md b/.nextchanges/cli/apps-init-auth-mode.md new file mode 100644 index 00000000000..6044a71fb16 --- /dev/null +++ b/.nextchanges/cli/apps-init-auth-mode.md @@ -0,0 +1 @@ +* Added `--auth-mode` and `--set ..authMode=obo|sp|both` to `databricks apps init` so AppKit resources can be accessed on behalf of the user, by the service principal, or both. The default stays service principal. ([#6886](https://github.com/databricks/cli/pull/6886)) diff --git a/acceptance/apps/init/auth-mode/out.test.toml b/acceptance/apps/init/auth-mode/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/apps/init/auth-mode/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/apps/init/auth-mode/output.txt b/acceptance/apps/init/auth-mode/output.txt new file mode 100644 index 00000000000..f03bb342363 --- /dev/null +++ b/acceptance/apps/init/auth-mode/output.txt @@ -0,0 +1,415 @@ + +=== Default: every resource bound to the service principal +>>> [CLI] apps init --template ./template --skip-install --name app-sp --features analytics,genie,jobs --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g1 --set jobs.job.id=42 --set jobs.secret.scope=s --set jobs.secret.key=k +✔ Creating project + +✔ Project created successfully! + + Location: [TEST_TMP_DIR]/app-sp + Files: 3 + +command: ['npm', 'run', 'start'] +env: + - name: DATABRICKS_WAREHOUSE_ID + valueFrom: sql-warehouse + - name: DATABRICKS_GENIE_SPACE_ID + valueFrom: genie-space + - name: DATABRICKS_JOB_ID + valueFrom: job + - name: SECRET_KEY + valueFrom: secret + - name: SECRET_SCOPE + valueFrom: secret +bundle: + name: app-sp + +variables: + sql_warehouse_id: + description: SQL Warehouse ID + genie_space_id: + description: Genie Space ID + genie_space_name: + description: Genie Space + job_id: + description: Job ID + secret_key: + description: Job token + secret_scope: + description: Job token + +resources: + apps: + app: + name: "app-sp" + description: "A Databricks App powered by AppKit" + source_code_path: ./ + + # Start the app automatically on deploy. + lifecycle: + started: true + user_api_scopes: + - dashboards.genie + + # The resources which this app has access to. + resources: + - name: sql-warehouse + sql_warehouse: + id: ${var.sql_warehouse_id} + permission: CAN_USE + - name: genie-space + genie_space: + name: ${var.genie_space_name} + space_id: ${var.genie_space_id} + permission: CAN_RUN + - name: job + job: + id: ${var.job_id} + permission: CAN_MANAGE_RUN + - name: secret + secret: + scope: ${var.secret_scope} + key: ${var.secret_key} + permission: READ + +targets: + default: + default: true + workspace: + host: [DATABRICKS_URL] + + variables: + sql_warehouse_id: wh1 + genie_space_id: g1 + job_id: 42 + secret_key: k + secret_scope: s + +=== App-wide OBO, job kept on the service principal +>>> [CLI] apps init --template ./template --skip-install --name app-obo --features analytics,genie,jobs --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g1 --set jobs.job.id=42 --set jobs.secret.scope=s --set jobs.secret.key=k --auth-mode obo --set jobs.job.authMode=sp +Note: these resources cannot be accessed on behalf of the user and use the service principal: secret +✔ Creating project + +✔ Project created successfully! + + Location: [TEST_TMP_DIR]/app-obo + Files: 3 + +command: ['npm', 'run', 'start'] +env: + - name: DATABRICKS_WAREHOUSE_ID + value: wh1 + - name: DATABRICKS_GENIE_SPACE_ID + value: g1 + - name: DATABRICKS_JOB_ID + valueFrom: job + - name: SECRET_KEY + valueFrom: secret + - name: SECRET_SCOPE + valueFrom: secret +bundle: + name: app-obo + +variables: + job_id: + description: Job ID + secret_key: + description: Job token + secret_scope: + description: Job token + +resources: + apps: + app: + name: "app-obo" + description: "A Databricks App powered by AppKit" + source_code_path: ./ + + # Start the app automatically on deploy. + lifecycle: + started: true + user_api_scopes: + - sql + - genie + - ai-gateway + + # The resources which this app has access to. + resources: + - name: job + job: + id: ${var.job_id} + permission: CAN_MANAGE_RUN + - name: secret + secret: + scope: ${var.secret_scope} + key: ${var.secret_key} + permission: READ + +targets: + default: + default: true + workspace: + host: [DATABRICKS_URL] + + variables: + job_id: 42 + secret_key: k + secret_scope: s + +=== Mixed: warehouse both, genie OBO +>>> [CLI] apps init --template ./template --skip-install --name app-mixed --features analytics,genie,jobs --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g1 --set jobs.job.id=42 --set jobs.secret.scope=s --set jobs.secret.key=k --set analytics.sql-warehouse.authMode=both --set genie.genie-space.authMode=obo +✔ Creating project + +✔ Project created successfully! + + Location: [TEST_TMP_DIR]/app-mixed + Files: 3 + +command: ['npm', 'run', 'start'] +env: + - name: DATABRICKS_WAREHOUSE_ID + valueFrom: sql-warehouse + - name: DATABRICKS_GENIE_SPACE_ID + value: g1 + - name: DATABRICKS_JOB_ID + valueFrom: job + - name: SECRET_KEY + valueFrom: secret + - name: SECRET_SCOPE + valueFrom: secret +bundle: + name: app-mixed + +variables: + sql_warehouse_id: + description: SQL Warehouse ID + job_id: + description: Job ID + secret_key: + description: Job token + secret_scope: + description: Job token + +resources: + apps: + app: + name: "app-mixed" + description: "A Databricks App powered by AppKit" + source_code_path: ./ + + # Start the app automatically on deploy. + lifecycle: + started: true + user_api_scopes: + - sql + - genie + - ai-gateway + + # The resources which this app has access to. + resources: + - name: sql-warehouse + sql_warehouse: + id: ${var.sql_warehouse_id} + permission: CAN_USE + - name: job + job: + id: ${var.job_id} + permission: CAN_MANAGE_RUN + - name: secret + secret: + scope: ${var.secret_scope} + key: ${var.secret_key} + permission: READ + +targets: + default: + default: true + workspace: + host: [DATABRICKS_URL] + + variables: + sql_warehouse_id: wh1 + job_id: 42 + secret_key: k + secret_scope: s + +=== App-wide OBO: resources without a scope stay on the service principal +>>> [CLI] apps init --template ./template --skip-install --name app-obo-lenient --features analytics,genie,jobs --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g1 --set jobs.job.id=42 --set jobs.secret.scope=s --set jobs.secret.key=k --auth-mode obo +Note: these resources cannot be accessed on behalf of the user and use the service principal: job, secret +✔ Creating project + +✔ Project created successfully! + + Location: [TEST_TMP_DIR]/app-obo-lenient + Files: 3 + +command: ['npm', 'run', 'start'] +env: + - name: DATABRICKS_WAREHOUSE_ID + value: wh1 + - name: DATABRICKS_GENIE_SPACE_ID + value: g1 + - name: DATABRICKS_JOB_ID + valueFrom: job + - name: SECRET_KEY + valueFrom: secret + - name: SECRET_SCOPE + valueFrom: secret +bundle: + name: app-obo-lenient + +variables: + job_id: + description: Job ID + secret_key: + description: Job token + secret_scope: + description: Job token + +resources: + apps: + app: + name: "app-obo-lenient" + description: "A Databricks App powered by AppKit" + source_code_path: ./ + + # Start the app automatically on deploy. + lifecycle: + started: true + user_api_scopes: + - sql + - genie + - ai-gateway + + # The resources which this app has access to. + resources: + - name: job + job: + id: ${var.job_id} + permission: CAN_MANAGE_RUN + - name: secret + secret: + scope: ${var.secret_scope} + key: ${var.secret_key} + permission: READ + +targets: + default: + default: true + workspace: + host: [DATABRICKS_URL] + + variables: + job_id: 42 + secret_key: k + secret_scope: s + +=== Analytics OBO, genie space on the service principal: plugin scopes still emitted +>>> [CLI] apps init --template ./template --skip-install --name app-genie-sp --features analytics,genie,jobs --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g1 --set jobs.job.id=42 --set jobs.secret.scope=s --set jobs.secret.key=k --set analytics.sql-warehouse.authMode=obo --set genie.genie-space.authMode=sp +✔ Creating project + +✔ Project created successfully! + + Location: [TEST_TMP_DIR]/app-genie-sp + Files: 3 + +command: ['npm', 'run', 'start'] +env: + - name: DATABRICKS_WAREHOUSE_ID + value: wh1 + - name: DATABRICKS_GENIE_SPACE_ID + valueFrom: genie-space + - name: DATABRICKS_JOB_ID + valueFrom: job + - name: SECRET_KEY + valueFrom: secret + - name: SECRET_SCOPE + valueFrom: secret +bundle: + name: app-genie-sp + +variables: + genie_space_id: + description: Genie Space ID + genie_space_name: + description: Genie Space + job_id: + description: Job ID + secret_key: + description: Job token + secret_scope: + description: Job token + +resources: + apps: + app: + name: "app-genie-sp" + description: "A Databricks App powered by AppKit" + source_code_path: ./ + + # Start the app automatically on deploy. + lifecycle: + started: true + user_api_scopes: + - sql + - ai-gateway + - genie + + # The resources which this app has access to. + resources: + - name: genie-space + genie_space: + name: ${var.genie_space_name} + space_id: ${var.genie_space_id} + permission: CAN_RUN + - name: job + job: + id: ${var.job_id} + permission: CAN_MANAGE_RUN + - name: secret + secret: + scope: ${var.secret_scope} + key: ${var.secret_key} + permission: READ + +targets: + default: + default: true + workspace: + host: [DATABRICKS_URL] + + variables: + genie_space_id: g1 + job_id: 42 + secret_key: k + secret_scope: s + +=== Errors +>>> errcode [CLI] apps init --template ./template --skip-install --name app-err --features jobs --set jobs.job.authMode=obo +Error: resource "job" of plugin "jobs" cannot be accessed on behalf of the user; use jobs.job.authMode=sp + +Exit code: 1 + +>>> errcode [CLI] apps init --template ./template --skip-install --name app-err --features jobs --set jobs.secret.authMode=sp +Error: resource "secret" of plugin "jobs" is always accessed by the app's service principal; remove --set jobs.secret.authMode=sp + +Exit code: 1 + +>>> errcode [CLI] apps init --template ./template --skip-install --name app-err --features jobs --auth-mode both +Error: invalid --auth-mode value: "both" (must be obo or sp; use --set ..authMode=both for a single resource) + +Exit code: 1 + +=== Validate: OBO resource whose scope was removed from user_api_scopes +>>> [CLI] apps init --template ./template --skip-install --name app-val --features analytics --set analytics.sql-warehouse.id=wh1 --set analytics.sql-warehouse.authMode=obo +✔ Creating project + +✔ Project created successfully! + + Location: [TEST_TMP_DIR]/app-val + Files: 3 + + +>>> errcode [CLI] apps validate --path app-val +Error: resource "sql-warehouse" is accessed on behalf of the user but its scope "sql" is missing from user_api_scopes in databricks.yml + +Exit code: 1 diff --git a/acceptance/apps/init/auth-mode/script b/acceptance/apps/init/auth-mode/script new file mode 100644 index 00000000000..94e8a4eb1b0 --- /dev/null +++ b/acceptance/apps/init/auth-mode/script @@ -0,0 +1,37 @@ +init() { + name=$1; shift + trace $CLI apps init --template ./template --skip-install --name $name \ + --features analytics,genie,jobs \ + --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g1 \ + --set jobs.job.id=42 --set jobs.secret.scope=s --set jobs.secret.key=k "$@" + cat $name/app.yaml $name/databricks.yml + rm -rf $name +} + +title "Default: every resource bound to the service principal" +init app-sp + +title "App-wide OBO, job kept on the service principal" +init app-obo --auth-mode obo --set jobs.job.authMode=sp + +title "Mixed: warehouse both, genie OBO" +init app-mixed --set analytics.sql-warehouse.authMode=both --set genie.genie-space.authMode=obo + +title "App-wide OBO: resources without a scope stay on the service principal" +init app-obo-lenient --auth-mode obo + +title "Analytics OBO, genie space on the service principal: plugin scopes still emitted" +init app-genie-sp --set analytics.sql-warehouse.authMode=obo --set genie.genie-space.authMode=sp + +title "Errors" +trace errcode $CLI apps init --template ./template --skip-install --name app-err --features jobs --set jobs.job.authMode=obo +trace errcode $CLI apps init --template ./template --skip-install --name app-err --features jobs --set jobs.secret.authMode=sp +trace errcode $CLI apps init --template ./template --skip-install --name app-err --features jobs --auth-mode both + +title "Validate: OBO resource whose scope was removed from user_api_scopes" +trace $CLI apps init --template ./template --skip-install --name app-val --features analytics \ + --set analytics.sql-warehouse.id=wh1 --set analytics.sql-warehouse.authMode=obo +grep -v -- '- sql$' app-val/databricks.yml > app-val/databricks.yml.new +mv app-val/databricks.yml.new app-val/databricks.yml +trace errcode $CLI apps validate --path app-val +rm -rf app-val diff --git a/acceptance/apps/init/auth-mode/template/app.yaml.tmpl b/acceptance/apps/init/auth-mode/template/app.yaml.tmpl new file mode 100644 index 00000000000..66311d1213e --- /dev/null +++ b/acceptance/apps/init/auth-mode/template/app.yaml.tmpl @@ -0,0 +1,5 @@ +command: ['npm', 'run', 'start'] +{{- if .appEnv}} +env: +{{.appEnv}} +{{- end}} diff --git a/acceptance/apps/init/auth-mode/template/appkit.plugins.json b/acceptance/apps/init/auth-mode/template/appkit.plugins.json new file mode 100644 index 00000000000..0dfa00ca91d --- /dev/null +++ b/acceptance/apps/init/auth-mode/template/appkit.plugins.json @@ -0,0 +1,75 @@ +{ + "version": "1.0", + "plugins": { + "analytics": { + "name": "analytics", + "displayName": "Analytics", + "description": "SQL analytics", + "package": "@databricks/appkit", + "scopes": ["ai-gateway"], + "resources": { + "required": [ + { + "type": "sql_warehouse", + "alias": "SQL Warehouse", + "resourceKey": "sql-warehouse", + "description": "SQL Warehouse for analytics queries", + "permission": "CAN_USE", + "scope": "sql", + "fields": {"id": {"env": "DATABRICKS_WAREHOUSE_ID", "description": "SQL Warehouse ID"}} + } + ], + "optional": [] + } + }, + "genie": { + "name": "genie", + "displayName": "Genie", + "description": "Genie spaces", + "package": "@databricks/appkit", + "scopes": ["genie"], + "resources": { + "required": [ + { + "type": "genie_space", + "alias": "Genie Space", + "resourceKey": "genie-space", + "description": "Genie Space", + "permission": "CAN_RUN", + "scope": "genie", + "fields": {"id": {"env": "DATABRICKS_GENIE_SPACE_ID", "description": "Genie Space ID"}} + } + ], + "optional": [] + } + }, + "jobs": { + "name": "jobs", + "displayName": "Jobs", + "description": "Lakeflow jobs", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "job", + "alias": "Job", + "resourceKey": "job", + "description": "Job to trigger", + "permission": "CAN_MANAGE_RUN", + "fields": {"id": {"env": "DATABRICKS_JOB_ID", "description": "Job ID"}} + }, + { + "type": "secret", + "alias": "Secret", + "resourceKey": "secret", + "description": "Job token", + "permission": "READ", + "appOnly": true, + "fields": {"scope": {"env": "SECRET_SCOPE"}, "key": {"env": "SECRET_KEY"}} + } + ], + "optional": [] + } + } + } +} diff --git a/acceptance/apps/init/auth-mode/template/databricks.yml.tmpl b/acceptance/apps/init/auth-mode/template/databricks.yml.tmpl new file mode 100644 index 00000000000..6a749066692 --- /dev/null +++ b/acceptance/apps/init/auth-mode/template/databricks.yml.tmpl @@ -0,0 +1,55 @@ +bundle: + name: {{.projectName}} +{{- if .bundle.variables}} + +variables: +{{.bundle.variables}} +{{- end}} + +resources: + apps: + app: + name: "{{.projectName}}" + description: "{{.appDescription}}" + source_code_path: ./ + + # Start the app automatically on deploy. + lifecycle: + started: true + +{{- if .bundle.userApiScopes}} + user_api_scopes: +{{.bundle.userApiScopes}} +{{- else if or .plugins.genie .plugins.files .plugins.serving}} + user_api_scopes: +{{- if .plugins.genie}} + - dashboards.genie +{{- end}} +{{- if .plugins.files}} + - files.files +{{- end}} +{{- if .plugins.serving}} + - serving.serving-endpoints +{{- end}} +{{- else}} + # Uncomment to enable on behalf of user API scopes. Available scopes: sql, dashboards.genie, files.files, serving.serving-endpoints + # user_api_scopes: + # - sql +{{- end}} +{{- if .bundle.resources}} + + # The resources which this app has access to. + resources: +{{.bundle.resources}} +{{- end}} + +targets: + default: + default: true + workspace: + host: {{.workspaceHost}} +{{- if .bundle.targetVariables}} + + variables: +{{.bundle.targetVariables}} +{{- end}} diff --git a/acceptance/apps/init/auth-mode/test.toml b/acceptance/apps/init/auth-mode/test.toml new file mode 100644 index 00000000000..b7713e67a70 --- /dev/null +++ b/acceptance/apps/init/auth-mode/test.toml @@ -0,0 +1,6 @@ +Cloud = false + +[[Repls]] +# Fix path with reverse slashes in the output for Windows. +Old = 'TEST_TMP_DIR]\\(app-[a-z-]+)' +New = 'TEST_TMP_DIR]/${1}' diff --git a/cmd/apps/init.go b/cmd/apps/init.go index b44249d9277..588b131f62d 100644 --- a/cmd/apps/init.go +++ b/cmd/apps/init.go @@ -54,6 +54,10 @@ const ( // are not provided upfront. When set to "true", --set requirement and // resource validation are skipped. agenticModeEnvVar = "DATABRICKS_APPS_AGENTIC_MODE" + + // authModeField is the reserved --set field that selects a resource's auth mode + // (e.g., --set analytics.sql-warehouse.authMode=obo). + authModeField = "authMode" ) // normalizeVersion converts a version string to the template tag format "template-vX.X.X". @@ -94,6 +98,7 @@ func newInitCmd() *cobra.Command { autoApprove bool skipInstall bool packageManager string + authMode string ) cmd := &cobra.Command{ @@ -135,6 +140,11 @@ Examples: --set analytics.sql-warehouse.id=wh1 \ --set reporting.sql-warehouse.id=wh2 + # Access resources on behalf of the user, except the job + databricks apps init --name my-app --features=analytics,jobs --auth-mode obo \ + --set jobs.job.authMode=sp \ + --set analytics.sql-warehouse.id=abc123 --set jobs.job.id=42 + # Create, deploy, and run with dev-remote databricks apps init --name my-app --deploy --run=dev-remote @@ -148,6 +158,8 @@ Resource configuration (--set): Set resource values using --set plugin.resourceKey.field=value Keys are defined in the template's appkit.plugins.json manifest. Multi-field resources (e.g., database, secret) require all fields to be set together. + Use --set plugin.resourceKey.authMode=obo|sp|both to choose how the app accesses a resource: + on behalf of the user (obo), as the app's service principal (sp), or both. Environment variables: DATABRICKS_APPKIT_TEMPLATE_PATH Override the default template source`, @@ -181,6 +193,7 @@ Environment variables: skipInstall: skipInstall, packageManager: packageManager, packageManagerChanged: cmd.Flags().Changed("package-manager"), + authMode: authMode, }) }, } @@ -200,6 +213,7 @@ Environment variables: cmd.Flags().BoolVar(&deploy, "deploy", false, "Deploy the app after creation") cmd.Flags().StringVar(&run, "run", "", "Run the app after creation (none, dev, dev-remote)") cmd.Flags().BoolVar(&autoApprove, "auto-approve", false, "Skip confirmation prompts for optional resources. Optional resources are only configured when their values are provided via --set.") + cmd.Flags().StringVar(&authMode, "auth-mode", "", "Default resource auth mode: obo (on behalf of the user) or sp (service principal). With obo, resources that cannot be accessed on behalf of the user use sp. Prompts if not provided in interactive mode, otherwise sp.") cmd.Flags().BoolVar(&skipInstall, "skip-install", false, "Skip installing project dependencies (e.g. npm install / uv sync). Cannot be combined with --run.") cmd.Flags().StringVar(&packageManager, "package-manager", "", "Node.js package manager (pnpm, npm; default: inferred from template)") @@ -226,6 +240,7 @@ type createOptions struct { skipInstall bool packageManager string packageManagerChanged bool + authMode string // --auth-mode default for all resources } // parseSetValues parses --set key=value pairs into the resourceValues map. @@ -243,6 +258,9 @@ func parseSetValues(setValues []string, m *manifest.Manifest) (map[string]string return nil, fmt.Errorf("invalid --set key %q, expected plugin.resourceKey.field", key) } pluginName, resourceKey, fieldName := parts[0], parts[1], parts[2] + if fieldName == authModeField { + continue + } plugin := m.GetPluginByName(pluginName) if plugin == nil { @@ -293,6 +311,82 @@ func parseSetValues(setValues []string, m *manifest.Manifest) (map[string]string return rv, nil } +// parseSetAuthModes parses --set plugin.resourceKey.authMode=value pairs into a map of resource key to auth mode. +// Other --set pairs are ignored (see parseSetValues). +func parseSetAuthModes(setValues []string, m *manifest.Manifest) (map[string]string, error) { + modes := make(map[string]string) + for _, sv := range setValues { + key, value, _ := strings.Cut(sv, "=") + parts := strings.SplitN(key, ".", 3) + if len(parts) != 3 || parts[2] != authModeField { + continue + } + pluginName, resourceKey := parts[0], parts[1] + + plugin := m.GetPluginByName(pluginName) + if plugin == nil { + return nil, fmt.Errorf("unknown plugin %q in --set %q; available: %v", pluginName, sv, m.GetPluginNames()) + } + all := append(plugin.Resources.Required, plugin.Resources.Optional...) + idx := slices.IndexFunc(all, func(r manifest.Resource) bool { return r.Key() == resourceKey }) + if idx < 0 { + return nil, fmt.Errorf("plugin %q has no resource with key %q", pluginName, resourceKey) + } + r := all[idx] + + switch value { + case generator.AuthModeSP, generator.AuthModeOBO, generator.AuthModeBoth: + default: + return nil, fmt.Errorf("invalid auth mode %q in --set %q (must be obo, sp, or both)", value, sv) + } + if r.AppOnly { + return nil, fmt.Errorf("resource %q of plugin %q is always accessed by the app's service principal; remove --set %s", resourceKey, pluginName, sv) + } + if value != generator.AuthModeSP && r.Scope == "" { + return nil, fmt.Errorf("resource %q of plugin %q cannot be accessed on behalf of the user; use %s.%s.%s=sp", resourceKey, pluginName, pluginName, resourceKey, authModeField) + } + modes[r.AuthKey()] = value + } + return modes, nil +} + +// resolveAuthModes returns the auth mode of each resource. Precedence, highest first: +// --set authMode, the prompted choice, the --auth-mode default, then sp. +// setModes and promptedModes only hold modes their resources support. The --auth-mode +// default applies only to resources that can be accessed on behalf of the user; the keys +// of the other resources it does not apply to are returned as keptSP. +func resolveAuthModes(resources []manifest.Resource, setModes, promptedModes map[string]string, defaultMode string) (modes map[string]string, keptSP []string) { + modes = make(map[string]string) + for _, r := range resources { + mode, ok := setModes[r.AuthKey()] + if !ok { + mode, ok = promptedModes[r.AuthKey()] + } + if !ok && defaultMode == generator.AuthModeOBO { + if r.AppOnly || r.Scope == "" { + keptSP = append(keptSP, r.Key()) + continue + } + mode = defaultMode + } + if mode == generator.AuthModeOBO || mode == generator.AuthModeBoth { + modes[r.AuthKey()] = mode + } + } + return modes, keptSP +} + +// resourceConfigured reports whether any of the resource's field values are set. +// Resource values are keyed "resourceKey.fieldName" (see parseSetValues). +func resourceConfigured(r manifest.Resource, values map[string]string) bool { + for _, fieldName := range r.FieldNames() { + if values[r.Key()+"."+fieldName] != "" { + return true + } + } + return false +} + // pluginHasResourceField checks whether a plugin declares a resource with the given key and field name. func pluginHasResourceField(p *manifest.Plugin, resourceKey, fieldName string) bool { for _, r := range append(p.Resources.Required, p.Resources.Optional...) { @@ -332,6 +426,7 @@ type tmplBundle struct { Variables string Resources string TargetVariables string + UserAPIScopes string } // dotEnvVars holds the generated .env file content. @@ -390,7 +485,8 @@ func parseDeployAndRunFlags(deploy bool, run string) (bool, prompt.RunMode, erro // promptForPluginsAndDeps prompts for plugins and their resource dependencies using the manifest. // skipDeployRunPrompt indicates whether to skip prompting for deploy/run (because flags were provided). -func promptForPluginsAndDeps(ctx context.Context, m *manifest.Manifest, preSelectedPlugins []string, skipDeployRunPrompt, autoApprove bool) (*prompt.CreateProjectConfig, error) { +// promptAuthMode indicates whether to prompt for how the app accesses resources (--auth-mode was not provided). +func promptForPluginsAndDeps(ctx context.Context, m *manifest.Manifest, preSelectedPlugins []string, skipDeployRunPrompt, autoApprove, promptAuthMode bool) (*prompt.CreateProjectConfig, error) { config := &prompt.CreateProjectConfig{ Dependencies: make(map[string]string), Features: preSelectedPlugins, // Reuse Features field for plugin names @@ -463,6 +559,24 @@ func promptForPluginsAndDeps(ctx context.Context, m *manifest.Manifest, preSelec } } + // Step 3b: Auth mode for resources that can be accessed on behalf of the user. + if promptAuthMode { + var configured []manifest.Resource + for _, r := range append(resources, optionalResources...) { + for k := range config.Dependencies { + if strings.HasPrefix(k, r.Key()+".") { + configured = append(configured, r) + break + } + } + } + modes, err := promptForAuthModes(ctx, configured, theme) + if err != nil { + return nil, err + } + config.AuthModes = modes + } + // Step 4: Description config.Description = prompt.DefaultAppDescription err := huh.NewInput(). @@ -490,6 +604,79 @@ func promptForPluginsAndDeps(ctx context.Context, m *manifest.Manifest, preSelec return config, nil } +// promptForAuthModes asks once how the app accesses its resources and, for "Mixed", +// asks per resource that can be accessed on behalf of the user. +// Resources that cannot are listed as info and resolve to sp. +// Returns a map of resource key to auth mode, or nil if nothing needs to be asked. +func promptForAuthModes(ctx context.Context, resources []manifest.Resource, theme *huh.Theme) (map[string]string, error) { + var capable []manifest.Resource + for _, r := range resources { + if r.Scope != "" && !r.AppOnly { + capable = append(capable, r) + } + } + if len(capable) == 0 { + return nil, nil + } + + const mixed = "mixed" + appMode := generator.AuthModeSP + err := huh.NewSelect[string](). + Title("How should the app access resources?"). + Options( + huh.NewOption("Service principal", generator.AuthModeSP), + huh.NewOption("On behalf of user", generator.AuthModeOBO), + huh.NewOption("Mixed (choose per resource)", mixed), + ). + Value(&appMode). + WithTheme(theme). + Run() + if err != nil { + return nil, err + } + + modes := make(map[string]string, len(capable)) + for _, r := range capable { + mode := appMode + if appMode == mixed { + mode = generator.AuthModeSP + err := huh.NewSelect[string](). + Title(fmt.Sprintf("How should the app access %s?", r.Alias)). + Options( + huh.NewOption("On behalf of user", generator.AuthModeOBO), + huh.NewOption("Service principal", generator.AuthModeSP), + huh.NewOption("Both", generator.AuthModeBoth), + ). + Value(&mode). + WithTheme(theme). + Run() + if err != nil { + return nil, err + } + } + modes[r.AuthKey()] = mode + prompt.PrintAnswered(ctx, r.Alias, authModeLabel(mode)) + } + for _, r := range resources { + if r.Scope == "" || r.AppOnly { + prompt.PrintAnswered(ctx, r.Alias, authModeLabel(generator.AuthModeSP)+" (only option)") + } + } + return modes, nil +} + +// authModeLabel returns a user-facing label for an auth mode. +func authModeLabel(mode string) string { + switch mode { + case generator.AuthModeOBO: + return "on behalf of user" + case generator.AuthModeBoth: + return "service principal and on behalf of user" + default: + return "service principal" + } +} + // promptForResource prompts the user for a resource value. // Returns a map of value keys to values. For single-field resources the key is "resource_key.field". // For multi-field resources, keys use "resource_key.field_name". @@ -1014,8 +1201,15 @@ func runCreate(ctx context.Context, opts createOptions) (runErr error) { return errors.New("--skip-install cannot be combined with --run (dev/dev-remote require dependencies to be installed)") } + switch opts.authMode { + case "", generator.AuthModeSP, generator.AuthModeOBO: + default: + return fmt.Errorf("invalid --auth-mode value: %q (must be obo or sp; use --set ..authMode=both for a single resource)", opts.authMode) + } + var selectedPlugins []string var resourceValues map[string]string + var promptedAuthModes map[string]string var shouldDeploy bool var runMode prompt.RunMode isInteractive := cmdio.IsPromptSupported(ctx) @@ -1323,12 +1517,13 @@ func runCreate(ctx context.Context, opts createOptions) (runErr error) { } } else if isInteractive && !opts.pluginsChanged && !flagsMode { // Interactive mode without --plugins flag: prompt for plugins, dependencies, description - config, err := promptForPluginsAndDeps(ctx, m, selectedPlugins, skipDeployRunPrompt, opts.autoApprove) + config, err := promptForPluginsAndDeps(ctx, m, selectedPlugins, skipDeployRunPrompt, opts.autoApprove, opts.authMode == "") if err != nil { return err } selectedPlugins = config.Features // Features field holds plugin names resourceValues = config.Dependencies + promptedAuthModes = config.AuthModes if config.Description != "" { opts.description = config.Description } @@ -1376,9 +1571,27 @@ func runCreate(ctx context.Context, opts createOptions) (runErr error) { maps.Copy(resourceValues, setVals) } + setAuthModes, err := parseSetAuthModes(opts.setValues, m) + if err != nil { + return err + } + // Always include mandatory plugins regardless of user selection or flags. selectedPlugins = appendUnique(selectedPlugins, m.GetMandatoryPluginNames()...) + // Only optional resources that were actually configured end up in the project, + // so unconfigured ones must not be resolved or listed in the keptSP note. + authResources := m.CollectResources(selectedPlugins) + for _, r := range m.CollectOptionalResources(selectedPlugins) { + if resourceConfigured(r, resourceValues) { + authResources = append(authResources, r) + } + } + authModes, keptSP := resolveAuthModes(authResources, setAuthModes, promptedAuthModes, opts.authMode) + if len(keptSP) > 0 { + cmdio.LogString(ctx, "Note: these resources cannot be accessed on behalf of the user and use the service principal: "+strings.Join(keptSP, ", ")) + } + // Warn when --features adds plugins that the pre-rendered template // cannot inject (its server.ts and app.yaml are already finalised). if skipPluginSelection && opts.pluginsChanged { @@ -1463,6 +1676,7 @@ func runCreate(ctx context.Context, opts createOptions) (runErr error) { WorkspaceHost: workspaceHost, Profile: profile, ResourceValues: resourceValues, + AuthModes: authModes, } // Generate configurations from selected plugins @@ -1493,6 +1707,7 @@ func runCreate(ctx context.Context, opts createOptions) (runErr error) { Variables: bundleVars, Resources: bundleRes, TargetVariables: targetVars, + UserAPIScopes: generator.GenerateUserAPIScopes(selectedPluginList, genConfig), }, DotEnv: dotEnvVars{ Content: generator.GenerateDotEnv(selectedPluginList, genConfig), @@ -1968,6 +2183,7 @@ func templateData(vars templateVars) map[string]any { "variables": vars.Bundle.Variables, "resources": vars.Bundle.Resources, "targetVariables": vars.Bundle.TargetVariables, + "userApiScopes": vars.Bundle.UserAPIScopes, }, "dotEnv": map[string]any{ "content": vars.DotEnv.Content, diff --git a/cmd/apps/init_test.go b/cmd/apps/init_test.go index 5727c6e4526..44fa9bb7922 100644 --- a/cmd/apps/init_test.go +++ b/cmd/apps/init_test.go @@ -1626,6 +1626,97 @@ func TestSetFirstAppNameNoResources(t *testing.T) { assert.Equal(t, "myapp", bundleName.Value) } +func authModeManifest() *manifest.Manifest { + return &manifest.Manifest{ + Plugins: map[string]manifest.Plugin{ + "analytics": { + Name: "analytics", + Resources: manifest.Resources{ + Required: []manifest.Resource{ + {Type: "sql_warehouse", ResourceKey: "sql-warehouse", Scope: "sql", PluginName: "analytics", Fields: map[string]manifest.ResourceField{"id": {}}}, + {Type: "secret", ResourceKey: "secret", AppOnly: true, PluginName: "analytics", Fields: map[string]manifest.ResourceField{"scope": {}, "key": {}}}, + }, + }, + }, + "jobs": { + Name: "jobs", + Resources: manifest.Resources{ + Required: []manifest.Resource{ + {Type: "job", ResourceKey: "job", PluginName: "jobs", Fields: map[string]manifest.ResourceField{"id": {}}}, + }, + }, + }, + }, + } +} + +func TestParseSetAuthModes(t *testing.T) { + m := authModeManifest() + tests := []struct { + name string + setValues []string + want map[string]string + wantErr string + }{ + {name: "values ignored", setValues: []string{"analytics.sql-warehouse.id=wh"}, want: map[string]string{}}, + {name: "obo", setValues: []string{"analytics.sql-warehouse.authMode=obo"}, want: map[string]string{"sql_warehouse:sql-warehouse": "obo"}}, + {name: "both", setValues: []string{"analytics.sql-warehouse.authMode=both"}, want: map[string]string{"sql_warehouse:sql-warehouse": "both"}}, + {name: "sp without scope", setValues: []string{"jobs.job.authMode=sp"}, want: map[string]string{"job:job": "sp"}}, + {name: "invalid value", setValues: []string{"analytics.sql-warehouse.authMode=user"}, wantErr: "invalid auth mode"}, + {name: "unknown resource", setValues: []string{"analytics.nope.authMode=obo"}, wantErr: "no resource with key"}, + {name: "app only", setValues: []string{"analytics.secret.authMode=sp"}, wantErr: "always accessed by the app's service principal"}, + {name: "obo without scope", setValues: []string{"jobs.job.authMode=obo"}, wantErr: "cannot be accessed on behalf of the user"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseSetAuthModes(tt.setValues, m) + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } + + // parseSetValues must accept and skip the authMode pseudo-field. + rv, err := parseSetValues([]string{"analytics.sql-warehouse.authMode=obo", "analytics.sql-warehouse.id=wh"}, m) + require.NoError(t, err) + assert.Equal(t, map[string]string{"sql-warehouse.id": "wh"}, rv) +} + +func TestResolveAuthModes(t *testing.T) { + m := authModeManifest() + resources := m.CollectResources([]string{"analytics", "jobs"}) + + got, keptSP := resolveAuthModes(resources, nil, nil, "") + assert.Empty(t, got) + assert.Empty(t, keptSP) + + // The --auth-mode obo default skips resources without a scope and app-only resources. + got, keptSP = resolveAuthModes(resources, nil, nil, "obo") + assert.Equal(t, map[string]string{"sql_warehouse:sql-warehouse": "obo"}, got) + assert.Equal(t, []string{"secret", "job"}, keptSP) + + // An explicit --set wins and is not reported as kept on sp. + got, keptSP = resolveAuthModes(resources, map[string]string{"job:job": "sp", "sql_warehouse:sql-warehouse": "sp"}, nil, "obo") + assert.Empty(t, got) + assert.Equal(t, []string{"secret"}, keptSP) + + got, _ = resolveAuthModes(resources, map[string]string{"sql_warehouse:sql-warehouse": "both"}, map[string]string{"sql_warehouse:sql-warehouse": "obo"}, "") + assert.Equal(t, map[string]string{"sql_warehouse:sql-warehouse": "both"}, got) + + got, _ = resolveAuthModes(resources, nil, map[string]string{"sql_warehouse:sql-warehouse": "obo"}, "") + assert.Equal(t, map[string]string{"sql_warehouse:sql-warehouse": "obo"}, got) +} + +func TestResourceConfigured(t *testing.T) { + r := manifest.Resource{ResourceKey: "wh", Fields: map[string]manifest.ResourceField{"id": {}}} + assert.False(t, resourceConfigured(r, nil)) + assert.False(t, resourceConfigured(r, map[string]string{"other.id": "x"})) + assert.True(t, resourceConfigured(r, map[string]string{"wh.id": "x"})) +} + func TestPackageManagerValidation(t *testing.T) { tests := []struct { name string diff --git a/cmd/apps/validate.go b/cmd/apps/validate.go index e84a0221631..2bcef5237e0 100644 --- a/cmd/apps/validate.go +++ b/cmd/apps/validate.go @@ -36,6 +36,9 @@ Dependencies are installed when node_modules is absent. Scripts use the package manager's normal workspace behavior. For pnpm workspaces, define root scripts that run the checks for the packages you want to validate. +For AppKit projects (appkit.plugins.json), it first checks that resources accessed on +behalf of the user are not app-only and that their scopes are in user_api_scopes. + Examples: # Validate the current directory databricks apps validate @@ -75,6 +78,10 @@ func runValidate(cmd *cobra.Command) error { SkipTests: skipTests, } + if err := validation.ValidateAuthModes(projectPath); err != nil { + return err + } + // Get validator for project type validator := validation.GetProjectValidator(projectPath) if validator == nil { diff --git a/libs/apps/generator/generator.go b/libs/apps/generator/generator.go index 9d8b2f3d9f1..65267b81bb2 100644 --- a/libs/apps/generator/generator.go +++ b/libs/apps/generator/generator.go @@ -40,6 +40,30 @@ type Config struct { // ResourceValues maps resource value keys to values. // Keys use "resource_key.field_name" format (e.g., "sql-warehouse.id" -> "abc123"). ResourceValues map[string]string + // AuthModes maps a resource's AuthKey to its auth mode (AuthModeSP, AuthModeOBO, or AuthModeBoth). + // Resources without an entry use AuthModeSP. + AuthModes map[string]string +} + +// Auth modes describe how the app accesses a resource. +const ( + // AuthModeSP binds the resource to the app's service principal. + AuthModeSP = "sp" + // AuthModeOBO injects the resource id and accesses it on behalf of the user. + AuthModeOBO = "obo" + // AuthModeBoth binds the resource to the service principal and also grants the user scope. + AuthModeBoth = "both" +) + +// isOBOOnly returns true if the resource is accessed only on behalf of the user (not bound to the service principal). +func isOBOOnly(r manifest.Resource, cfg Config) bool { + return cfg.AuthModes[r.AuthKey()] == AuthModeOBO +} + +// needsScope returns true if the resource is accessed on behalf of the user. +func needsScope(r manifest.Resource, cfg Config) bool { + mode := cfg.AuthModes[r.AuthKey()] + return mode == AuthModeOBO || mode == AuthModeBoth } // hasResourceValues returns true if any value exists in cfg for the given resource. @@ -60,10 +84,12 @@ func GenerateBundleVariables(plugins []manifest.Plugin, cfg Config) string { for _, p := range plugins { for _, r := range p.Resources.Required { - lines = append(lines, generateVariableLines(r)...) + if !isOBOOnly(r, cfg) { + lines = append(lines, generateVariableLines(r)...) + } } for _, r := range p.Resources.Optional { - if hasResourceValues(r, cfg) { + if hasResourceValues(r, cfg) && !isOBOOnly(r, cfg) { lines = append(lines, generateVariableLines(r)...) } } @@ -101,6 +127,9 @@ func GenerateBundleResources(plugins []manifest.Plugin, cfg Config) string { for _, p := range plugins { // Required resources for _, r := range p.Resources.Required { + if isOBOOnly(r, cfg) { + continue + } resource := generateResourceYAML(r, 8) if resource != "" { blocks = append(blocks, resource) @@ -108,7 +137,7 @@ func GenerateBundleResources(plugins []manifest.Plugin, cfg Config) string { } // Optional resources (only if value provided) for _, r := range p.Resources.Optional { - if hasResourceValues(r, cfg) { + if hasResourceValues(r, cfg) && !isOBOOnly(r, cfg) { resource := generateResourceYAML(r, 8) if resource != "" { blocks = append(blocks, resource) @@ -128,10 +157,12 @@ func GenerateTargetVariables(plugins []manifest.Plugin, cfg Config) string { for _, p := range plugins { for _, r := range p.Resources.Required { - lines = append(lines, generateTargetVarLines(r, cfg)...) + if !isOBOOnly(r, cfg) { + lines = append(lines, generateTargetVarLines(r, cfg)...) + } } for _, r := range p.Resources.Optional { - if hasResourceValues(r, cfg) { + if hasResourceValues(r, cfg) && !isOBOOnly(r, cfg) { lines = append(lines, generateTargetVarLines(r, cfg)...) } } @@ -231,17 +262,18 @@ func GenerateDotEnvExample(plugins []manifest.Plugin) string { // GenerateAppEnv generates the env entries for app.yaml. // Each resource field with an Env mapping produces a YAML list entry with // name (the env var) and valueFrom (the resource key in databricks.yml resources). +// Resources accessed only on behalf of the user get a literal value instead of valueFrom. // Includes both required resources and optional resources that have values. func GenerateAppEnv(plugins []manifest.Plugin, cfg Config) string { var lines []string for _, p := range plugins { for _, r := range p.Resources.Required { - lines = append(lines, appEnvLines(r)...) + lines = append(lines, appEnvLines(r, cfg)...) } for _, r := range p.Resources.Optional { if hasResourceValues(r, cfg) { - lines = append(lines, appEnvLines(r)...) + lines = append(lines, appEnvLines(r, cfg)...) } } } @@ -250,8 +282,9 @@ func GenerateAppEnv(plugins []manifest.Plugin, cfg Config) string { } // appEnvLines returns app.yaml env entries for a resource. -// Each field with an Env produces "- name: \n valueFrom: ". -func appEnvLines(r manifest.Resource) []string { +// Each field with an Env produces "- name: \n valueFrom: ", +// or "- name: \n value: " when the resource is accessed only on behalf of the user. +func appEnvLines(r manifest.Resource, cfg Config) []string { var lines []string for _, fieldName := range r.FieldNames() { field := r.Fields[fieldName] @@ -261,6 +294,13 @@ func appEnvLines(r manifest.Resource) []string { if field.LocalOnly { continue } + if isOBOOnly(r, cfg) { + lines = append(lines, + " - name: "+field.Env, + " value: "+quoteYAMLValue(cfg.ResourceValues[r.Key()+"."+fieldName]), + ) + continue + } lines = append(lines, " - name: "+field.Env, " valueFrom: "+r.Key(), @@ -269,6 +309,43 @@ func appEnvLines(r manifest.Resource) []string { return lines } +// GenerateUserAPIScopes generates the user_api_scopes entries for databricks.yml. +// Output is indented with 8 spaces for insertion under "user_api_scopes:". +// Returns "" unless at least one resource is accessed on behalf of the user. The scopes are +// those of the resources accessed on behalf of the user, followed by the plugins' capability +// scopes, deduplicated in first-seen order. +func GenerateUserAPIScopes(plugins []manifest.Plugin, cfg Config) string { + var scopes []string + for _, p := range plugins { + for _, r := range p.Resources.Required { + if needsScope(r, cfg) { + scopes = append(scopes, r.Scope) + } + } + for _, r := range p.Resources.Optional { + if hasResourceValues(r, cfg) && needsScope(r, cfg) { + scopes = append(scopes, r.Scope) + } + } + } + if len(scopes) == 0 { + return "" + } + for _, p := range plugins { + scopes = append(scopes, p.Scopes...) + } + + var lines []string + seen := make(map[string]bool) + for _, s := range scopes { + if !seen[s] { + seen[s] = true + lines = append(lines, " - "+s) + } + } + return strings.Join(lines, "\n") +} + // appResourceSpec defines how a manifest resource type maps to DABs AppResource YAML. type appResourceSpec struct { yamlKey string // DABs YAML key under the resource entry (e.g., "sql_warehouse", "uc_securable") diff --git a/libs/apps/generator/generator_test.go b/libs/apps/generator/generator_test.go index ca8b19b82ac..286b7639d81 100644 --- a/libs/apps/generator/generator_test.go +++ b/libs/apps/generator/generator_test.go @@ -1147,3 +1147,87 @@ func TestDedupeResourcesRequiredWinsOverOptional(t *testing.T) { res := generator.GenerateBundleResources(deduped, generator.Config{}) assert.Equal(t, 1, strings.Count(res, "- name: postgres")) } + +func authModePlugins() []manifest.Plugin { + return []manifest.Plugin{ + { + Name: "analytics", + Scopes: []string{"ai-gateway", "sql"}, + Resources: manifest.Resources{ + Required: []manifest.Resource{ + { + Type: "sql_warehouse", ResourceKey: "sql-warehouse", Scope: "sql", + Fields: map[string]manifest.ResourceField{"id": {Env: "DATABRICKS_WAREHOUSE_ID"}}, + }, + { + Type: "genie_space", ResourceKey: "genie-space", Scope: "genie", + Fields: map[string]manifest.ResourceField{"id": {Env: "GENIE_SPACE_ID"}}, + }, + }, + }, + }, + } +} + +func authModeConfig(modes map[string]string) generator.Config { + return generator.Config{ + ResourceValues: map[string]string{"sql-warehouse.id": "wh1", "genie-space.id": "g1", "genie-space.name": "space"}, + AuthModes: modes, + } +} + +func TestAuthModeSPIsDefault(t *testing.T) { + plugins := authModePlugins() + sp := authModeConfig(map[string]string{"sql_warehouse:sql-warehouse": generator.AuthModeSP}) + unset := authModeConfig(nil) + + assert.Equal(t, generator.GenerateBundleResources(plugins, unset), generator.GenerateBundleResources(plugins, sp)) + assert.Equal(t, generator.GenerateAppEnv(plugins, unset), generator.GenerateAppEnv(plugins, sp)) + assert.Empty(t, generator.GenerateUserAPIScopes(plugins, unset)) + assert.Empty(t, generator.GenerateUserAPIScopes(plugins, sp)) +} + +func TestAuthModeOBO(t *testing.T) { + plugins := authModePlugins() + cfg := authModeConfig(map[string]string{"sql_warehouse:sql-warehouse": generator.AuthModeOBO}) + + assert.NotContains(t, generator.GenerateBundleResources(plugins, cfg), "sql-warehouse") + assert.NotContains(t, generator.GenerateBundleVariables(plugins, cfg), "sql_warehouse_id") + assert.NotContains(t, generator.GenerateTargetVariables(plugins, cfg), "sql_warehouse_id") + assert.Equal(t, " - name: DATABRICKS_WAREHOUSE_ID\n value: wh1\n - name: GENIE_SPACE_ID\n valueFrom: genie-space", + generator.GenerateAppEnv(plugins, cfg)) + assert.Equal(t, " - sql\n - ai-gateway", generator.GenerateUserAPIScopes(plugins, cfg)) +} + +func TestAuthModeBoth(t *testing.T) { + plugins := authModePlugins() + cfg := authModeConfig(map[string]string{"sql_warehouse:sql-warehouse": generator.AuthModeBoth, "genie_space:genie-space": generator.AuthModeOBO}) + + assert.Contains(t, generator.GenerateBundleResources(plugins, cfg), "- name: sql-warehouse") + assert.NotContains(t, generator.GenerateBundleResources(plugins, cfg), "genie-space") + assert.Contains(t, generator.GenerateAppEnv(plugins, cfg), "valueFrom: sql-warehouse") + assert.Equal(t, " - sql\n - genie\n - ai-gateway", generator.GenerateUserAPIScopes(plugins, cfg)) +} + +func TestAuthModeKeyDoesNotCollideAcrossTypes(t *testing.T) { + // Two resources share the resourceKey "data" but differ in type, so they must + // not share an auth mode: making the warehouse OBO must leave the app-only + // secret bound to the service principal and must not emit its empty scope. + plugins := []manifest.Plugin{ + {Name: "a", Resources: manifest.Resources{Required: []manifest.Resource{ + {Type: "sql_warehouse", ResourceKey: "data", Scope: "sql", Fields: map[string]manifest.ResourceField{"id": {Env: "WAREHOUSE_ID"}}}, + }}}, + {Name: "b", Resources: manifest.Resources{Required: []manifest.Resource{ + {Type: "secret", ResourceKey: "data", AppOnly: true, Fields: map[string]manifest.ResourceField{"token": {Env: "TOKEN"}}}, + }}}, + } + cfg := generator.Config{ + ResourceValues: map[string]string{"data.id": "wh1", "data.token": "s"}, + AuthModes: map[string]string{"sql_warehouse:data": generator.AuthModeOBO}, + } + + env := generator.GenerateAppEnv(plugins, cfg) + assert.Contains(t, env, " - name: WAREHOUSE_ID\n value: wh1") // warehouse: OBO literal + assert.Contains(t, env, " - name: TOKEN\n valueFrom: data") // secret: still SP-bound + assert.Equal(t, " - sql", generator.GenerateUserAPIScopes(plugins, cfg)) +} diff --git a/libs/apps/manifest/manifest.go b/libs/apps/manifest/manifest.go index b74f68bc699..1539b76fd64 100644 --- a/libs/apps/manifest/manifest.go +++ b/libs/apps/manifest/manifest.go @@ -35,6 +35,13 @@ type Resource struct { Permission string `json:"permission"` // e.g., "CAN_USE" Fields map[string]ResourceField `json:"fields"` // field definitions with env var mappings + // Scope is the user_api_scope needed to use this resource on behalf of the user. + // Empty when the resource type cannot be used on behalf of the user. + Scope string `json:"scope,omitempty"` + + // AppOnly resources (e.g., secret, database) are always bound to the service principal. + AppOnly bool `json:"appOnly,omitempty"` + // PluginName is the machine name of the plugin (e.g., "lakebase"). // Set during resource collection. Not part of the JSON manifest. PluginName string `json:"-"` @@ -49,6 +56,14 @@ func (r Resource) Key() string { return r.ResourceKey } +// AuthKey identifies a resource for auth-mode lookups using the same type+key +// identity as resource de-duplication (CollectResources, generator.DedupeResources). +// Two resources that collapse to one in the generated bundle therefore share an +// auth mode, while two resources with the same key but different types do not collide. +func (r Resource) AuthKey() string { + return r.Type + ":" + r.Key() +} + // VarPrefix returns the variable name prefix derived from the resource key. // Hyphens are replaced with underscores for YAML variable name compatibility. func (r Resource) VarPrefix() string { @@ -81,6 +96,9 @@ type Plugin struct { Resources Resources `json:"resources"` OnSetupMessage string `json:"onSetupMessage"` + // Scopes are user_api_scopes the plugin needs that are not tied to a resource (e.g., "ai-gateway"). + Scopes []string `json:"scopes,omitempty"` + // Stability is one of "beta", "ga", or empty. // Stored as a plain string so unknown future values round-trip unchanged. // See https://github.com/databricks/appkit/pull/264. diff --git a/libs/apps/prompt/prompt.go b/libs/apps/prompt/prompt.go index 8e49e9a94d1..6bbbaaa2d28 100644 --- a/libs/apps/prompt/prompt.go +++ b/libs/apps/prompt/prompt.go @@ -118,6 +118,7 @@ type CreateProjectConfig struct { Dependencies map[string]string // e.g., {"sql_warehouse_id": "abc123"} Deploy bool // Whether to deploy the app after creation RunMode RunMode // How to run the app after creation + AuthModes map[string]string // resource AuthKey -> auth mode chosen in the prompt } // App name constraints. diff --git a/libs/apps/validation/authmode.go b/libs/apps/validation/authmode.go new file mode 100644 index 00000000000..9c723310007 --- /dev/null +++ b/libs/apps/validation/authmode.go @@ -0,0 +1,129 @@ +package validation + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "slices" + + "github.com/databricks/cli/libs/apps/manifest" + "go.yaml.in/yaml/v3" +) + +type appYAML struct { + Env []struct { + Name string `yaml:"name"` + Value *string `yaml:"value"` + } `yaml:"env"` +} + +type bundleYAML struct { + Include []string `yaml:"include"` + Resources struct { + Apps map[string]struct { + UserAPIScopes []string `yaml:"user_api_scopes"` + Resources []struct { + Name string `yaml:"name"` + } `yaml:"resources"` + } `yaml:"apps"` + } `yaml:"resources"` +} + +// ValidateAuthModes checks that resources accessed on behalf of the user are consistent +// with the manifest. A resource is treated as accessed on behalf of the user when app.yaml +// sets one of its env vars to a literal value and databricks.yml does not bind it. +// Such a resource must not be app-only, and its scope must be in user_api_scopes. +// Projects without appkit.plugins.json, app.yaml, or databricks.yml are not checked. +func ValidateAuthModes(projectPath string) error { + if !manifest.HasManifest(projectPath) { + return nil + } + m, err := manifest.Load(projectPath) + if err != nil { + return err + } + + var app appYAML + if ok, err := readYAML(filepath.Join(projectPath, "app.yaml"), &app); !ok { + return err + } + var root bundleYAML + if ok, err := readYAML(filepath.Join(projectPath, "databricks.yml"), &root); !ok { + return err + } + + // Follow the root-level include globs so split-config bundles, where the app + // resource and user_api_scopes live in separate files, are seen too. + bundles := []bundleYAML{root} + for _, pattern := range root.Include { + matches, err := filepath.Glob(filepath.Join(projectPath, pattern)) + if err != nil { + return fmt.Errorf("invalid include pattern %q: %w", pattern, err) + } + for _, path := range matches { + var inc bundleYAML + ok, err := readYAML(path, &inc) + if err != nil { + return err + } + if ok { + bundles = append(bundles, inc) + } + } + } + + bound := make(map[string]bool) + var scopes []string + for _, b := range bundles { + for _, a := range b.Resources.Apps { + for _, r := range a.Resources { + bound[r.Name] = true + } + scopes = append(scopes, a.UserAPIScopes...) + } + } + + byEnv := make(map[string]manifest.Resource) + for _, p := range m.GetPlugins() { + for _, r := range append(p.Resources.Required, p.Resources.Optional...) { + for _, f := range r.Fields { + if f.Env != "" { + byEnv[f.Env] = r + } + } + } + } + + var errs []error + checked := make(map[string]bool) + for _, e := range app.Env { + r, ok := byEnv[e.Name] + if !ok || e.Value == nil || bound[r.Key()] || checked[r.Key()] { + continue + } + checked[r.Key()] = true + switch { + case r.AppOnly: + errs = append(errs, fmt.Errorf("resource %q must be bound to the app's service principal in databricks.yml, but app.yaml sets %s to a literal value", r.Key(), e.Name)) + case r.Scope != "" && !slices.Contains(scopes, r.Scope): + errs = append(errs, fmt.Errorf("resource %q is accessed on behalf of the user but its scope %q is missing from user_api_scopes in databricks.yml", r.Key(), r.Scope)) + } + } + return errors.Join(errs...) +} + +// readYAML parses path into v. Returns false with a nil error if the file does not exist. +func readYAML(path string, v any) (bool, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + if err := yaml.Unmarshal(data, v); err != nil { + return false, fmt.Errorf("parse %s: %w", filepath.Base(path), err) + } + return true, nil +} diff --git a/libs/apps/validation/authmode_test.go b/libs/apps/validation/authmode_test.go new file mode 100644 index 00000000000..309de7dd9d4 --- /dev/null +++ b/libs/apps/validation/authmode_test.go @@ -0,0 +1,102 @@ +package validation + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const authModeManifest = `{ + "plugins": { + "analytics": { + "resources": { + "required": [ + {"type": "sql_warehouse", "resourceKey": "sql-warehouse", "scope": "sql", "fields": {"id": {"env": "WAREHOUSE_ID"}}}, + {"type": "secret", "resourceKey": "secret", "appOnly": true, "fields": {"scope": {"env": "SECRET_SCOPE"}, "key": {"env": "SECRET_KEY"}}} + ] + } + } + } +}` + +func writeAuthModeProject(t *testing.T, appYAML, bundleYAML string) string { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "appkit.plugins.json"), []byte(authModeManifest), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "app.yaml"), []byte(appYAML), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks.yml"), []byte(bundleYAML), 0o644)) + return dir +} + +func TestValidateAuthModes(t *testing.T) { + const boundSecret = ` +resources: + apps: + app: + resources: + - name: secret +` + tests := []struct { + name string + appYAML string + bundle string + wantErr string + }{ + { + name: "sp", + appYAML: "env:\n - name: WAREHOUSE_ID\n valueFrom: sql-warehouse\n", + bundle: "resources:\n apps:\n app:\n resources:\n - name: sql-warehouse\n", + }, + { + name: "obo with scope", + appYAML: "env:\n - name: WAREHOUSE_ID\n value: wh1\n", + bundle: "resources:\n apps:\n app:\n user_api_scopes:\n - sql\n", + }, + { + name: "bound with literal value is not obo", + appYAML: "env:\n - name: WAREHOUSE_ID\n value: wh1\n", + bundle: "resources:\n apps:\n app:\n resources:\n - name: sql-warehouse\n", + }, + { + name: "obo missing scope", + appYAML: "env:\n - name: WAREHOUSE_ID\n value: wh1\n", + bundle: boundSecret, + wantErr: `resource "sql-warehouse" is accessed on behalf of the user but its scope "sql" is missing from user_api_scopes in databricks.yml`, + }, + { + name: "app only not bound", + appYAML: "env:\n - name: SECRET_SCOPE\n value: s\n - name: SECRET_KEY\n value: k\n", + bundle: "resources:\n apps:\n app:\n user_api_scopes:\n - sql\n", + wantErr: `resource "secret" must be bound to the app's service principal in databricks.yml, but app.yaml sets SECRET_SCOPE to a literal value`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateAuthModes(writeAuthModeProject(t, tt.appYAML, tt.bundle)) + if tt.wantErr == "" { + assert.NoError(t, err) + return + } + assert.EqualError(t, err, tt.wantErr) + }) + } +} + +func TestValidateAuthModesWithoutManifest(t *testing.T) { + assert.NoError(t, ValidateAuthModes(t.TempDir())) +} + +func TestValidateAuthModesFollowsIncludes(t *testing.T) { + // An OBO resource whose user_api_scopes live in an included file must not be + // flagged as missing its scope. + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "appkit.plugins.json"), []byte(authModeManifest), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "app.yaml"), []byte("env:\n - name: WAREHOUSE_ID\n value: wh1\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks.yml"), []byte("include:\n - resources/*.yml\n"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "resources"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "resources", "app.yml"), []byte("resources:\n apps:\n app:\n user_api_scopes:\n - sql\n"), 0o644)) + + assert.NoError(t, ValidateAuthModes(dir)) +}