direct: auto-generate RemapState copiers from resource types

bundle/direct/dresources/README.md

@@ -73,35 +73,16 @@ on field behaviour might result in recreate. See dstate/migrate.go on how to han
 
 Declaring a field under `hashed_fields` in `resources.yml` makes the engine persist only a `sha256:<hex>` content hash of it in state (via `CompactState`) rather than the full contents, for large equality-only fields that are never read back from state (e.g. `dashboards.serialized_dashboard`).
 
-## RemapState is a dumb copy; DoRead owns all remapping
-
-`RemapState` converts `RemoteType` to `StateType` only because `StateType` is typically a
-subset of `RemoteType`. It must be a field-by-field copy (or no-op), never a place for
-logic. In particular, do not remap a differently-named field there (e.g. `state.x = remote.status.x`).
-Any remapping the API requires belongs in `DoRead`: add `x` directly to `RemoteType` and
-populate it from `status.x` inside `DoRead`.
-
-This is not a style preference — the plan classifier depends on it. Because `RemapState` is a
-dumb copy, a field absent from `RemoteType` is always nil/zero in the remapped remote state the
-planner compares against: there is nothing to copy it from. So "path present in `StateType` but
-absent from `RemoteType`" reliably means "field the API accepts on write but never echoes on
-read," and its remote value is guaranteed meaningless (see `isFieldMissingInRemote` /
-`missing_in_remote` in `bundle_plan.go`).
-
-A `RemapState` that instead synthesizes `state.x = remote.status.x` breaks this: `x` is absent
-from `RemoteType`, so the classifier treats it as input-only and skips it (`missing_in_remote`),
-yet the field actually has a real remote value under `status.x`. Genuine remote drift is then
-silently suppressed. The fix is to make the field genuinely present in `RemoteType`, not to
-smuggle it in via `RemapState`.
-
-So when a field comes back under a different path than `StateType` uses:
-
-1. Add the field to `RemoteType` (the struct returned by `DoRead`).
-2. Populate it in `DoRead` by mapping from whatever the API returns under the other name.
-3. Keep `RemapState` a trivial subset copy.
-
-This makes the field present in `InputType`, `StateType`, and `RemoteType`, so it participates
-in normal drift detection and is no longer subject to the `missing_in_remote` suppression.
+## RemoteType should be a superset of StateType; DoRead owns all remapping
+
+`RemoteType` should be a superset of `StateType`, so the framework can derive the state from
+the remote automatically: `buildCopiers` (in `all.go`) compiles a copier per resource at package
+init, matching fields by JSON name, filtering `ForceSendFields`, and applying only lossless
+conversions. New resources should not define `RemapState`.
+
+Do all remapping in `DoRead`. When the API returns a field under a different path than
+`StateType` uses, add the field directly to `RemoteType` and populate it in `DoRead` from
+whatever the API returns, so it participates in normal drift detection.
 
 ## OverrideChangeDesc
 

bundle/direct/dresources/adapter.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/databricks/cli/bundle/deployplan"
 	"github.com/databricks/cli/libs/calladapt"
+	"github.com/databricks/cli/libs/structs/structcopy"
 	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/databricks/cli/libs/structs/structvar"
 	"github.com/databricks/databricks-sdk-go"
@@ -130,6 +131,7 @@ type Adapter struct {
 	doCreate     *calladapt.BoundCaller
 
 	// Optional:
+	copier             *structcopy.Copier
 	doDelete           *calladapt.BoundCaller
 	prepareInputConfig *calladapt.BoundCaller
 	isEmptyState       *calladapt.BoundCaller
@@ -169,6 +171,7 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC
 	adapter := &Adapter{
 		prepareState:            nil,
 		remapState:              nil,
+		copier:                  nil,
 		doRefresh:               nil,
 		doDelete:                nil,
 		doCreate:                nil,
@@ -233,11 +236,15 @@ func (a *Adapter) initMethods(resource any) error {
 		return err
 	}
 
-	// RemapState is optional when remote type already matches state type.
+	// RemapState is optional when remote type already matches state type, or when an
+	// auto-generated copier exists for this resource (see buildCopiers).
 	a.remapState, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "RemapState")
 	if err != nil {
 		return err
 	}
+	if a.remapState == nil {
+		a.copier = copiers[reflect.TypeOf(resource)]
+	}
 
 	a.doRefresh, err = prepareCallRequired(resource, "DoRead")
 	if err != nil {
@@ -363,15 +370,16 @@ func (a *Adapter) validate() error {
 		validations = append(validations, "DoDelete state", a.doDelete.InTypes[2], stateType)
 	}
 
-	// If RemapState is implemented, validate its signature.
-	// Otherwise require remote type to equal state type so remapping isn't needed.
+	// If RemapState is implemented, validate its signature. Otherwise the remote type
+	// must equal the state type (no remapping needed) or an auto-generated copier must
+	// exist for this resource (built and validated in buildCopiers).
 	if a.remapState != nil {
 		validations = append(
 			validations,
 			"RemapState input", a.remapState.InTypes[0], remoteType,
 			"RemapState return", a.remapState.OutTypes[0], stateType,
 		)
-	} else if remoteType != stateType {
+	} else if remoteType != stateType && a.copier == nil {
 		return fmt.Errorf("RemapState method not found and remote type %v must match state type %v", remoteType, stateType)
 	}
 
@@ -531,15 +539,19 @@ func (a *Adapter) PrepareState(input any) (any, error) {
 }
 
 func (a *Adapter) RemapState(remoteState any) (any, error) {
-	if a.remapState == nil {
-		return remoteState, nil
+	if a.remapState != nil {
+		outs, err := a.remapState.Call(remoteState)
+		if err != nil {
+			return nil, err
+		}
+		return outs[0], nil
 	}
-
-	outs, err := a.remapState.Call(remoteState)
-	if err != nil {
-		return nil, err
+	if a.copier != nil {
+		return a.copier.Copy(remoteState), nil
 	}
-	return outs[0], nil
+	// No custom method and no copier: validate() only allows this when
+	// remoteType == stateType, so the remote is already the state type.
+	return remoteState, nil
 }
 
 func (a *Adapter) DoRead(ctx context.Context, id string) (any, error) {

bundle/direct/dresources/all.go

@@ -2,7 +2,12 @@ package dresources
 
 import (
 	"fmt"
+	"reflect"
+	"slices"
+	"strings"
 
+	"github.com/databricks/cli/libs/calladapt"
+	"github.com/databricks/cli/libs/structs/structcopy"
 	"github.com/databricks/databricks-sdk-go"
 )
 
@@ -81,6 +86,68 @@ var SupportedResources = map[string]any{
 	"internal_immutable_snapshots": (*ResourceSnapshot)(nil),
 }
 
+// copiers holds an auto-generated RemapState copier for every resource whose
+// remote type differs from its state type and that does not supply a custom RemapState.
+// It is built once, at package initialization, from SupportedResources — so a resource
+// whose types cannot be safely copied fails at load, not at deploy time, and no separate
+// test can forget to cover it. Resources that need real remapping logic keep a custom
+// RemapState method and are skipped here.
+var copiers = buildCopiers()
+
+func buildCopiers() map[reflect.Type]*structcopy.Copier {
+	iface := reflect.TypeFor[IResource]()
+	out := make(map[reflect.Type]*structcopy.Copier)
+	seen := make(map[reflect.Type]bool)
+	var errs []string
+
+	for resourceType, resource := range SupportedResources {
+		implType := reflect.TypeOf(resource)
+		if seen[implType] {
+			continue // same implementation registered under several keys (permissions, grants)
+		}
+		seen[implType] = true
+
+		remap, err := calladapt.PrepareCall(resource, iface, "RemapState")
+		if err != nil {
+			errs = append(errs, fmt.Sprintf("%s: RemapState: %v", resourceType, err))
+			continue
+		}
+		if remap != nil {
+			continue // custom override
+		}
+
+		prepareState, err := calladapt.PrepareCall(resource, iface, "PrepareState")
+		if err != nil || prepareState == nil {
+			errs = append(errs, fmt.Sprintf("%s: PrepareState: %v", resourceType, err))
+			continue
+		}
+		doRead, err := calladapt.PrepareCall(resource, iface, "DoRead")
+		if err != nil || doRead == nil {
+			errs = append(errs, fmt.Sprintf("%s: DoRead: %v", resourceType, err))
+			continue
+		}
+
+		stateType := prepareState.OutTypes[0]
+		remoteType := doRead.OutTypes[0]
+		if remoteType == stateType {
+			continue // identity: the adapter returns the remote unchanged, no copier needed
+		}
+
+		copier, err := structcopy.Compile(remoteType, stateType)
+		if err != nil {
+			errs = append(errs, fmt.Sprintf("%s: %v (implement RemapState for this resource)", resourceType, err))
+			continue
+		}
+		out[implType] = copier
+	}
+
+	if len(errs) > 0 {
+		slices.Sort(errs)
+		panic("dresources: cannot build RemapState copiers:\n" + strings.Join(errs, "\n"))
+	}
+	return out
+}
+
 func InitAll(client *databricks.WorkspaceClient) (map[string]*Adapter, error) {
 	result := make(map[string]*Adapter)
 	for resourceType, resource := range SupportedResources {

bundle/direct/dresources/app.go

@@ -73,18 +73,8 @@ func (*ResourceApp) PrepareState(input *resources.App) *AppState {
 	return s
 }
 
-// RemapState maps the remote AppRemote to AppState for diff comparison.
 // DoRead populates config, git_source, and source_code_path from the active
 // deployment when one exists, enabling drift detection for out-of-band redeploys.
-// Started is derived from compute status so the planner can detect start/stop changes.
-func (*ResourceApp) RemapState(remote *AppRemote) *AppState {
-	return &AppState{
-		App:       remote.App,
-		Config:    remote.Config,
-		Lifecycle: remote.Lifecycle,
-	}
-}
-
 func (r *ResourceApp) DoRead(ctx context.Context, id string) (*AppRemote, error) {
 	app, err := r.client.Apps.GetByName(ctx, id)
 	if err != nil {

bundle/direct/dresources/catalog.go

@@ -22,22 +22,6 @@ func (*ResourceCatalog) PrepareState(input *resources.Catalog) *catalog.CreateCa
 	return &input.CreateCatalog
 }
 
-func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *catalog.CreateCatalog {
-	return &catalog.CreateCatalog{
-		Comment:                   info.Comment,
-		ConnectionName:            info.ConnectionName,
-		CustomMaxRetentionHours:   info.CustomMaxRetentionHours,
-		ManagedEncryptionSettings: info.ManagedEncryptionSettings,
-		Name:                      info.Name,
-		Options:                   info.Options,
-		Properties:                info.Properties,
-		ProviderName:              info.ProviderName,
-		ShareName:                 info.ShareName,
-		StorageRoot:               info.StorageRoot,
-		ForceSendFields:           utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields),
-	}
-}
-
 func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*catalog.CatalogInfo, error) {
 	return r.client.Catalogs.GetByName(ctx, id)
 }

bundle/direct/dresources/cluster.go

@@ -102,54 +102,6 @@ func (r *ResourceCluster) PrepareState(input *resources.Cluster) *ClusterState {
 	return s
 }
 
-// RemapState maps the remote ClusterRemote to ClusterState for diff comparison.
-// Started is derived from cluster state so the planner can detect start/stop changes.
-func (r *ResourceCluster) RemapState(input *ClusterRemote) *ClusterState {
-	spec := &ClusterState{
-		ClusterSpec: compute.ClusterSpec{
-			ApplyPolicyDefaultValues:   input.ApplyPolicyDefaultValues,
-			Autoscale:                  input.Autoscale,
-			AutoterminationMinutes:     input.AutoterminationMinutes,
-			AwsAttributes:              input.AwsAttributes,
-			AzureAttributes:            input.AzureAttributes,
-			ClusterLogConf:             input.ClusterLogConf,
-			ClusterName:                input.ClusterName,
-			CustomTags:                 input.CustomTags,
-			DataSecurityMode:           input.DataSecurityMode,
-			DependencyMode:             input.DependencyMode,
-			DockerImage:                input.DockerImage,
-			DriverInstancePoolId:       input.DriverInstancePoolId,
-			DriverNodeTypeId:           input.DriverNodeTypeId,
-			DriverNodeTypeFlexibility:  input.DriverNodeTypeFlexibility,
-			EnableElasticDisk:          input.EnableElasticDisk,
-			EnableLocalDiskEncryption:  input.EnableLocalDiskEncryption,
-			GcpAttributes:              input.GcpAttributes,
-			InitScripts:                input.InitScripts,
-			InstancePoolId:             input.InstancePoolId,
-			IsSingleNode:               input.IsSingleNode,
-			Kind:                       input.Kind,
-			NodeTypeId:                 input.NodeTypeId,
-			NumWorkers:                 input.NumWorkers,
-			PolicyId:                   input.PolicyId,
-			RemoteDiskThroughput:       input.RemoteDiskThroughput,
-			RuntimeEngine:              input.RuntimeEngine,
-			SingleUserName:             input.SingleUserName,
-			SparkConf:                  input.SparkConf,
-			SparkEnvVars:               input.SparkEnvVars,
-			SparkVersion:               input.SparkVersion,
-			SshPublicKeys:              input.SshPublicKeys,
-			TotalInitialRemoteDiskSize: input.TotalInitialRemoteDiskSize,
-			UseMlRuntime:               input.UseMlRuntime,
-			WorkloadType:               input.WorkloadType,
-			WorkerNodeTypeFlexibility:  input.WorkerNodeTypeFlexibility,
-			ForceSendFields:            utils.FilterFields[compute.ClusterSpec](input.ForceSendFields),
-		},
-		Lifecycle: input.Lifecycle,
-		Libraries: input.Libraries,
-	}
-	return spec
-}
-
 func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*ClusterRemote, error) {
 	var details *compute.ClusterDetails
 	var libraries []compute.Library

bundle/direct/dresources/cluster_policy.go

@@ -30,19 +30,6 @@ func (*ResourceClusterPolicy) PrepareState(input *resources.ClusterPolicy) *comp
 }
 
 // RemapState copies the config fields shared by Policy and CreatePolicy;
-// output-only fields (policy_id, created_at_timestamp, creator_user_name, is_default) are not in the state.
-func (*ResourceClusterPolicy) RemapState(remote *compute.Policy) *compute.CreatePolicy {
-	return &compute.CreatePolicy{
-		Definition:                      remote.Definition,
-		Description:                     remote.Description,
-		Libraries:                       remote.Libraries,
-		MaxClustersPerUser:              remote.MaxClustersPerUser,
-		Name:                            remote.Name,
-		PolicyFamilyDefinitionOverrides: remote.PolicyFamilyDefinitionOverrides,
-		PolicyFamilyId:                  remote.PolicyFamilyId,
-		ForceSendFields:                 utils.FilterFields[compute.CreatePolicy](remote.ForceSendFields),
-	}
-}
 
 func (r *ResourceClusterPolicy) DoRead(ctx context.Context, id string) (*compute.Policy, error) {
 	return r.client.ClusterPolicies.GetByPolicyId(ctx, id)