diff --git a/api/v1beta1/clusterpromotion_types.go b/api/v1beta1/clusterpromotion_types.go index bab8f200..7da41c23 100644 --- a/api/v1beta1/clusterpromotion_types.go +++ b/api/v1beta1/clusterpromotion_types.go @@ -170,6 +170,13 @@ type ProfileSpec struct { // +optional ValidateHealths []libsveltosv1beta1.ValidateHealth `json:"validateHealths,omitempty"` + // HealthMonitoring, when enabled, continuously monitors the health of the resources + // this profile deploys in each matching cluster. Unlike ValidateHealths, which are + // evaluated once during deployment, these checks run for as long as the profile + // is deployed. Results are available as HealthCheckReports. + // +optional + HealthMonitoring *HealthMonitoring `json:"healthMonitoring,omitempty"` + // Define additional Kustomize inline Patches applied for all resources on this profile // Within the Patch Spec you can use templating // +listType=atomic diff --git a/api/v1beta1/spec.go b/api/v1beta1/spec.go index 2372fc0b..7bcd78ff 100644 --- a/api/v1beta1/spec.go +++ b/api/v1beta1/spec.go @@ -958,6 +958,55 @@ type Clusters struct { Clusters []corev1.ObjectReference `json:"clusters,omitempty"` } +const ( + // HealthMonitoringClusterSummaryUIDLabel is set, with the UID of the ClusterSummary as value, on the + // HealthCheck and ClusterHealthCheck instances created for a ClusterSummary when health monitoring is enabled. + HealthMonitoringClusterSummaryUIDLabel = "projectsveltos.io/health-monitoring-clustersummary-uid" + + // HealthMonitoringClusterSummaryNamespaceAnnotation and HealthMonitoringClusterSummaryNameAnnotation + // are set on the same instances to identify the ClusterSummary they have been created for. + HealthMonitoringClusterSummaryNamespaceAnnotation = "projectsveltos.io/health-monitoring-clustersummary-namespace" + HealthMonitoringClusterSummaryNameAnnotation = "projectsveltos.io/health-monitoring-clustersummary-name" +) + +// HealthMonitoring configures the continuous health monitoring of the resources a profile deploys. +// Both ClusterProfile and Profile can enable it with the default checks. Referencing additional +// HealthChecks (healthChecks) is reserved to ClusterProfile: see the field. +// +// +kubebuilder:validation:XValidation:rule="self.enabled || !has(self.healthChecks) || size(self.healthChecks) == 0",message="healthChecks can only be set when enabled is true" +// +kubebuilder:validation:XValidation:rule="!self.enabled || !has(self.defaultChecks) || self.defaultChecks || (has(self.healthChecks) && size(self.healthChecks) > 0)",message="defaultChecks cannot be false when healthChecks is empty: there would be nothing to monitor" +// +//nolint:lll // kubebuilder marker +type HealthMonitoring struct { + // Enabled turns health monitoring on for the resources deployed by this profile. + // When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + // matching cluster once the profile is provisioned there, and removes them + // when monitoring is disabled or the cluster does not match anymore. + Enabled bool `json:"enabled"` + + // DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + // Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + // Set it to false to monitor only the HealthChecks listed in HealthChecks. + // +kubebuilder:default:=true + // +optional + DefaultChecks *bool `json:"defaultChecks,omitempty"` + + // HealthChecks is a list of names of HealthCheck instances to monitor in addition + // to the default checks. Those are deployed as they are: selecting only the + // resources deployed by this profile is up to the HealthCheck. + // + // Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + // default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + // report it produces is shared by every profile that references it on a given cluster, with + // no filtering by namespace. Allowing a Profile to reference one would let a tenant name + // a HealthCheck it does not own, and learn about resources outside its namespace through + // the shared report. The default checks are safe for a Profile: they select only the + // resources that profile deployed. + // +listType=set + // +optional + HealthChecks []string `json:"healthChecks,omitempty"` +} + type Spec struct { // ClusterSelector identifies clusters to associate to. // +optional @@ -1110,6 +1159,13 @@ type Spec struct { // +optional ValidateHealths []libsveltosv1beta1.ValidateHealth `json:"validateHealths,omitempty"` + // HealthMonitoring, when enabled, continuously monitors the health of the resources + // this profile deploys in each matching cluster. Unlike ValidateHealths, which are + // evaluated once during deployment, these checks run for as long as the profile + // is deployed. Results are available as HealthCheckReports. + // +optional + HealthMonitoring *HealthMonitoring `json:"healthMonitoring,omitempty"` + // Define additional Kustomize inline Patches applied for all resources on this profile // Within the Patch Spec you can use templating // +listType=atomic diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 0046326f..35a7f67e 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -854,6 +854,31 @@ func (in *HandOverError) DeepCopy() *HandOverError { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HealthMonitoring) DeepCopyInto(out *HealthMonitoring) { + *out = *in + if in.DefaultChecks != nil { + in, out := &in.DefaultChecks, &out.DefaultChecks + *out = new(bool) + **out = **in + } + if in.HealthChecks != nil { + in, out := &in.HealthChecks, &out.HealthChecks + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HealthMonitoring. +func (in *HealthMonitoring) DeepCopy() *HealthMonitoring { + if in == nil { + return nil + } + out := new(HealthMonitoring) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HelmChart) DeepCopyInto(out *HelmChart) { *out = *in @@ -1305,6 +1330,11 @@ func (in *ProfileSpec) DeepCopyInto(out *ProfileSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.HealthMonitoring != nil { + in, out := &in.HealthMonitoring, &out.HealthMonitoring + *out = new(HealthMonitoring) + (*in).DeepCopyInto(*out) + } if in.Patches != nil { in, out := &in.Patches, &out.Patches *out = make([]apiv1beta1.Patch, len(*in)) @@ -1532,6 +1562,11 @@ func (in *Spec) DeepCopyInto(out *Spec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.HealthMonitoring != nil { + in, out := &in.HealthMonitoring, &out.HealthMonitoring + *out = new(HealthMonitoring) + (*in).DeepCopyInto(*out) + } if in.Patches != nil { in, out := &in.Patches, &out.Patches *out = make([]apiv1beta1.Patch, len(*in)) diff --git a/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml b/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml index 43d9a545..38228a56 100644 --- a/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml +++ b/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml @@ -237,6 +237,55 @@ spec: `ExtraLabels`, the value from `ExtraLabels` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is empty: + there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml b/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml index 81720985..7c2d0d5f 100644 --- a/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml +++ b/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml @@ -136,6 +136,55 @@ spec: type: object type: array x-kubernetes-list-type: atomic + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is + empty: there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml index b5bcdebb..a17b9724 100644 --- a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml +++ b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml @@ -273,6 +273,55 @@ spec: `ExtraLabels`, the value from `ExtraLabels` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is + empty: there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/config/crd/bases/config.projectsveltos.io_profiles.yaml b/config/crd/bases/config.projectsveltos.io_profiles.yaml index e33f95e2..f357ecc8 100644 --- a/config/crd/bases/config.projectsveltos.io_profiles.yaml +++ b/config/crd/bases/config.projectsveltos.io_profiles.yaml @@ -237,6 +237,55 @@ spec: `ExtraLabels`, the value from `ExtraLabels` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is empty: + there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 7dd01cb0..eb34aceb 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -171,21 +171,10 @@ rules: - lib.projectsveltos.io resources: - clusterhealthchecks - - configurationgroups/status - - debuggingconfigurations - - eventtriggers - - sveltosclusters - - sveltosclusters/status - verbs: - - get - - list - - watch -- apiGroups: - - lib.projectsveltos.io - resources: - clustersets - configurationbundles - configurationgroups + - healthchecks - reloaders - resourcesummaries - sets @@ -222,6 +211,18 @@ rules: - list - update - watch +- apiGroups: + - lib.projectsveltos.io + resources: + - configurationgroups/status + - debuggingconfigurations + - eventtriggers + - sveltosclusters + - sveltosclusters/status + verbs: + - get + - list + - watch - apiGroups: - lib.projectsveltos.io resources: diff --git a/controllers/clustersummary_controller.go b/controllers/clustersummary_controller.go index b2df69c4..7f540a4c 100644 --- a/controllers/clustersummary_controller.go +++ b/controllers/clustersummary_controller.go @@ -333,6 +333,13 @@ func (r *ClusterSummaryReconciler) reconcileDelete( r.updateDeletedInstancs(clusterSummaryScope) + // HealthCheck and ClusterHealthCheck live in the management cluster: they can be removed + // whatever the state of the managed cluster is. + if err := r.removeHealthMonitoring(ctx, clusterSummaryScope.ClusterSummary, logger); err != nil { + logger.V(logs.LogInfo).Error(err, "failed to remove health monitoring resources") + return reconcile.Result{RequeueAfter: deleteRequeueAfter}, nil + } + isReady, err := r.isReady(ctx, clusterSummaryScope.ClusterSummary, logger) if err != nil { return reconcile.Result{RequeueAfter: deleteRequeueAfter}, nil @@ -507,6 +514,14 @@ func (r *ClusterSummaryReconciler) reconcileNormal(ctx context.Context, } } + // Health monitoring is independent from deployment: it must be reconciled even when + // the ClusterSummary has nothing left to deploy (see shouldReconcile) + if err := r.reconcileHealthMonitoring(ctx, clusterSummaryScope.ClusterSummary, logger); err != nil { + logger.V(logs.LogInfo).Error(err, "failed to reconcile health monitoring") + r.setNextReconcileTime(clusterSummaryScope, normalRequeueAfter) + return reconcile.Result{RequeueAfter: normalRequeueAfter}, nil + } + if !r.shouldReconcile(clusterSummaryScope, logger) { logger.V(logs.LogInfo).Info("ClusterSummary does not need a reconciliation") return reconcile.Result{}, nil diff --git a/controllers/clustersummary_controller_test.go b/controllers/clustersummary_controller_test.go index e83cadbb..f64cb0b5 100644 --- a/controllers/clustersummary_controller_test.go +++ b/controllers/clustersummary_controller_test.go @@ -18,6 +18,7 @@ package controllers_test import ( "context" + "errors" "fmt" "sync" "time" @@ -36,6 +37,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -1116,6 +1118,111 @@ var _ = Describe("ClustersummaryController", func() { Expect(result.RequeueAfter).To(BeZero()) }) + It("reconcileDelete removes the health monitoring resources, even when the cluster is not found", func() { + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + clusterSummary.Spec.ClusterProfileSpec.PolicyRefs = []configv1beta1.PolicyRef{ + {Namespace: randomString(), Name: randomString(), Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind)}, + } + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureResources, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + } + + // No cluster. + initObjects := []client.Object{ + clusterProfile, + clusterSummary, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(initObjects...).WithObjects(initObjects...).Build() + + dep := fakedeployer.GetClient(context.TODO(), textlogger.NewLogger(textlogger.NewConfig()), c) + clusterSummaryReconciler := getClusterSummaryReconciler(c, dep) + + Expect(controllers.ReconcileHealthMonitoring(clusterSummaryReconciler, context.TODO(), clusterSummary, + textlogger.NewLogger(textlogger.NewConfig()))).To(Succeed()) + key := types.NamespacedName{Name: controllers.GetHealthMonitoringName(clusterSummary)} + Expect(c.Get(context.TODO(), key, &libsveltosv1beta1.HealthCheck{})).To(Succeed()) + Expect(c.Get(context.TODO(), key, &libsveltosv1beta1.ClusterHealthCheck{})).To(Succeed()) + + clusterSummaryScope := getClusterSummaryScope(c, textlogger.NewLogger(textlogger.NewConfig()), + clusterProfile, clusterSummary) + _, err := controllers.ReconcileDelete(clusterSummaryReconciler, context.TODO(), clusterSummaryScope, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + + err = c.Get(context.TODO(), key, &libsveltosv1beta1.HealthCheck{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + err = c.Get(context.TODO(), key, &libsveltosv1beta1.ClusterHealthCheck{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("reconcileNormal reconciles health monitoring even when there is nothing to deploy", func() { + // OneTime, with nothing to deploy: shouldReconcile is false. Helm releases are still managed. + clusterSummary.Spec.ClusterProfileSpec.SyncMode = configv1beta1.SyncModeOneTime + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + } + clusterSummary.Status.HelmReleaseSummaries = []configv1beta1.HelmChartSummary{ + {ReleaseNamespace: randomString(), ReleaseName: randomString(), Status: configv1beta1.HelmChartStatusManaging}, + } + + initObjects := []client.Object{ + clusterProfile, + clusterSummary, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(initObjects...).WithObjects(initObjects...).Build() + + dep := fakedeployer.GetClient(context.TODO(), textlogger.NewLogger(textlogger.NewConfig()), c) + clusterSummaryReconciler := getClusterSummaryReconciler(c, dep) + + clusterSummaryScope := getClusterSummaryScope(c, textlogger.NewLogger(textlogger.NewConfig()), + clusterProfile, clusterSummary) + result, err := controllers.ReconcileNormal(clusterSummaryReconciler, context.TODO(), clusterSummaryScope, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + Expect(result.RequeueAfter).To(BeZero()) + + key := types.NamespacedName{Name: controllers.GetHealthMonitoringName(clusterSummary)} + Expect(c.Get(context.TODO(), key, &libsveltosv1beta1.ClusterHealthCheck{})).To(Succeed()) + }) + + It("reconcileNormal requeues, without failing, when health monitoring cannot be reconciled", func() { + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + } + clusterSummary.Status.HelmReleaseSummaries = []configv1beta1.HelmChartSummary{ + {ReleaseNamespace: randomString(), ReleaseName: randomString(), Status: configv1beta1.HelmChartStatusManaging}, + } + + initObjects := []client.Object{ + clusterProfile, + clusterSummary, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(initObjects...).WithObjects(initObjects...). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*libsveltosv1beta1.HealthCheck); ok { + return errors.New("create failure") + } + return c.Create(ctx, obj, opts...) + }, + }).Build() + + dep := fakedeployer.GetClient(context.TODO(), textlogger.NewLogger(textlogger.NewConfig()), c) + clusterSummaryReconciler := getClusterSummaryReconciler(c, dep) + + clusterSummaryScope := getClusterSummaryScope(c, textlogger.NewLogger(textlogger.NewConfig()), + clusterProfile, clusterSummary) + result, err := controllers.ReconcileNormal(clusterSummaryReconciler, context.TODO(), clusterSummaryScope, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + Expect(result.RequeueAfter).To(Equal(controllers.NormalRequeueAfter)) + }) + It("reconcileDelete removes the DeletedInstances entry once the finalizer is removed", func() { // No cluster. initObjects := []client.Object{ diff --git a/controllers/clustersummary_predicates.go b/controllers/clustersummary_predicates.go index 119a348d..931ff85d 100644 --- a/controllers/clustersummary_predicates.go +++ b/controllers/clustersummary_predicates.go @@ -373,6 +373,14 @@ func (p ClusterSummaryPredicate) Update(e event.UpdateEvent) bool { return true } + // Feature summaries are compared by hash below, so a feature becoming Provisioned does not + // trigger a reconciliation. Health monitoring starts when all features are provisioned and + // depends on the Helm releases deployed: reconcile when any of those changes. + if isHealthMonitoringEnabled(newClusterSummary) && healthMonitoringInputsChanged(oldClusterSummary, newClusterSummary) { + log.V(logs.LogVerbose).Info("ClusterSummary state relevant to health monitoring changed. Reconciling ClusterSummary.") + return true + } + oldFS := getFeatureSummaryForFeatureID(oldClusterSummary, libsveltosv1beta1.FeatureHelm) newFS := getFeatureSummaryForFeatureID(newClusterSummary, libsveltosv1beta1.FeatureHelm) if !isFeatureSummarySame(oldFS, newFS) { diff --git a/controllers/export_test.go b/controllers/export_test.go index efed97d2..0b79821e 100644 --- a/controllers/export_test.go +++ b/controllers/export_test.go @@ -17,6 +17,8 @@ limitations under the License. package controllers import ( + "strings" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/client-go/rest" @@ -34,6 +36,7 @@ func SetEventRecorder(r *ClusterSummaryReconciler, recorder events.EventRecorder const ( PausedClustersRequeueAfter = pausedClustersRequeueAfter DeleteRequeueAfter = deleteRequeueAfter + NormalRequeueAfter = normalRequeueAfter ) var ( @@ -87,6 +90,7 @@ var ( ShouldRedeploy = (*ClusterSummaryReconciler).shouldRedeploy CanRemoveFinalizer = (*ClusterSummaryReconciler).canRemoveFinalizer ReconcileDelete = (*ClusterSummaryReconciler).reconcileDelete + ReconcileNormal = (*ClusterSummaryReconciler).reconcileNormal CleanupBeforeFinalizerRemoval = (*ClusterSummaryReconciler).cleanupBeforeFinalizerRemoval AreDependenciesDeployed = (*ClusterSummaryReconciler).areDependenciesDeployed AreDependentsRemoved = (*ClusterSummaryReconciler).areDependentsRemoved @@ -365,3 +369,32 @@ var ( func NewReleaseInfoForApply(applyHash []byte, skipApply bool) *releaseInfo { return &releaseInfo{applyHash: applyHash, skipApply: skipApply} } + +var ( + GetHealthMonitoringName = getHealthMonitoringName + GetMonitoringResourceSelectors = getMonitoringResourceSelectors + BuildMonitoringHealthCheck = buildMonitoringHealthCheck + BuildMonitoringClusterHealthCheck = buildMonitoringClusterHealthCheck + QuoteLuaString = quoteLuaString + ReconcileHealthMonitoring = (*ClusterSummaryReconciler).reconcileHealthMonitoring + RemoveHealthMonitoring = (*ClusterSummaryReconciler).removeHealthMonitoring + DefaultHealthCheckLivenessName = defaultHealthCheckLivenessName + HealthMonitoringFlappingEvaluation = healthMonitoringFlappingEvaluations +) + +// NewHelmReleases returns helm releases for tests, from "/" strings +func NewHelmReleases(namespaceAndNames ...string) []helmRelease { + releases := make([]helmRelease, 0, len(namespaceAndNames)) + for _, namespaceAndName := range namespaceAndNames { + parts := strings.SplitN(namespaceAndName, "/", 2) + releases = append(releases, helmRelease{namespace: parts[0], name: parts[1]}) + } + return releases +} + +// RenderHealthMonitoringLua renders the health monitoring script for tests +func RenderHealthMonitoringLua(ownerKind, ownerName string, releases []helmRelease) string { + return renderHealthMonitoringLua(&healthMonitoringLuaConfig{ + ownerKind: ownerKind, ownerName: ownerName, helmReleases: releases, + }) +} diff --git a/controllers/health_monitoring.go b/controllers/health_monitoring.go new file mode 100644 index 00000000..75a9fec8 --- /dev/null +++ b/controllers/health_monitoring.go @@ -0,0 +1,534 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "crypto/sha256" + _ "embed" // used to embed the Lua library + "encoding/hex" + "fmt" + "reflect" + "sort" + "strings" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/deployer" + "github.com/projectsveltos/libsveltos/lib/eventreasons" + logs "github.com/projectsveltos/libsveltos/lib/logsettings" +) + +//+kubebuilder:rbac:groups=lib.projectsveltos.io,resources=healthchecks,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=lib.projectsveltos.io,resources=clusterhealthchecks,verbs=get;list;watch;create;update;patch;delete + +const ( + healthMonitoringNamePrefix = "hm-" + + // healthChecksIgnoredReason is the reason of the Warning event emitted when a Profile lists + // HealthChecks, which only a ClusterProfile can do + healthChecksIgnoredReason = "HealthChecksIgnored" + + // defaultHealthCheckLivenessName is the name of the liveness check referencing the + // HealthCheck generated by addon-controller + defaultHealthCheckLivenessName = "deployed-resources" + + // healthMonitoringFlappingEvaluations is the number of consecutive evaluations a + // resource must be found not Healthy before it is reported + healthMonitoringFlappingEvaluations = 3 + + statefulSetKind = "StatefulSet" + daemonSetKind = "DaemonSet" + jobKind = "Job" + persistentVolumeClaimKind = "PersistentVolumeClaim" + + batchGroupName = "batch" + coreGroupName = "" + + // monitoredAPIVersion is the version all the monitored kinds are served from + // (apps/v1, batch/v1 and v1) + monitoredAPIVersion = "v1" +) + +var ( + // healthMonitoringLua is the Lua code evaluating the health of the resources. The + // Lua "config" table is prepended to it, see healthMonitoringLuaConfig. + // + //go:embed healthmonitoring/health.lua + healthMonitoringLua string + + // monitoredKinds are the kinds the default health checks cover + monitoredKinds = []monitoredKind{ + {group: appsGroupName, kind: deploymentKind}, + {group: appsGroupName, kind: statefulSetKind}, + {group: appsGroupName, kind: daemonSetKind}, + {group: batchGroupName, kind: jobKind}, + {group: coreGroupName, kind: persistentVolumeClaimKind}, + } +) + +// monitoredKind identifies a kind (version is always monitoredAPIVersion) covered by the default health checks +type monitoredKind struct { + group string + kind string +} + +// helmRelease identifies a Helm release deployed by a ClusterSummary +type helmRelease struct { + namespace string + name string +} + +// healthMonitoringLuaConfig contains what the Lua code needs to select, among all the +// resources found in the cluster, the ones deployed by a given profile. +type healthMonitoringLuaConfig struct { + ownerKind string + ownerName string + helmReleases []helmRelease +} + +// isHealthMonitoringEnabled returns true if health monitoring has been requested +func isHealthMonitoringEnabled(clusterSummary *configv1beta1.ClusterSummary) bool { + hm := clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring + return hm != nil && hm.Enabled +} + +// useDefaultHealthChecks returns true if the health of the resources deployed by the +// profile must be monitored. Default is true. +func useDefaultHealthChecks(hm *configv1beta1.HealthMonitoring) bool { + return hm.DefaultChecks == nil || *hm.DefaultChecks +} + +// getHealthMonitoringName returns the name of the HealthCheck and of the ClusterHealthCheck +// created for a ClusterSummary. Both are cluster scoped: the name is derived from the +// ClusterSummary UID so that it is unique, short and stable. +func getHealthMonitoringName(clusterSummary *configv1beta1.ClusterSummary) string { + sum := sha256.Sum256([]byte(clusterSummary.UID)) + return healthMonitoringNamePrefix + hex.EncodeToString(sum[:])[:12] +} + +func getHealthMonitoringLabels(clusterSummary *configv1beta1.ClusterSummary) map[string]string { + return map[string]string{ + configv1beta1.HealthMonitoringClusterSummaryUIDLabel: string(clusterSummary.UID), + } +} + +func getHealthMonitoringAnnotations(clusterSummary *configv1beta1.ClusterSummary) map[string]string { + return map[string]string{ + configv1beta1.HealthMonitoringClusterSummaryNamespaceAnnotation: clusterSummary.Namespace, + configv1beta1.HealthMonitoringClusterSummaryNameAnnotation: clusterSummary.Name, + } +} + +// getClusterSummaryClusterRef returns a reference to the cluster a ClusterSummary targets +func getClusterSummaryClusterRef(clusterSummary *configv1beta1.ClusterSummary) corev1.ObjectReference { + clusterRef := corev1.ObjectReference{ + Namespace: clusterSummary.Spec.ClusterNamespace, + Name: clusterSummary.Spec.ClusterName, + } + if clusterSummary.Spec.ClusterType == libsveltosv1beta1.ClusterTypeSveltos { + clusterRef.Kind = libsveltosv1beta1.SveltosClusterKind + clusterRef.APIVersion = libsveltosv1beta1.GroupVersion.String() + } else { + clusterRef.Kind = clusterKind + clusterRef.APIVersion = clusterv1.GroupVersion.String() + } + return clusterRef +} + +// areAllFeaturesProvisioned returns true if the ClusterSummary has features and all of them are provisioned +func areAllFeaturesProvisioned(clusterSummary *configv1beta1.ClusterSummary) bool { + if len(clusterSummary.Status.FeatureSummaries) == 0 { + return false + } + for i := range clusterSummary.Status.FeatureSummaries { + if clusterSummary.Status.FeatureSummaries[i].Status != libsveltosv1beta1.FeatureStatusProvisioned { + return false + } + } + return true +} + +// healthMonitoringInputsChanged returns true if what health monitoring depends on changed between +// two versions of a ClusterSummary: all features becoming (or ceasing to be) provisioned, and the +// Helm releases managed by the ClusterSummary. +func healthMonitoringInputsChanged(oldClusterSummary, newClusterSummary *configv1beta1.ClusterSummary) bool { + if areAllFeaturesProvisioned(oldClusterSummary) != areAllFeaturesProvisioned(newClusterSummary) { + return true + } + + return !reflect.DeepEqual(getManagedHelmReleases(oldClusterSummary), getManagedHelmReleases(newClusterSummary)) +} + +// getManagedHelmReleases returns the Helm releases currently managed by the ClusterSummary, sorted +func getManagedHelmReleases(clusterSummary *configv1beta1.ClusterSummary) []helmRelease { + releases := make([]helmRelease, 0, len(clusterSummary.Status.HelmReleaseSummaries)) + for i := range clusterSummary.Status.HelmReleaseSummaries { + summary := &clusterSummary.Status.HelmReleaseSummaries[i] + if summary.Status != configv1beta1.HelmChartStatusManaging { + continue + } + releases = append(releases, helmRelease{namespace: summary.ReleaseNamespace, name: summary.ReleaseName}) + } + + sort.Slice(releases, func(i, j int) bool { + if releases[i].namespace != releases[j].namespace { + return releases[i].namespace < releases[j].namespace + } + return releases[i].name < releases[j].name + }) + + return releases +} + +// deploysRawResources returns true if the profile deploys anything but Helm charts. +// Resources deployed by Sveltos (everything but Helm charts) carry the ReasonLabel label. +func deploysRawResources(clusterSummary *configv1beta1.ClusterSummary) bool { + spec := &clusterSummary.Spec.ClusterProfileSpec + return len(spec.PolicyRefs) > 0 || len(spec.KustomizationRefs) > 0 +} + +// getMonitoringResourceSelectors returns the selectors for the resources to evaluate: +// - for the resources deployed by Sveltos, every object with the label added by Sveltos; +// - for each Helm release namespace, every object in that namespace (the Lua code +// keeps only the ones belonging to a release of the profile). +func getMonitoringResourceSelectors(deployRaw bool, releases []helmRelease) []libsveltosv1beta1.ResourceSelector { + selectors := make([]libsveltosv1beta1.ResourceSelector, 0) + + if deployRaw { + for _, mk := range monitoredKinds { + selectors = append(selectors, libsveltosv1beta1.ResourceSelector{ + Group: mk.group, Version: monitoredAPIVersion, Kind: mk.kind, + LabelFilters: []libsveltosv1beta1.LabelFilter{ + {Key: deployer.ReasonLabel, Operation: libsveltosv1beta1.OperationHas}, + }, + }) + } + } + + namespaces := make(map[string]bool) + for i := range releases { + namespaces[releases[i].namespace] = true + } + sortedNamespaces := make([]string, 0, len(namespaces)) + for ns := range namespaces { + sortedNamespaces = append(sortedNamespaces, ns) + } + sort.Strings(sortedNamespaces) + + for _, ns := range sortedNamespaces { + for _, mk := range monitoredKinds { + selectors = append(selectors, libsveltosv1beta1.ResourceSelector{ + Group: mk.group, Version: monitoredAPIVersion, Kind: mk.kind, Namespace: ns, + }) + } + } + + return selectors +} + +// quoteLuaString returns s as a Lua string literal (Lua 5.1 syntax: no \x or \u escapes) +func quoteLuaString(s string) string { + var b strings.Builder + b.WriteByte('"') + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c == '\\': + b.WriteString(`\\`) + case c == '"': + b.WriteString(`\"`) + case c == '\n': + b.WriteString(`\n`) + case c == '\r': + b.WriteString(`\r`) + case c < ' ' || c == 0x7f: + fmt.Fprintf(&b, `\%03d`, c) + default: + b.WriteByte(c) + } + } + b.WriteByte('"') + return b.String() +} + +// renderHealthMonitoringLua returns the script evaluating the health of the resources deployed +// by a profile: the Lua "config" table followed by the embedded Lua library. +func renderHealthMonitoringLua(config *healthMonitoringLuaConfig) string { + var b strings.Builder + b.WriteString("local config = {\n") + fmt.Fprintf(&b, " ownerKind = %s,\n", quoteLuaString(config.ownerKind)) + fmt.Fprintf(&b, " ownerName = %s,\n", quoteLuaString(config.ownerName)) + b.WriteString(" helmReleases = {\n") + for _, release := range config.helmReleases { + fmt.Fprintf(&b, " [%s] = true,\n", quoteLuaString(release.namespace+"/"+release.name)) + } + b.WriteString(" },\n}\n\n") + b.WriteString(healthMonitoringLua) + return b.String() +} + +// buildMonitoringHealthCheck returns the HealthCheck evaluating the health of the resources deployed +// by the profile owning the ClusterSummary. Returns nil if there is nothing to monitor. +func buildMonitoringHealthCheck(clusterSummary *configv1beta1.ClusterSummary, profileKind, profileName string, +) *libsveltosv1beta1.HealthCheck { + + releases := getManagedHelmReleases(clusterSummary) + selectors := getMonitoringResourceSelectors(deploysRawResources(clusterSummary), releases) + if len(selectors) == 0 { + return nil + } + + script := renderHealthMonitoringLua(&healthMonitoringLuaConfig{ + ownerKind: profileKind, ownerName: profileName, helmReleases: releases, + }) + + return &libsveltosv1beta1.HealthCheck{ + ObjectMeta: metav1.ObjectMeta{ + Name: getHealthMonitoringName(clusterSummary), + Labels: getHealthMonitoringLabels(clusterSummary), + Annotations: getHealthMonitoringAnnotations(clusterSummary), + }, + Spec: libsveltosv1beta1.HealthCheckSpec{ + ResourceSelectors: selectors, + EvaluateHealth: script, + EvaluateIndependently: true, + Flapping: &libsveltosv1beta1.FlappingPolicy{ + ConsecutiveEvaluations: healthMonitoringFlappingEvaluations, + }, + }, + } +} + +func getHealthCheckLiveness(livenessName, healthCheckName string) libsveltosv1beta1.LivenessCheck { + return libsveltosv1beta1.LivenessCheck{ + Name: livenessName, + Type: libsveltosv1beta1.LivenessTypeHealthCheck, + LivenessSourceRef: &corev1.ObjectReference{ + APIVersion: libsveltosv1beta1.GroupVersion.String(), + Kind: libsveltosv1beta1.HealthCheckKind, + Name: healthCheckName, + }, + } +} + +// buildMonitoringClusterHealthCheck returns the ClusterHealthCheck targeting the cluster of the +// ClusterSummary. Its liveness checks are the generated HealthCheck (when generatedHealthCheck is not +// empty) and the HealthChecks listed by the user. +func buildMonitoringClusterHealthCheck(clusterSummary *configv1beta1.ClusterSummary, + generatedHealthCheck string, userHealthChecks []string) *libsveltosv1beta1.ClusterHealthCheck { + + livenessChecks := make([]libsveltosv1beta1.LivenessCheck, 0, len(userHealthChecks)+1) + if generatedHealthCheck != "" { + livenessChecks = append(livenessChecks, + getHealthCheckLiveness(defaultHealthCheckLivenessName, generatedHealthCheck)) + } + + sortedUserHealthChecks := append([]string{}, userHealthChecks...) + sort.Strings(sortedUserHealthChecks) + for _, name := range sortedUserHealthChecks { + livenessChecks = append(livenessChecks, getHealthCheckLiveness(name, name)) + } + + return &libsveltosv1beta1.ClusterHealthCheck{ + ObjectMeta: metav1.ObjectMeta{ + Name: getHealthMonitoringName(clusterSummary), + Labels: getHealthMonitoringLabels(clusterSummary), + Annotations: getHealthMonitoringAnnotations(clusterSummary), + }, + Spec: libsveltosv1beta1.ClusterHealthCheckSpec{ + ClusterRefs: []corev1.ObjectReference{getClusterSummaryClusterRef(clusterSummary)}, + LivenessChecks: livenessChecks, + // Results are consumed from the HealthCheckReports. No notification is requested. + Notifications: []libsveltosv1beta1.Notification{}, + }, + } +} + +// reconcileHealthMonitoring makes sure the HealthCheck and ClusterHealthCheck needed to monitor the +// ClusterSummary exist (or do not exist when monitoring is not enabled). +// Instances are created only once all features are provisioned, so a deployment still +// in progress is not reported as unhealthy. Once created they are kept, so that the health of an +// update is reported. +func (r *ClusterSummaryReconciler) reconcileHealthMonitoring(ctx context.Context, + clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) error { + + if !clusterSummary.DeletionTimestamp.IsZero() { + return nil + } + + // A DryRun ClusterSummary does not deploy anything: there is nothing to monitor + if clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeDryRun { + return nil + } + + if !isHealthMonitoringEnabled(clusterSummary) { + return r.removeHealthMonitoring(ctx, clusterSummary, logger) + } + + if !areAllFeaturesProvisioned(clusterSummary) { + logger.V(logs.LogDebug).Info("not all features are provisioned yet. Health monitoring is deferred.") + return nil + } + + profileRef, err := configv1beta1.GetProfileRef(clusterSummary) + if err != nil { + return err + } + + hm := clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring + healthChecks := r.getAllowedHealthChecks(clusterSummary, profileRef.Kind, hm) + + generatedHealthCheck := "" + if useDefaultHealthChecks(hm) { + healthCheck := buildMonitoringHealthCheck(clusterSummary, profileRef.Kind, profileRef.Name) + if healthCheck != nil { + if err := createOrUpdateMonitoringHealthCheck(ctx, r.Client, healthCheck); err != nil { + return err + } + generatedHealthCheck = healthCheck.Name + } + } + + if generatedHealthCheck == "" && len(healthChecks) == 0 { + // Nothing to monitor (yet): the profile does not deploy any of the monitored kinds + return r.removeHealthMonitoring(ctx, clusterSummary, logger) + } + + if generatedHealthCheck == "" { + // The generated HealthCheck is not needed anymore + if err := deleteMonitoringHealthCheck(ctx, r.Client, clusterSummary); err != nil { + return err + } + } + + clusterHealthCheck := buildMonitoringClusterHealthCheck(clusterSummary, generatedHealthCheck, healthChecks) + return createOrUpdateMonitoringClusterHealthCheck(ctx, r.Client, clusterHealthCheck) +} + +// getAllowedHealthChecks returns the HealthChecks listed in healthMonitoring that can be monitored. +// Only a ClusterProfile can list HealthChecks: a HealthCheck is cluster scoped and referenced by name, +// and the report it produces is shared by all the profiles referencing it on a cluster, so a (namespaced) +// Profile could otherwise learn about resources outside its namespace. For a Profile the list is +// ignored, and a Warning event says so. +func (r *ClusterSummaryReconciler) getAllowedHealthChecks(clusterSummary *configv1beta1.ClusterSummary, + profileKind string, hm *configv1beta1.HealthMonitoring) []string { + + if profileKind != configv1beta1.ProfileKind || len(hm.HealthChecks) == 0 { + return hm.HealthChecks + } + + recordEventf(r.eventRecorder, clusterSummary, nil, corev1.EventTypeWarning, healthChecksIgnoredReason, + eventreasons.ActionReconciling, + "healthMonitoring.healthChecks is ignored: only a ClusterProfile can reference HealthChecks, "+ + "a Profile gets the default health checks only") + + return nil +} + +// removeHealthMonitoring removes the HealthCheck and ClusterHealthCheck created for the ClusterSummary +func (r *ClusterSummaryReconciler) removeHealthMonitoring(ctx context.Context, + clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) error { + + logger.V(logs.LogDebug).Info("removing health monitoring resources, if any") + + // The ClusterHealthCheck goes first: removing it makes healthcheck-manager remove the + // HealthCheck from the managed cluster. + if err := deleteIfOwned(ctx, r.Client, &libsveltosv1beta1.ClusterHealthCheck{}, clusterSummary); err != nil { + return err + } + + return deleteMonitoringHealthCheck(ctx, r.Client, clusterSummary) +} + +func deleteMonitoringHealthCheck(ctx context.Context, c client.Client, + clusterSummary *configv1beta1.ClusterSummary) error { + + return deleteIfOwned(ctx, c, &libsveltosv1beta1.HealthCheck{}, clusterSummary) +} + +// deleteIfOwned deletes the instance created for the ClusterSummary, if any. It does nothing +// if the instance does not exist, or if the CRD is not installed. +func deleteIfOwned(ctx context.Context, c client.Client, obj client.Object, + clusterSummary *configv1beta1.ClusterSummary) error { + + obj.SetName(getHealthMonitoringName(clusterSummary)) + err := c.Get(ctx, client.ObjectKeyFromObject(obj), obj) + if err != nil { + if apierrors.IsNotFound(err) || meta.IsNoMatchError(err) { + return nil + } + return err + } + + // Never delete an instance this ClusterSummary did not create + if obj.GetLabels()[configv1beta1.HealthMonitoringClusterSummaryUIDLabel] != string(clusterSummary.UID) { + return nil + } + + err = c.Delete(ctx, obj) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +func createOrUpdateMonitoringHealthCheck(ctx context.Context, c client.Client, + desired *libsveltosv1beta1.HealthCheck) error { + + healthCheck := &libsveltosv1beta1.HealthCheck{ObjectMeta: metav1.ObjectMeta{Name: desired.Name}} + _, err := controllerutil.CreateOrUpdate(ctx, c, healthCheck, func() error { + healthCheck.Labels = mergeStringMaps(healthCheck.Labels, desired.Labels) + healthCheck.Annotations = mergeStringMaps(healthCheck.Annotations, desired.Annotations) + healthCheck.Spec = desired.Spec + return nil + }) + return err +} + +func createOrUpdateMonitoringClusterHealthCheck(ctx context.Context, c client.Client, + desired *libsveltosv1beta1.ClusterHealthCheck) error { + + clusterHealthCheck := &libsveltosv1beta1.ClusterHealthCheck{ObjectMeta: metav1.ObjectMeta{Name: desired.Name}} + _, err := controllerutil.CreateOrUpdate(ctx, c, clusterHealthCheck, func() error { + clusterHealthCheck.Labels = mergeStringMaps(clusterHealthCheck.Labels, desired.Labels) + clusterHealthCheck.Annotations = mergeStringMaps(clusterHealthCheck.Annotations, desired.Annotations) + clusterHealthCheck.Spec = desired.Spec + return nil + }) + return err +} + +// mergeStringMaps returns current with all the entries of desired set +func mergeStringMaps(current, desired map[string]string) map[string]string { + if current == nil { + current = make(map[string]string, len(desired)) + } + for k, v := range desired { + current[k] = v + } + return current +} diff --git a/controllers/health_monitoring_test.go b/controllers/health_monitoring_test.go new file mode 100644 index 00000000..168357f9 --- /dev/null +++ b/controllers/health_monitoring_test.go @@ -0,0 +1,1147 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers_test + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "time" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + lua "github.com/yuin/gopher-lua" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + "k8s.io/klog/v2/textlogger" + "k8s.io/utils/ptr" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/event" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + "github.com/projectsveltos/addon-controller/controllers" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/deployer" + fakedeployer "github.com/projectsveltos/libsveltos/lib/deployer/fake" + sveltoslua "github.com/projectsveltos/libsveltos/lib/lua" +) + +const ( + hmOwnerKind = "ClusterProfile" + hmOwnerName = "platform-stack" + + hmNamespace = "web-namespace" + hmUserHealthCheck = "my-cert-expiry-check" + + hmHelmReleaseNameAnnotation = "meta.helm.sh/release-name" + hmHelmReleaseNamespaceAnnotation = "meta.helm.sh/release-namespace" + + hmStatusProgressing = "Progressing" + + hmCertManagerRelease = "cert-manager/cert-manager" + hmGrafanaRelease = "monitoring/grafana" +) + +// healthResult is what the health monitoring Lua code reports for one resource +type healthResult struct { + name string + status string + message string +} + +// luaHealthOutput is the structure returned by the evaluate Lua function +type luaHealthOutput struct { + Resources []struct { + Resource struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + } `json:"resource"` + Status string `json:"status"` + Message string `json:"message"` + } `json:"resources"` +} + +// kindOf returns the Kubernetes kind of a typed object, e.g. Deployment for *appsv1.Deployment +func kindOf(obj interface{}) string { + return reflect.TypeOf(obj).Elem().Name() +} + +// toLuaResource converts a typed object to the table the HealthCheck Lua code receives. Resources reach +// the Lua code decoded from JSON, hence the round trip. +func toLuaResource(obj client.Object) lua.LValue { + gvk, err := apiutil.GVKForObject(obj, scheme) + Expect(err).To(BeNil()) + obj.GetObjectKind().SetGroupVersionKind(gvk) + + jsonResource, err := json.Marshal(obj) + Expect(err).To(BeNil()) + decoded := map[string]interface{}{} + Expect(json.Unmarshal(jsonResource, &decoded)).To(Succeed()) + + return sveltoslua.MapToTable(decoded) +} + +// evaluateHealthLua runs the health monitoring script against resources +func evaluateHealthLua(script string, resources ...client.Object) []healthResult { + l := lua.NewState() + defer l.Close() + + Expect(l.DoString(script)).To(Succeed()) + + resourcesTable := l.NewTable() + for i := range resources { + resourcesTable.Append(toLuaResource(resources[i])) + } + l.SetGlobal("resources", resourcesTable) + + Expect(l.CallByParam(lua.P{Fn: l.GetGlobal("evaluate"), NRet: 1, Protect: true})).To(Succeed()) + + tbl, ok := l.Get(-1).(*lua.LTable) + Expect(ok).To(BeTrue()) + + jsonOutput, err := json.Marshal(sveltoslua.ToGoValue(tbl)) + Expect(err).To(BeNil()) + var output luaHealthOutput + Expect(json.Unmarshal(jsonOutput, &output)).To(Succeed()) + + var results []healthResult + for i := range output.Resources { + results = append(results, healthResult{ + name: output.Resources[i].Resource.Metadata.Name, + status: output.Resources[i].Status, + message: output.Resources[i].Message, + }) + } + return results +} + +func ownerAnnotations() map[string]string { + return map[string]string{ + deployer.OwnerKind: hmOwnerKind, + deployer.OwnerName: hmOwnerName, + } +} + +func helmAnnotations(namespace, release string) map[string]string { + return map[string]string{ + hmHelmReleaseNameAnnotation: release, + hmHelmReleaseNamespaceAnnotation: namespace, + } +} + +func hmMetadata(name, uid string, annotations map[string]string, generation int64) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Name: name, Namespace: hmNamespace, UID: types.UID(uid), Generation: generation, Annotations: annotations, + } +} + +func hmDeployment(name, uid string, annotations map[string]string, generation int64, + spec *appsv1.DeploymentSpec, status *appsv1.DeploymentStatus) *appsv1.Deployment { + + return &appsv1.Deployment{ObjectMeta: hmMetadata(name, uid, annotations, generation), Spec: *spec, Status: *status} +} + +func hmStatefulSet(name, uid string, annotations map[string]string, generation int64, + spec *appsv1.StatefulSetSpec, status *appsv1.StatefulSetStatus) *appsv1.StatefulSet { + + return &appsv1.StatefulSet{ObjectMeta: hmMetadata(name, uid, annotations, generation), Spec: *spec, Status: *status} +} + +func hmDaemonSet(name, uid string, annotations map[string]string, + spec *appsv1.DaemonSetSpec, status *appsv1.DaemonSetStatus) *appsv1.DaemonSet { + + return &appsv1.DaemonSet{ObjectMeta: hmMetadata(name, uid, annotations, 1), Spec: *spec, Status: *status} +} + +func hmJob(name, uid string, conditions ...batchv1.JobCondition) *batchv1.Job { + return &batchv1.Job{ + ObjectMeta: hmMetadata(name, uid, ownerAnnotations(), 1), + Status: batchv1.JobStatus{Conditions: conditions}, + } +} + +func jobCondition(conditionType batchv1.JobConditionType, reason string) batchv1.JobCondition { + return batchv1.JobCondition{Type: conditionType, Status: corev1.ConditionTrue, Reason: reason} +} + +func rollingUpdateStrategy(partition int32) appsv1.StatefulSetUpdateStrategy { + return appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{Partition: ptr.To(partition)}, + } +} + +var _ = Describe("Health monitoring: Lua health rules", func() { + var script string + + BeforeEach(func() { + script = controllers.RenderHealthMonitoringLua(hmOwnerKind, hmOwnerName, + controllers.NewHelmReleases(hmCertManagerRelease, hmGrafanaRelease)) + }) + + It("reports nothing for a Deployment that completed its rollout", func() { + deployment := hmDeployment("web", "uid-1", ownerAnnotations(), 2, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)}, + &appsv1.DeploymentStatus{ObservedGeneration: 2, Replicas: 3, UpdatedReplicas: 3, AvailableReplicas: 3}) + + Expect(evaluateHealthLua(script, deployment)).To(BeEmpty()) + }) + + It("reports a Deployment in the middle of a rollout as Progressing", func() { + deployment := hmDeployment("web", "uid-1", ownerAnnotations(), 2, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)}, + &appsv1.DeploymentStatus{ObservedGeneration: 2, Replicas: 4, UpdatedReplicas: 2, AvailableReplicas: 3}) + + results := evaluateHealthLua(script, deployment) + Expect(results).To(HaveLen(1)) + Expect(results[0].status).To(Equal("Progressing")) + Expect(results[0].message).To(ContainSubstring("2 out of 3 new replicas have been updated")) + }) + + It("reports a Deployment whose new generation was not observed yet as Progressing", func() { + deployment := hmDeployment("web", "uid-1", ownerAnnotations(), 3, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)}, + &appsv1.DeploymentStatus{ObservedGeneration: 2, Replicas: 3, UpdatedReplicas: 3, AvailableReplicas: 3}) + + results := evaluateHealthLua(script, deployment) + Expect(results).To(HaveLen(1)) + Expect(results[0].status).To(Equal("Progressing")) + Expect(results[0].message).To(ContainSubstring("observed deployment generation less than desired generation")) + }) + + It("reports a Deployment with updated replicas not available as Progressing", func() { + deployment := hmDeployment("web", "uid-1", ownerAnnotations(), 2, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)}, + &appsv1.DeploymentStatus{ObservedGeneration: 2, Replicas: 3, UpdatedReplicas: 3, AvailableReplicas: 1}) + + results := evaluateHealthLua(script, deployment) + Expect(results).To(HaveLen(1)) + Expect(results[0].status).To(Equal("Progressing")) + Expect(results[0].message).To(ContainSubstring("1 of 3 updated replicas are available")) + }) + + It("reports a Deployment that exceeded its progress deadline as Degraded", func() { + deployment := hmDeployment("web", "uid-1", ownerAnnotations(), 2, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)}, + &appsv1.DeploymentStatus{ + ObservedGeneration: 2, Replicas: 3, UpdatedReplicas: 3, + Conditions: []appsv1.DeploymentCondition{ + { + Type: appsv1.DeploymentProgressing, Status: corev1.ConditionFalse, + Reason: "ProgressDeadlineExceeded", + }, + }, + }) + + results := evaluateHealthLua(script, deployment) + Expect(results).To(HaveLen(1)) + Expect(results[0].status).To(Equal("Degraded")) + }) + + It("reports a paused Deployment as Suspended", func() { + deployment := hmDeployment("web", "uid-1", ownerAnnotations(), 2, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3), Paused: true}, + &appsv1.DeploymentStatus{ObservedGeneration: 2}) + + results := evaluateHealthLua(script, deployment) + Expect(results).To(HaveLen(1)) + Expect(results[0].status).To(Equal("Suspended")) + }) + + It("ignores resources not deployed by the profile", func() { + broken := &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)} + notObserved := &appsv1.DeploymentStatus{ObservedGeneration: 1} + + // Deployed by another profile + otherProfile := ownerAnnotations() + otherProfile[deployer.OwnerName] = "another-profile" + // Same name, deployed by a Profile instead of a ClusterProfile + otherKind := ownerAnnotations() + otherKind[deployer.OwnerKind] = "Profile" + + results := evaluateHealthLua(script, + hmDeployment("no-annotation", "uid-1", nil, 2, broken, notObserved), + hmDeployment("other-profile", "uid-2", otherProfile, 2, broken, notObserved), + hmDeployment("other-kind", "uid-3", otherKind, 2, broken, notObserved), + // Helm release not deployed by this profile + hmDeployment("other-release", "uid-4", helmAnnotations(testReleaseNameKyverno, testReleaseNameKyverno), 2, + broken, notObserved), + // Same release name in a different namespace + hmDeployment("other-namespace", "uid-5", helmAnnotations(hmNamespace, "cert-manager"), 2, broken, notObserved), + ) + + Expect(results).To(BeEmpty()) + }) + + It("evaluates resources belonging to a Helm release of the profile", func() { + broken := &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)} + notObserved := &appsv1.DeploymentStatus{ObservedGeneration: 1} + + results := evaluateHealthLua(script, + hmDeployment("cert-manager", "uid-1", helmAnnotations("cert-manager", "cert-manager"), 2, broken, notObserved), + hmDeployment("grafana", "uid-2", helmAnnotations("monitoring", "grafana"), 2, broken, notObserved), + ) + + Expect(results).To(HaveLen(2)) + Expect(results[0].name).To(Equal("cert-manager")) + Expect(results[1].name).To(Equal("grafana")) + }) + + It("evaluates once a resource selected by more than one selector", func() { + annotations := ownerAnnotations() + for k, v := range helmAnnotations("cert-manager", "cert-manager") { + annotations[k] = v + } + deployment := hmDeployment("web", "same-uid", annotations, 2, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)}, &appsv1.DeploymentStatus{ObservedGeneration: 1}) + + Expect(evaluateHealthLua(script, deployment, deployment)).To(HaveLen(1)) + }) + + It("ignores kinds it has no rule for", func() { + replicaSet := &appsv1.ReplicaSet{ObjectMeta: hmMetadata("rs", "uid-1", ownerAnnotations(), 1)} + Expect(evaluateHealthLua(script, replicaSet)).To(BeEmpty()) + }) + + It("evaluates StatefulSets", func() { + healthy := hmStatefulSet("healthy", "uid-1", ownerAnnotations(), 1, + &appsv1.StatefulSetSpec{Replicas: ptr.To[int32](3), UpdateStrategy: rollingUpdateStrategy(0)}, + &appsv1.StatefulSetStatus{ObservedGeneration: 1, ReadyReplicas: 3, UpdatedReplicas: 3}) + notReady := hmStatefulSet("not-ready", "uid-2", ownerAnnotations(), 1, + &appsv1.StatefulSetSpec{Replicas: ptr.To[int32](3)}, + &appsv1.StatefulSetStatus{ObservedGeneration: 1, ReadyReplicas: 1}) + notObserved := hmStatefulSet("not-observed", "uid-3", ownerAnnotations(), 2, + &appsv1.StatefulSetSpec{Replicas: ptr.To[int32](3)}, + &appsv1.StatefulSetStatus{ObservedGeneration: 1, ReadyReplicas: 3}) + rolling := hmStatefulSet("rolling", "uid-4", ownerAnnotations(), 1, + &appsv1.StatefulSetSpec{Replicas: ptr.To[int32](3), UpdateStrategy: rollingUpdateStrategy(0)}, + &appsv1.StatefulSetStatus{ObservedGeneration: 1, ReadyReplicas: 3, UpdatedReplicas: 1}) + onDelete := hmStatefulSet("on-delete", "uid-5", ownerAnnotations(), 1, + &appsv1.StatefulSetSpec{ + Replicas: ptr.To[int32](3), + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{Type: appsv1.OnDeleteStatefulSetStrategyType}, + }, + &appsv1.StatefulSetStatus{ObservedGeneration: 1, ReadyReplicas: 3, UpdatedReplicas: 0}) + + results := evaluateHealthLua(script, healthy, notReady, notObserved, rolling, onDelete) + + Expect(results).To(HaveLen(3)) + Expect(results[0].name).To(Equal("not-ready")) + Expect(results[0].message).To(ContainSubstring("Waiting for 2 pods to be ready")) + Expect(results[1].name).To(Equal("not-observed")) + Expect(results[2].name).To(Equal("rolling")) + Expect(results[2].message).To(ContainSubstring("1 out of 3 new pods have been updated")) + for i := range results { + Expect(results[i].status).To(Equal("Progressing")) + } + }) + + It("evaluates DaemonSets", func() { + healthy := hmDaemonSet("healthy", "uid-1", ownerAnnotations(), &appsv1.DaemonSetSpec{}, + &appsv1.DaemonSetStatus{ + ObservedGeneration: 1, DesiredNumberScheduled: 3, UpdatedNumberScheduled: 3, NumberAvailable: 3, + }) + updating := hmDaemonSet("updating", "uid-2", ownerAnnotations(), &appsv1.DaemonSetSpec{}, + &appsv1.DaemonSetStatus{ + ObservedGeneration: 1, DesiredNumberScheduled: 3, UpdatedNumberScheduled: 1, NumberAvailable: 3, + }) + unavailable := hmDaemonSet("unavailable", "uid-3", ownerAnnotations(), &appsv1.DaemonSetSpec{}, + &appsv1.DaemonSetStatus{ + ObservedGeneration: 1, DesiredNumberScheduled: 3, UpdatedNumberScheduled: 3, NumberAvailable: 2, + }) + onDelete := hmDaemonSet("on-delete", "uid-4", ownerAnnotations(), + &appsv1.DaemonSetSpec{UpdateStrategy: appsv1.DaemonSetUpdateStrategy{Type: appsv1.OnDeleteDaemonSetStrategyType}}, + &appsv1.DaemonSetStatus{ObservedGeneration: 1, DesiredNumberScheduled: 3}) + + results := evaluateHealthLua(script, healthy, updating, unavailable, onDelete) + + Expect(results).To(HaveLen(2)) + Expect(results[0].name).To(Equal("updating")) + Expect(results[0].message).To(ContainSubstring("1 out of 3 new pods have been updated")) + Expect(results[1].name).To(Equal("unavailable")) + Expect(results[1].message).To(ContainSubstring("2 of 3 updated pods are available")) + }) + + It("evaluates Jobs", func() { + completed := hmJob("completed", "uid-1", jobCondition(batchv1.JobComplete, "")) + running := hmJob("running", "uid-2") + failed := hmJob("failed", "uid-3", jobCondition(batchv1.JobFailed, "BackoffLimitExceeded")) + suspended := hmJob("suspended", "uid-4", jobCondition(batchv1.JobSuspended, "")) + // A Failed condition wins over a Suspended one + failedAndSuspended := hmJob("failed-and-suspended", "uid-5", + jobCondition(batchv1.JobSuspended, ""), jobCondition(batchv1.JobFailed, "DeadlineExceeded")) + // A condition that is not true does not count + resumed := hmJob("resumed", "uid-6", batchv1.JobCondition{Type: batchv1.JobSuspended, Status: corev1.ConditionFalse}) + + results := evaluateHealthLua(script, completed, running, failed, suspended, failedAndSuspended, resumed) + + Expect(results).To(HaveLen(5)) + Expect(results[0]).To(Equal(healthResult{name: "running", status: hmStatusProgressing, message: "Job has not completed yet"})) + Expect(results[1].name).To(Equal("failed")) + Expect(results[1].status).To(Equal("Degraded")) + Expect(results[1].message).To(ContainSubstring("BackoffLimitExceeded")) + Expect(results[2]).To(Equal(healthResult{name: "suspended", status: "Suspended", message: "Job is suspended"})) + Expect(results[3].name).To(Equal("failed-and-suspended")) + Expect(results[3].status).To(Equal("Degraded")) + Expect(results[4].name).To(Equal("resumed")) + Expect(results[4].status).To(Equal("Progressing")) + }) + + It("evaluates PersistentVolumeClaims", func() { + pvc := func(name, uid string, phase corev1.PersistentVolumeClaimPhase) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: hmMetadata(name, uid, ownerAnnotations(), 1), + Status: corev1.PersistentVolumeClaimStatus{Phase: phase}, + } + } + + results := evaluateHealthLua(script, + pvc("bound", "uid-1", corev1.ClaimBound), + pvc("pending", "uid-2", corev1.ClaimPending), + pvc("lost", "uid-3", corev1.ClaimLost), + pvc("no-phase", "uid-4", ""), + ) + + Expect(results).To(HaveLen(3)) + Expect(results[0]).To(Equal(healthResult{name: "pending", status: hmStatusProgressing, message: "PersistentVolumeClaim is not bound yet"})) + Expect(results[1].name).To(Equal("lost")) + Expect(results[1].status).To(Equal("Degraded")) + Expect(results[2].name).To(Equal("no-phase")) + Expect(results[2].status).To(Equal("Progressing")) + }) + + It("quotes values placed in the Lua configuration", func() { + Expect(controllers.QuoteLuaString(`plain`)).To(Equal(`"plain"`)) + Expect(controllers.QuoteLuaString(`a"b\c`)).To(Equal(`"a\"b\\c"`)) + Expect(controllers.QuoteLuaString("a\nb\rc")).To(Equal(`"a\nb\rc"`)) + Expect(controllers.QuoteLuaString("a\x01b")).To(Equal(`"a\001b"`)) + + // A malicious value stays a string and cannot break out of the configuration + malicious := `"}} os.exit(1) --` + script := controllers.RenderHealthMonitoringLua(malicious, malicious, + controllers.NewHelmReleases("ns/"+malicious)) + Expect(evaluateHealthLua(script)).To(BeEmpty()) + }) +}) + +func newHealthMonitoringClusterSummary(clusterType libsveltosv1beta1.ClusterType) *configv1beta1.ClusterSummary { + return &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: randomString(), + Name: randomString(), + UID: types.UID(randomString()), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: configv1beta1.GroupVersion.String(), Kind: configv1beta1.ClusterProfileKind, + Name: hmOwnerName, UID: types.UID(randomString()), + }, + }, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: randomString(), + ClusterName: randomString(), + ClusterType: clusterType, + }, + } +} + +func provisionAllFeatures(clusterSummary *configv1beta1.ClusterSummary) { + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + {FeatureID: libsveltosv1beta1.FeatureResources, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + } +} + +func addHelmReleases(clusterSummary *configv1beta1.ClusterSummary, namespaceAndNames ...string) { + for _, namespaceAndName := range namespaceAndNames { + parts := strings.SplitN(namespaceAndName, "/", 2) + clusterSummary.Status.HelmReleaseSummaries = append(clusterSummary.Status.HelmReleaseSummaries, + configv1beta1.HelmChartSummary{ + ReleaseNamespace: parts[0], ReleaseName: parts[1], Status: configv1beta1.HelmChartStatusManaging, + }) + } +} + +var _ = Describe("Health monitoring: resources generated for a ClusterSummary", func() { + It("selects the resources deployed by Sveltos by label and the Helm ones by namespace", func() { + selectors := controllers.GetMonitoringResourceSelectors(true, + controllers.NewHelmReleases("monitoring/grafana", "cert-manager/cert-manager", "monitoring/loki")) + + // 5 kinds with the label, then 5 kinds in each of the two (sorted) namespaces + type kindGroup struct{ kind, group string } + kinds := []kindGroup{ + {kindOf(&appsv1.Deployment{}), appsv1.GroupName}, + {kindOf(&appsv1.StatefulSet{}), appsv1.GroupName}, + {kindOf(&appsv1.DaemonSet{}), appsv1.GroupName}, + {kindOf(&batchv1.Job{}), batchv1.GroupName}, + {kindOf(&corev1.PersistentVolumeClaim{}), corev1.GroupName}, + } + Expect(selectors).To(HaveLen(len(kinds) * 3)) + + for i, kg := range kinds { + raw := selectors[i] + Expect(raw.Kind).To(Equal(kg.kind)) + Expect(raw.Group).To(Equal(kg.group)) + Expect(raw.Namespace).To(BeEmpty()) + Expect(raw.LabelFilters).To(Equal([]libsveltosv1beta1.LabelFilter{ + {Key: "projectsveltos.io/reason", Operation: libsveltosv1beta1.OperationHas}, + })) + + for j, namespace := range []string{"cert-manager", "monitoring"} { + helm := selectors[len(kinds)*(j+1)+i] + Expect(helm.Kind).To(Equal(kg.kind)) + Expect(helm.Group).To(Equal(kg.group)) + Expect(helm.Namespace).To(Equal(namespace)) + Expect(helm.LabelFilters).To(BeEmpty()) + } + } + for i := range selectors { + Expect(selectors[i].Version).To(Equal(appsv1.SchemeGroupVersion.Version)) + } + + Expect(controllers.GetMonitoringResourceSelectors(false, nil)).To(BeEmpty()) + }) + + It("builds a HealthCheck for the resources and Helm releases of the ClusterSummary", func() { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + clusterSummary.Spec.ClusterProfileSpec.PolicyRefs = []configv1beta1.PolicyRef{ + {Kind: "ConfigMap", Namespace: hmNamespace, Name: "policies"}, + } + addHelmReleases(clusterSummary, hmCertManagerRelease) + // A release managed by another ClusterSummary must not be monitored + clusterSummary.Status.HelmReleaseSummaries = append(clusterSummary.Status.HelmReleaseSummaries, + configv1beta1.HelmChartSummary{ + ReleaseNamespace: testReleaseNameKyverno, ReleaseName: testReleaseNameKyverno, Status: configv1beta1.HelmChartStatusConflict, + }) + + healthCheck := controllers.BuildMonitoringHealthCheck(clusterSummary, hmOwnerKind, hmOwnerName) + Expect(healthCheck).ToNot(BeNil()) + + Expect(healthCheck.Name).To(Equal(controllers.GetHealthMonitoringName(clusterSummary))) + Expect(healthCheck.Name).To(HavePrefix("hm-")) + Expect(healthCheck.Labels).To(HaveKeyWithValue( + configv1beta1.HealthMonitoringClusterSummaryUIDLabel, string(clusterSummary.UID))) + Expect(healthCheck.Annotations).To(HaveKeyWithValue( + configv1beta1.HealthMonitoringClusterSummaryNamespaceAnnotation, clusterSummary.Namespace)) + Expect(healthCheck.Annotations).To(HaveKeyWithValue( + configv1beta1.HealthMonitoringClusterSummaryNameAnnotation, clusterSummary.Name)) + + Expect(healthCheck.Spec.EvaluateIndependently).To(BeTrue()) + Expect(healthCheck.Spec.Flapping).ToNot(BeNil()) + Expect(healthCheck.Spec.Flapping.ConsecutiveEvaluations).To(Equal(int32(controllers.HealthMonitoringFlappingEvaluation))) + // 5 kinds for resources, 5 kinds for the one Helm namespace + Expect(healthCheck.Spec.ResourceSelectors).To(HaveLen(10)) + Expect(healthCheck.Spec.EvaluateHealth).To(ContainSubstring(`ownerName = "platform-stack"`)) + Expect(healthCheck.Spec.EvaluateHealth).To(ContainSubstring(`["cert-manager/cert-manager"] = true`)) + Expect(healthCheck.Spec.EvaluateHealth).ToNot(ContainSubstring(testReleaseNameKyverno)) + }) + + It("builds no HealthCheck when the ClusterSummary deploys nothing to monitor", func() { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + Expect(controllers.BuildMonitoringHealthCheck(clusterSummary, hmOwnerKind, hmOwnerName)).To(BeNil()) + }) + + It("builds a ClusterHealthCheck targeting only the cluster of the ClusterSummary", func() { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + + clusterHealthCheck := controllers.BuildMonitoringClusterHealthCheck(clusterSummary, "hm-123", + []string{"zeta-check", "alpha-check"}) + + Expect(clusterHealthCheck.Name).To(Equal(controllers.GetHealthMonitoringName(clusterSummary))) + Expect(clusterHealthCheck.Spec.ClusterRefs).To(Equal([]corev1.ObjectReference{ + { + APIVersion: libsveltosv1beta1.GroupVersion.String(), Kind: libsveltosv1beta1.SveltosClusterKind, + Namespace: clusterSummary.Spec.ClusterNamespace, Name: clusterSummary.Spec.ClusterName, + }, + })) + // An empty selector matches no cluster: only ClusterRefs apply + Expect(clusterHealthCheck.Spec.ClusterSelector.MatchLabels).To(BeEmpty()) + Expect(clusterHealthCheck.Spec.ClusterSelector.MatchExpressions).To(BeEmpty()) + // Required by the CRD: must be an empty list, not nil (nil is serialized as null) + Expect(clusterHealthCheck.Spec.Notifications).ToNot(BeNil()) + Expect(clusterHealthCheck.Spec.Notifications).To(BeEmpty()) + + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(3)) + Expect(clusterHealthCheck.Spec.LivenessChecks[0].Name).To(Equal(controllers.DefaultHealthCheckLivenessName)) + Expect(clusterHealthCheck.Spec.LivenessChecks[0].LivenessSourceRef.Name).To(Equal("hm-123")) + Expect(clusterHealthCheck.Spec.LivenessChecks[1].LivenessSourceRef.Name).To(Equal("alpha-check")) + Expect(clusterHealthCheck.Spec.LivenessChecks[2].LivenessSourceRef.Name).To(Equal("zeta-check")) + for i := range clusterHealthCheck.Spec.LivenessChecks { + Expect(clusterHealthCheck.Spec.LivenessChecks[i].Type).To(Equal(libsveltosv1beta1.LivenessTypeHealthCheck)) + Expect(clusterHealthCheck.Spec.LivenessChecks[i].LivenessSourceRef.Kind).To(Equal(libsveltosv1beta1.HealthCheckKind)) + } + }) + + It("targets a CAPI cluster with a CAPI cluster reference", func() { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeCapi) + + clusterHealthCheck := controllers.BuildMonitoringClusterHealthCheck(clusterSummary, "hm-123", nil) + + Expect(clusterHealthCheck.Spec.ClusterRefs).To(Equal([]corev1.ObjectReference{ + { + APIVersion: clusterv1.GroupVersion.String(), Kind: "Cluster", + Namespace: clusterSummary.Spec.ClusterNamespace, Name: clusterSummary.Spec.ClusterName, + }, + })) + }) +}) + +var _ = Describe("Health monitoring: reconciliation", func() { + var ( + logger logr.Logger + c client.Client + r *controllers.ClusterSummaryReconciler + ) + + BeforeEach(func() { + logger = textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1))) + c = fake.NewClientBuilder().WithScheme(scheme).Build() + r = getClusterSummaryReconciler(c, fakedeployer.GetClient(context.TODO(), logger, c)) + }) + + monitoredClusterSummary := func() *configv1beta1.ClusterSummary { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + addHelmReleases(clusterSummary, hmCertManagerRelease) + return clusterSummary + } + + getHealthCheck := func(clusterSummary *configv1beta1.ClusterSummary) (*libsveltosv1beta1.HealthCheck, error) { + healthCheck := &libsveltosv1beta1.HealthCheck{} + err := c.Get(context.TODO(), types.NamespacedName{Name: controllers.GetHealthMonitoringName(clusterSummary)}, healthCheck) + return healthCheck, err + } + + getClusterHealthCheck := func(clusterSummary *configv1beta1.ClusterSummary) (*libsveltosv1beta1.ClusterHealthCheck, error) { + clusterHealthCheck := &libsveltosv1beta1.ClusterHealthCheck{} + err := c.Get(context.TODO(), + types.NamespacedName{Name: controllers.GetHealthMonitoringName(clusterSummary)}, clusterHealthCheck) + return clusterHealthCheck, err + } + + It("creates nothing while features are not all provisioned", func() { + clusterSummary := monitoredClusterSummary() + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + {FeatureID: libsveltosv1beta1.FeatureResources, Status: libsveltosv1beta1.FeatureStatusProvisioning}, + } + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + _, err = getClusterHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("creates the HealthCheck and the ClusterHealthCheck once all features are provisioned", func() { + clusterSummary := monitoredClusterSummary() + provisionAllFeatures(clusterSummary) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + healthCheck, err := getHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(healthCheck.Spec.ResourceSelectors).To(HaveLen(5)) + + clusterHealthCheck, err := getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(1)) + Expect(clusterHealthCheck.Spec.LivenessChecks[0].LivenessSourceRef.Name).To(Equal(healthCheck.Name)) + }) + + It("keeps the resources, and updates the HealthCheck, when the deployed Helm releases change", func() { + clusterSummary := monitoredClusterSummary() + provisionAllFeatures(clusterSummary) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + // The ClusterSummary is deploying again: nothing is touched + clusterSummary.Status.FeatureSummaries[0].Status = libsveltosv1beta1.FeatureStatusProvisioning + addHelmReleases(clusterSummary, testReleaseNameKyverno+"/"+testReleaseNameKyverno) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + healthCheck, err := getHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(healthCheck.Spec.ResourceSelectors).To(HaveLen(5)) + + // Provisioned again: the new release is monitored + provisionAllFeatures(clusterSummary) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + healthCheck, err = getHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(healthCheck.Spec.ResourceSelectors).To(HaveLen(10)) + Expect(healthCheck.Spec.EvaluateHealth).To(ContainSubstring(`["` + testReleaseNameKyverno + `/` + testReleaseNameKyverno + `"] = true`)) + }) + + It("monitors only the listed HealthChecks when defaultChecks is false", func() { + clusterSummary := monitoredClusterSummary() + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.DefaultChecks = ptr.To(false) + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.HealthChecks = []string{hmUserHealthCheck} + provisionAllFeatures(clusterSummary) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + + clusterHealthCheck, err := getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(1)) + Expect(clusterHealthCheck.Spec.LivenessChecks[0].LivenessSourceRef.Name).To(Equal(hmUserHealthCheck)) + }) + + It("monitors the default and the listed HealthChecks, and removes the generated one when not needed anymore", func() { + clusterSummary := monitoredClusterSummary() + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.HealthChecks = []string{hmUserHealthCheck} + provisionAllFeatures(clusterSummary) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + clusterHealthCheck, err := getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(2)) + _, err = getHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.DefaultChecks = ptr.To(false) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + clusterHealthCheck, err = getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(1)) + _, err = getHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("creates nothing when there is nothing to monitor", func() { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + provisionAllFeatures(clusterSummary) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getClusterHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("removes the resources when monitoring is disabled", func() { + clusterSummary := monitoredClusterSummary() + provisionAllFeatures(clusterSummary) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + // A HealthCheck written by the user must not be touched + userHealthCheck := &libsveltosv1beta1.HealthCheck{ObjectMeta: metav1.ObjectMeta{Name: hmUserHealthCheck}} + Expect(c.Create(context.TODO(), userHealthCheck)).To(Succeed()) + + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.Enabled = false + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + _, err = getClusterHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + Expect(c.Get(context.TODO(), client.ObjectKeyFromObject(userHealthCheck), userHealthCheck)).To(Succeed()) + }) + + It("removeHealthMonitoring deletes only what was created for the ClusterSummary", func() { + clusterSummary := monitoredClusterSummary() + provisionAllFeatures(clusterSummary) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + // Same name, but created for another ClusterSummary (different UID label) + other := clusterSummary.DeepCopy() + other.UID = types.UID(randomString()) + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), other, logger)).To(Succeed()) + _, err := getHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + _, err = getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + _, err = getHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + _, err = getClusterHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + + // Nothing left to remove is not an error + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + }) + + It("does nothing for a DryRun ClusterSummary", func() { + clusterSummary := monitoredClusterSummary() + clusterSummary.Spec.ClusterProfileSpec.SyncMode = configv1beta1.SyncModeDryRun + provisionAllFeatures(clusterSummary) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getClusterHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) +}) + +var _ = Describe("Health monitoring: HealthChecks of a Profile", func() { + var ( + logger logr.Logger + c client.Client + r *controllers.ClusterSummaryReconciler + recorder *events.FakeRecorder + ) + + BeforeEach(func() { + logger = textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1))) + c = fake.NewClientBuilder().WithScheme(scheme).Build() + r = getClusterSummaryReconciler(c, fakedeployer.GetClient(context.TODO(), logger, c)) + recorder = events.NewFakeRecorder(10) + controllers.SetEventRecorder(r, recorder) + }) + + // newProvisionedClusterSummary returns a provisioned ClusterSummary, owned by a profile of kind + // profileKind, listing a HealthCheck + newProvisionedClusterSummary := func(profileKind string) *configv1beta1.ClusterSummary { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + clusterSummary.OwnerReferences[0].Kind = profileKind + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{ + Enabled: true, HealthChecks: []string{hmUserHealthCheck}, + } + addHelmReleases(clusterSummary, hmCertManagerRelease) + provisionAllFeatures(clusterSummary) + return clusterSummary + } + + getClusterHealthCheck := func(clusterSummary *configv1beta1.ClusterSummary) (*libsveltosv1beta1.ClusterHealthCheck, error) { + clusterHealthCheck := &libsveltosv1beta1.ClusterHealthCheck{} + err := c.Get(context.TODO(), + types.NamespacedName{Name: controllers.GetHealthMonitoringName(clusterSummary)}, clusterHealthCheck) + return clusterHealthCheck, err + } + + It("a Profile gets the default checks only, and a warning explains why the HealthChecks are ignored", func() { + clusterSummary := newProvisionedClusterSummary(configv1beta1.ProfileKind) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + clusterHealthCheck, err := getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(1)) + Expect(clusterHealthCheck.Spec.LivenessChecks[0].Name).To(Equal(controllers.DefaultHealthCheckLivenessName)) + Expect(clusterHealthCheck.Spec.LivenessChecks[0].LivenessSourceRef.Name).ToNot(Equal(hmUserHealthCheck)) + + Expect(recorder.Events).To(Receive(And( + ContainSubstring("Warning HealthChecksIgnored"), ContainSubstring("only a ClusterProfile")))) + }) + + It("a Profile listing HealthChecks, with defaultChecks false, monitors nothing", func() { + clusterSummary := newProvisionedClusterSummary(configv1beta1.ProfileKind) + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.DefaultChecks = ptr.To(false) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getClusterHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + Expect(recorder.Events).To(Receive(ContainSubstring("Warning HealthChecksIgnored"))) + }) + + It("a Profile listing no HealthChecks gets no warning", func() { + clusterSummary := newProvisionedClusterSummary(configv1beta1.ProfileKind) + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.HealthChecks = nil + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(recorder.Events).ToNot(Receive()) + }) + + It("a ClusterProfile keeps its HealthChecks and gets no warning", func() { + clusterSummary := newProvisionedClusterSummary(configv1beta1.ClusterProfileKind) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + clusterHealthCheck, err := getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(2)) + Expect(recorder.Events).ToNot(Receive()) + }) +}) + +var _ = Describe("Health monitoring: ownership and failures", func() { + var ( + logger logr.Logger + c client.Client + r *controllers.ClusterSummaryReconciler + ) + + BeforeEach(func() { + logger = textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1))) + }) + + newReconciler := func(funcs interceptor.Funcs) { + c = fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(funcs).Build() + r = getClusterSummaryReconciler(c, fakedeployer.GetClient(context.TODO(), logger, c)) + } + + newProvisionedClusterSummary := func() *configv1beta1.ClusterSummary { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + addHelmReleases(clusterSummary, hmCertManagerRelease) + provisionAllFeatures(clusterSummary) + return clusterSummary + } + + nameOf := func(clusterSummary *configv1beta1.ClusterSummary) types.NamespacedName { + return types.NamespacedName{Name: controllers.GetHealthMonitoringName(clusterSummary)} + } + + It("never deletes an object with the generated name which another ClusterSummary created", func() { + newReconciler(interceptor.Funcs{}) + clusterSummary := newProvisionedClusterSummary() + + // No label at all, and the label of another ClusterSummary + for _, labels := range []map[string]string{ + nil, + {configv1beta1.HealthMonitoringClusterSummaryUIDLabel: randomString()}, + } { + healthCheck := &libsveltosv1beta1.HealthCheck{ + ObjectMeta: metav1.ObjectMeta{Name: nameOf(clusterSummary).Name, Labels: labels}} + clusterHealthCheck := &libsveltosv1beta1.ClusterHealthCheck{ + ObjectMeta: metav1.ObjectMeta{Name: nameOf(clusterSummary).Name, Labels: labels}} + Expect(c.Create(context.TODO(), healthCheck)).To(Succeed()) + Expect(c.Create(context.TODO(), clusterHealthCheck)).To(Succeed()) + + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + Expect(c.Get(context.TODO(), nameOf(clusterSummary), &libsveltosv1beta1.HealthCheck{})).To(Succeed()) + Expect(c.Get(context.TODO(), nameOf(clusterSummary), &libsveltosv1beta1.ClusterHealthCheck{})).To(Succeed()) + + Expect(c.Delete(context.TODO(), healthCheck)).To(Succeed()) + Expect(c.Delete(context.TODO(), clusterHealthCheck)).To(Succeed()) + } + }) + + It("does not touch the objects of another ClusterSummary when monitoring is disabled", func() { + newReconciler(interceptor.Funcs{}) + clusterSummary := newProvisionedClusterSummary() + clusterHealthCheck := &libsveltosv1beta1.ClusterHealthCheck{ObjectMeta: metav1.ObjectMeta{ + Name: nameOf(clusterSummary).Name, + Labels: map[string]string{configv1beta1.HealthMonitoringClusterSummaryUIDLabel: randomString()}, + }} + Expect(c.Create(context.TODO(), clusterHealthCheck)).To(Succeed()) + + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.Enabled = false + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + Expect(c.Get(context.TODO(), nameOf(clusterSummary), &libsveltosv1beta1.ClusterHealthCheck{})).To(Succeed()) + }) + + It("creates nothing for a ClusterSummary being deleted", func() { + newReconciler(interceptor.Funcs{}) + clusterSummary := newProvisionedClusterSummary() + clusterSummary.DeletionTimestamp = &metav1.Time{Time: time.Now()} + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + err := c.Get(context.TODO(), nameOf(clusterSummary), &libsveltosv1beta1.ClusterHealthCheck{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("creates nothing for a ClusterSummary with no feature summaries", func() { + newReconciler(interceptor.Funcs{}) + clusterSummary := newProvisionedClusterSummary() + clusterSummary.Status.FeatureSummaries = nil + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + err := c.Get(context.TODO(), nameOf(clusterSummary), &libsveltosv1beta1.ClusterHealthCheck{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("fails, creating nothing, when the profile owning the ClusterSummary cannot be found", func() { + newReconciler(interceptor.Funcs{}) + clusterSummary := newProvisionedClusterSummary() + clusterSummary.OwnerReferences = nil + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).ToNot(Succeed()) + + err := c.Get(context.TODO(), nameOf(clusterSummary), &libsveltosv1beta1.ClusterHealthCheck{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("builds the same HealthCheck whatever the order of the Helm releases", func() { + first := newProvisionedClusterSummary() + first.Status.HelmReleaseSummaries = nil + addHelmReleases(first, hmCertManagerRelease, hmGrafanaRelease, "monitoring/loki") + + second := first.DeepCopy() + second.Status.HelmReleaseSummaries = nil + addHelmReleases(second, "monitoring/loki", hmGrafanaRelease, hmCertManagerRelease) + + firstHealthCheck := controllers.BuildMonitoringHealthCheck(first, hmOwnerKind, hmOwnerName) + secondHealthCheck := controllers.BuildMonitoringHealthCheck(second, hmOwnerKind, hmOwnerName) + Expect(firstHealthCheck.Spec).To(Equal(secondHealthCheck.Spec)) + }) + + It("removing is not an error when the HealthCheck CRDs are not installed", func() { + newReconciler(interceptor.Funcs{ + Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, _ client.Object, + _ ...client.GetOption) error { + + return &meta.NoKindMatchError{} + }, + }) + + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), newProvisionedClusterSummary(), logger)).To(Succeed()) + }) + + It("removing fails when the object cannot be read", func() { + newReconciler(interceptor.Funcs{ + Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, _ client.Object, + _ ...client.GetOption) error { + + return errors.New("read failure") + }, + }) + + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), newProvisionedClusterSummary(), logger)).ToNot(Succeed()) + }) + + It("removing fails when an object cannot be deleted, and ignores one deleted in the meantime", func() { + deleteError := errors.New("delete failure") + newReconciler(interceptor.Funcs{ + Delete: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.DeleteOption) error { + return deleteError + }, + }) + clusterSummary := newProvisionedClusterSummary() + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), clusterSummary, logger)).ToNot(Succeed()) + + deleteError = apierrors.NewNotFound(schema.GroupResource{}, "deleted already") + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + }) +}) + +var _ = Describe("Health monitoring: validation of healthMonitoring", func() { + // createClusterProfile creates a ClusterProfile in the API server, so the validation rules of the CRD apply + createClusterProfile := func(healthMonitoring *configv1beta1.HealthMonitoring) error { + clusterProfile := &configv1beta1.ClusterProfile{ + ObjectMeta: metav1.ObjectMeta{Name: randomString()}, + Spec: configv1beta1.Spec{HealthMonitoring: healthMonitoring}, + } + err := testEnv.Create(context.TODO(), clusterProfile) + if err == nil { + DeferCleanup(func() { Expect(testEnv.Delete(context.TODO(), clusterProfile)).To(Succeed()) }) + } + return err + } + + It("accepts valid combinations", func() { + Expect(createClusterProfile(nil)).To(Succeed()) + Expect(createClusterProfile(&configv1beta1.HealthMonitoring{Enabled: false})).To(Succeed()) + Expect(createClusterProfile(&configv1beta1.HealthMonitoring{Enabled: true})).To(Succeed()) + Expect(createClusterProfile(&configv1beta1.HealthMonitoring{ + Enabled: true, HealthChecks: []string{hmUserHealthCheck}})).To(Succeed()) + Expect(createClusterProfile(&configv1beta1.HealthMonitoring{ + Enabled: true, DefaultChecks: ptr.To(false), HealthChecks: []string{hmUserHealthCheck}})).To(Succeed()) + }) + + It("rejects healthChecks when monitoring is not enabled", func() { + err := createClusterProfile(&configv1beta1.HealthMonitoring{ + Enabled: false, HealthChecks: []string{hmUserHealthCheck}}) + Expect(err).ToNot(BeNil()) + Expect(err.Error()).To(ContainSubstring("healthChecks can only be set when enabled is true")) + }) + + It("rejects defaultChecks false without healthChecks, as nothing would be monitored", func() { + err := createClusterProfile(&configv1beta1.HealthMonitoring{Enabled: true, DefaultChecks: ptr.To(false)}) + Expect(err).ToNot(BeNil()) + Expect(err.Error()).To(ContainSubstring("defaultChecks cannot be false when healthChecks is empty")) + }) +}) + +var _ = Describe("Health monitoring: ClusterSummary predicate", func() { + var predicate controllers.ClusterSummaryPredicate + + BeforeEach(func() { + predicate = controllers.ClusterSummaryPredicate{ + Logger: textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1))), + } + }) + + It("reconciles when all features become provisioned and health monitoring is enabled", func() { + oldClusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + oldClusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + oldClusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioning}, + } + + newClusterSummary := oldClusterSummary.DeepCopy() + newClusterSummary.Status.FeatureSummaries[0].Status = libsveltosv1beta1.FeatureStatusProvisioned + + Expect(predicate.Update(event.UpdateEvent{ObjectOld: oldClusterSummary, ObjectNew: newClusterSummary})).To(BeTrue()) + }) + + It("reconciles when the Helm releases managed by the ClusterSummary change", func() { + oldClusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + oldClusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + provisionAllFeatures(oldClusterSummary) + addHelmReleases(oldClusterSummary, hmCertManagerRelease) + + newClusterSummary := oldClusterSummary.DeepCopy() + addHelmReleases(newClusterSummary, testReleaseNameKyverno+"/"+testReleaseNameKyverno) + + Expect(predicate.Update(event.UpdateEvent{ObjectOld: oldClusterSummary, ObjectNew: newClusterSummary})).To(BeTrue()) + }) + + It("does not reconcile for a status change that does not matter to health monitoring", func() { + oldClusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + oldClusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + provisionAllFeatures(oldClusterSummary) + + newClusterSummary := oldClusterSummary.DeepCopy() + message := "something" + newClusterSummary.Status.Dependencies = &message + + Expect(predicate.Update(event.UpdateEvent{ObjectOld: oldClusterSummary, ObjectNew: newClusterSummary})).To(BeFalse()) + }) + + It("keeps ignoring features becoming provisioned when health monitoring is not enabled", func() { + oldClusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + oldClusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioning}, + } + + newClusterSummary := oldClusterSummary.DeepCopy() + newClusterSummary.Status.FeatureSummaries[0].Status = libsveltosv1beta1.FeatureStatusProvisioned + + Expect(predicate.Update(event.UpdateEvent{ObjectOld: oldClusterSummary, ObjectNew: newClusterSummary})).To(BeFalse()) + }) +}) diff --git a/controllers/healthmonitoring/health.lua b/controllers/healthmonitoring/health.lua new file mode 100644 index 00000000..4dc995eb --- /dev/null +++ b/controllers/healthmonitoring/health.lua @@ -0,0 +1,224 @@ +-- Sveltos health monitoring: health of the Deployments, StatefulSets, DaemonSets, Jobs and +-- PersistentVolumeClaims deployed by a profile. +-- +-- addon-controller prepends a "config" table to this script: +-- config.ownerKind ClusterProfile or Profile +-- config.ownerName name of the profile +-- config.helmReleases set of "/" for the Helm charts of the profile +-- +-- The health rules follow the ones of Argo CD (gitops-engine, pkg/health, Apache-2.0). + +local function isOwned(resource) + local annotations = (resource.metadata or {}).annotations or {} + + -- Resources deployed by Sveltos (raw YAML, Kustomize, ...) carry the owner annotations + if annotations["projectsveltos.io/owner-kind"] == config.ownerKind and + annotations["projectsveltos.io/owner-name"] == config.ownerName then + return true + end + + -- Resources deployed by a Helm chart carry the Helm release annotations + local release = annotations["meta.helm.sh/release-name"] + local namespace = annotations["meta.helm.sh/release-namespace"] + if release ~= nil and namespace ~= nil then + return config.helmReleases[namespace .. "/" .. release] == true + end + + return false +end + +local function deploymentHealth(resource) + local metadata = resource.metadata or {} + local spec = resource.spec or {} + local status = resource.status or {} + + if spec.paused then + return "Suspended", "Deployment is paused" + end + + if (metadata.generation or 0) > (status.observedGeneration or 0) then + return "Progressing", "Waiting for rollout to finish: observed deployment generation less than desired generation" + end + + for _, condition in ipairs(status.conditions or {}) do + if condition.type == "Progressing" and condition.reason == "ProgressDeadlineExceeded" then + return "Degraded", string.format("Deployment %q exceeded its progress deadline", metadata.name or "") + end + end + + local desired = spec.replicas or 1 + local updated = status.updatedReplicas or 0 + local current = status.replicas or 0 + local available = status.availableReplicas or 0 + + if updated < desired then + return "Progressing", string.format( + "Waiting for rollout to finish: %d out of %d new replicas have been updated", updated, desired) + end + if current > updated then + return "Progressing", string.format( + "Waiting for rollout to finish: %d old replicas are pending termination", current - updated) + end + if available < updated then + return "Progressing", string.format( + "Waiting for rollout to finish: %d of %d updated replicas are available", available, updated) + end + + return "Healthy", "" +end + +local function statefulSetHealth(resource) + local metadata = resource.metadata or {} + local spec = resource.spec or {} + local status = resource.status or {} + + local observed = status.observedGeneration or 0 + if observed == 0 or (metadata.generation or 0) > observed then + return "Progressing", "Waiting for statefulset spec update to be observed" + end + + local ready = status.readyReplicas or 0 + if spec.replicas ~= nil and ready < spec.replicas then + return "Progressing", string.format("Waiting for %d pods to be ready", spec.replicas - ready) + end + + local strategy = spec.updateStrategy or {} + if strategy.type == "OnDelete" then + return "Healthy", "" + end + + if strategy.type == "RollingUpdate" and strategy.rollingUpdate ~= nil then + local partition = strategy.rollingUpdate.partition + if partition ~= nil and spec.replicas ~= nil then + local expected = spec.replicas - partition + local updated = status.updatedReplicas or 0 + if updated < expected then + return "Progressing", string.format( + "Waiting for partitioned roll out to finish: %d out of %d new pods have been updated", updated, expected) + end + end + return "Healthy", "" + end + + if status.updateRevision ~= status.currentRevision then + return "Progressing", string.format("Waiting for rolling update to finish: current revision %s, update revision %s", + tostring(status.currentRevision), tostring(status.updateRevision)) + end + + return "Healthy", "" +end + +local function daemonSetHealth(resource) + local metadata = resource.metadata or {} + local spec = resource.spec or {} + local status = resource.status or {} + + if (metadata.generation or 0) > (status.observedGeneration or 0) then + return "Progressing", "Waiting for rollout to finish: observed daemon set generation less than desired generation" + end + + local strategy = spec.updateStrategy or {} + if strategy.type == "OnDelete" then + return "Healthy", "" + end + + local desired = status.desiredNumberScheduled or 0 + local updated = status.updatedNumberScheduled or 0 + local available = status.numberAvailable or 0 + + if updated < desired then + return "Progressing", string.format( + "Waiting for rollout to finish: %d out of %d new pods have been updated", updated, desired) + end + if available < desired then + return "Progressing", string.format( + "Waiting for rollout to finish: %d of %d updated pods are available", available, desired) + end + + return "Healthy", "" +end + +-- A Job is finished once the Job controller adds a Failed or a Complete condition. Until then it is +-- Progressing, a Job that is running is reported like Argo CD does. +local function jobHealth(resource) + local status = resource.status or {} + + local complete = false + local suspended = false + for _, condition in ipairs(status.conditions or {}) do + if condition.status == "True" then + if condition.type == "Failed" then + return "Degraded", string.format("Job failed: %s", condition.message or condition.reason or "unknown reason") + elseif condition.type == "Complete" then + complete = true + elseif condition.type == "Suspended" then + suspended = true + end + end + end + + if complete then + return "Healthy", "" + end + if suspended then + return "Suspended", "Job is suspended" + end + + return "Progressing", "Job has not completed yet" +end + +-- A PersistentVolumeClaim whose storage class waits for the first consumer stays Pending until a Pod +-- uses it: the phase alone cannot tell it from a claim that cannot be bound. +local function persistentVolumeClaimHealth(resource) + local phase = (resource.status or {}).phase + + if phase == "Bound" then + return "Healthy", "" + elseif phase == "Lost" then + return "Degraded", "PersistentVolumeClaim lost its PersistentVolume" + elseif phase == "Pending" then + return "Progressing", "PersistentVolumeClaim is not bound yet" + end + + return "Progressing", string.format("PersistentVolumeClaim has no known phase yet (%s)", tostring(phase)) +end + +local healthByKind = { + Deployment = deploymentHealth, + StatefulSet = statefulSetHealth, + DaemonSet = daemonSetHealth, + Job = jobHealth, + PersistentVolumeClaim = persistentVolumeClaimHealth, +} + +function evaluate() + local hs = {} + hs.status = "Healthy" + hs.message = "" + + local unhealthy = {} + -- The same object can be selected twice: by label (deployed by Sveltos) and by namespace (Helm) + local seen = {} + + for _, resource in ipairs(resources) do + local uid = (resource.metadata or {}).uid + local health = healthByKind[resource.kind] + + if health ~= nil and (uid == nil or not seen[uid]) and isOwned(resource) then + if uid ~= nil then + seen[uid] = true + end + + local status, message = health(resource) + if status ~= "Healthy" then + table.insert(unhealthy, {resource = resource, status = status, message = message}) + end + end + end + + if #unhealthy > 0 then + hs.resources = unhealthy + end + + return hs +end diff --git a/go.mod b/go.mod index 06380685..9a10cfc1 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/onsi/gomega v1.44.0 github.com/opencontainers/image-spec v1.1.1 github.com/pkg/errors v0.9.1 - github.com/projectsveltos/libsveltos v1.16.1-0.20261011133909-b577d6e6fe69 + github.com/projectsveltos/libsveltos v1.16.1-0.20261014064744-50f7977def53 github.com/prometheus/client_golang v1.25.0 github.com/sigstore/cosign/v3 v3.1.3 github.com/sigstore/sigstore v1.11.0 diff --git a/go.sum b/go.sum index c1c4d0c0..516bff9c 100644 --- a/go.sum +++ b/go.sum @@ -643,8 +643,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= -github.com/projectsveltos/libsveltos v1.16.1-0.20261011133909-b577d6e6fe69 h1:rg2W4If7Ejm/3OOLeGiu8GywpK8nFEOfbTsFdVHG+Ho= -github.com/projectsveltos/libsveltos v1.16.1-0.20261011133909-b577d6e6fe69/go.mod h1:wup25BAf1SHuG6FsY8D+tnm1U3/3NiuzcKec/In/fB4= +github.com/projectsveltos/libsveltos v1.16.1-0.20261014064744-50f7977def53 h1:7RH3CeioAs44oednP7fME8cDn51tNVXJIZHWLaUYhdw= +github.com/projectsveltos/libsveltos v1.16.1-0.20261014064744-50f7977def53/go.mod h1:wup25BAf1SHuG6FsY8D+tnm1U3/3NiuzcKec/In/fB4= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5 h1:khnc+994UszxZYu69J+R5FKiLA/Nk1JQj0EYAkwTWz0= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5/go.mod h1:yVL8KQFa9tmcxgwl9nwIMtKgtmIVC1zaFRSCfOwYvPY= github.com/projectsveltos/lua-utils/glua-runes v0.0.0-20251212200258-2b3cdcb7c0f5 h1:YbsebwRwTRhV8QacvEAdFqxcxHdeu7JTVtsBovbkgos= diff --git a/lib/crd/clusterprofiles.go b/lib/crd/clusterprofiles.go index be9c350f..7e32a1bc 100644 --- a/lib/crd/clusterprofiles.go +++ b/lib/crd/clusterprofiles.go @@ -256,6 +256,55 @@ spec: ` + "`" + `ExtraLabels` + "`" + `, the value from ` + "`" + `ExtraLabels` + "`" + ` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is empty: + there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/lib/crd/clusterpromotions.go b/lib/crd/clusterpromotions.go index b41a067a..09c24c74 100644 --- a/lib/crd/clusterpromotions.go +++ b/lib/crd/clusterpromotions.go @@ -155,6 +155,55 @@ spec: type: object type: array x-kubernetes-list-type: atomic + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is + empty: there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/lib/crd/clustersummaries.go b/lib/crd/clustersummaries.go index 99b608b2..145251b9 100644 --- a/lib/crd/clustersummaries.go +++ b/lib/crd/clustersummaries.go @@ -292,6 +292,55 @@ spec: ` + "`" + `ExtraLabels` + "`" + `, the value from ` + "`" + `ExtraLabels` + "`" + ` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is + empty: there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/lib/crd/profiles.go b/lib/crd/profiles.go index b31ba277..e8012088 100644 --- a/lib/crd/profiles.go +++ b/lib/crd/profiles.go @@ -256,6 +256,55 @@ spec: ` + "`" + `ExtraLabels` + "`" + `, the value from ` + "`" + `ExtraLabels` + "`" + ` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is empty: + there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml index 51c3b551..41151bc5 100644 --- a/manifest/manifest.yaml +++ b/manifest/manifest.yaml @@ -546,6 +546,55 @@ spec: `ExtraLabels`, the value from `ExtraLabels` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is empty: + there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed @@ -3041,6 +3090,55 @@ spec: type: object type: array x-kubernetes-list-type: atomic + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is + empty: there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed @@ -6712,6 +6810,55 @@ spec: `ExtraLabels`, the value from `ExtraLabels` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is + empty: there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed @@ -9378,6 +9525,55 @@ spec: `ExtraLabels`, the value from `ExtraLabels` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is empty: + there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed @@ -11930,21 +12126,10 @@ rules: - lib.projectsveltos.io resources: - clusterhealthchecks - - configurationgroups/status - - debuggingconfigurations - - eventtriggers - - sveltosclusters - - sveltosclusters/status - verbs: - - get - - list - - watch -- apiGroups: - - lib.projectsveltos.io - resources: - clustersets - configurationbundles - configurationgroups + - healthchecks - reloaders - resourcesummaries - sets @@ -11981,6 +12166,18 @@ rules: - list - update - watch +- apiGroups: + - lib.projectsveltos.io + resources: + - configurationgroups/status + - debuggingconfigurations + - eventtriggers + - sveltosclusters + - sveltosclusters/status + verbs: + - get + - list + - watch - apiGroups: - lib.projectsveltos.io resources: