From cef897a9802db5a60a940fed3d430351cfb541d7 Mon Sep 17 00:00:00 2001 From: Gianluca Mardente Date: Thu, 15 Oct 2026 08:38:18 +0200 Subject: [PATCH] feat: Health monitoring of the resources a profile deploys Adds an opt-in `healthMonitoring` section to the shared profile spec (ClusterProfile, Profile, ClusterPromotion): ```yaml spec: healthMonitoring: enabled: true defaultChecks: true # default healthChecks: [] # ClusterProfile only ``` When enabled, once every feature of a ClusterSummary is Provisioned, addon-controller creates one HealthCheck and one ClusterHealthCheck for that ClusterSummary. The existing pipeline does the rest: healthcheck-manager deploys the HealthCheck to the managed cluster, sveltos-agent evaluates it, and the HealthCheckReports reach the management cluster (also in pull mode and agentless). This answers "what is unhealthy on this cluster among what Sveltos deployed", separately from `validateHealths`, which gates a ClusterSummary reaching Provisioned at deploy time only. - Default checks cover Deployment, StatefulSet, DaemonSet, Job and PersistentVolumeClaim. The Lua rules are embedded (`controllers/healthmonitoring/health.lua`) and report Healthy, Progressing, Degraded or Suspended. - Only resources deployed by the profile are evaluated. Raw resources are selected by the `projectsveltos.io/reason` label and matched on the owner kind and name annotations. Helm chart content is matched on the Helm release annotations of the releases the ClusterSummary manages. - `healthChecks` adds user-written HealthChecks to the same ClusterHealthCheck. `defaultChecks: false` monitors only those. - Only a ClusterProfile can set `healthChecks`. A HealthCheck is cluster scoped and referenced by name, and its report is shared by every profile referencing it on a cluster, so a namespaced Profile could learn about resources outside its namespace. For a Profile the list is ignored, the default checks apply, and a Warning event `HealthChecksIgnored` says so. CRD validation rejects `healthChecks` when `enabled` is false and `defaultChecks: false` with no `healthChecks`. - The generated objects are named `hm-<12 hex of the SHA-256 of the ClusterSummary UID>`, carry the ClusterSummary UID label and namespace/name annotations, and are removed when monitoring is disabled, when nothing is left to monitor, or when the ClusterSummary is deleted. Removal only touches objects carrying that ClusterSummary's UID label, so a user's HealthCheck is never deleted. - Health monitoring is reconciled before `shouldReconcile`, so it works for a OneTime ClusterSummary with nothing left to deploy. The ClusterSummary update predicate reacts to features becoming Provisioned and to changes in the managed Helm releases. - DryRun ClusterSummaries are skipped. - A HealthCheck only evaluates objects that exist: a deleted resource is not reported (drift detection covers that). - Helm chart resources outside the release namespace are only partly matched. - sveltos-agent caches every object of each monitored kind in the cluster, about 20 KB of heap per object, independent of how many profiles enable monitoring. --- api/v1beta1/clusterpromotion_types.go | 7 + api/v1beta1/spec.go | 56 + api/v1beta1/zz_generated.deepcopy.go | 35 + ...fig.projectsveltos.io_clusterprofiles.yaml | 49 + ...g.projectsveltos.io_clusterpromotions.yaml | 49 + ...ig.projectsveltos.io_clustersummaries.yaml | 49 + .../config.projectsveltos.io_profiles.yaml | 49 + config/rbac/role.yaml | 25 +- controllers/clustersummary_controller.go | 15 + controllers/clustersummary_controller_test.go | 107 ++ controllers/clustersummary_predicates.go | 8 + controllers/export_test.go | 33 + controllers/health_monitoring.go | 534 ++++++++ controllers/health_monitoring_test.go | 1147 +++++++++++++++++ controllers/healthmonitoring/health.lua | 224 ++++ go.mod | 2 +- go.sum | 4 +- lib/crd/clusterprofiles.go | 49 + lib/crd/clusterpromotions.go | 49 + lib/crd/clustersummaries.go | 49 + lib/crd/profiles.go | 49 + manifest/manifest.yaml | 221 +++- 22 files changed, 2783 insertions(+), 27 deletions(-) create mode 100644 controllers/health_monitoring.go create mode 100644 controllers/health_monitoring_test.go create mode 100644 controllers/healthmonitoring/health.lua diff --git a/api/v1beta1/clusterpromotion_types.go b/api/v1beta1/clusterpromotion_types.go index bab8f200..7da41c23 100644 --- a/api/v1beta1/clusterpromotion_types.go +++ b/api/v1beta1/clusterpromotion_types.go @@ -170,6 +170,13 @@ type ProfileSpec struct { // +optional ValidateHealths []libsveltosv1beta1.ValidateHealth `json:"validateHealths,omitempty"` + // HealthMonitoring, when enabled, continuously monitors the health of the resources + // this profile deploys in each matching cluster. Unlike ValidateHealths, which are + // evaluated once during deployment, these checks run for as long as the profile + // is deployed. Results are available as HealthCheckReports. + // +optional + HealthMonitoring *HealthMonitoring `json:"healthMonitoring,omitempty"` + // Define additional Kustomize inline Patches applied for all resources on this profile // Within the Patch Spec you can use templating // +listType=atomic diff --git a/api/v1beta1/spec.go b/api/v1beta1/spec.go index 2372fc0b..7bcd78ff 100644 --- a/api/v1beta1/spec.go +++ b/api/v1beta1/spec.go @@ -958,6 +958,55 @@ type Clusters struct { Clusters []corev1.ObjectReference `json:"clusters,omitempty"` } +const ( + // HealthMonitoringClusterSummaryUIDLabel is set, with the UID of the ClusterSummary as value, on the + // HealthCheck and ClusterHealthCheck instances created for a ClusterSummary when health monitoring is enabled. + HealthMonitoringClusterSummaryUIDLabel = "projectsveltos.io/health-monitoring-clustersummary-uid" + + // HealthMonitoringClusterSummaryNamespaceAnnotation and HealthMonitoringClusterSummaryNameAnnotation + // are set on the same instances to identify the ClusterSummary they have been created for. + HealthMonitoringClusterSummaryNamespaceAnnotation = "projectsveltos.io/health-monitoring-clustersummary-namespace" + HealthMonitoringClusterSummaryNameAnnotation = "projectsveltos.io/health-monitoring-clustersummary-name" +) + +// HealthMonitoring configures the continuous health monitoring of the resources a profile deploys. +// Both ClusterProfile and Profile can enable it with the default checks. Referencing additional +// HealthChecks (healthChecks) is reserved to ClusterProfile: see the field. +// +// +kubebuilder:validation:XValidation:rule="self.enabled || !has(self.healthChecks) || size(self.healthChecks) == 0",message="healthChecks can only be set when enabled is true" +// +kubebuilder:validation:XValidation:rule="!self.enabled || !has(self.defaultChecks) || self.defaultChecks || (has(self.healthChecks) && size(self.healthChecks) > 0)",message="defaultChecks cannot be false when healthChecks is empty: there would be nothing to monitor" +// +//nolint:lll // kubebuilder marker +type HealthMonitoring struct { + // Enabled turns health monitoring on for the resources deployed by this profile. + // When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + // matching cluster once the profile is provisioned there, and removes them + // when monitoring is disabled or the cluster does not match anymore. + Enabled bool `json:"enabled"` + + // DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + // Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + // Set it to false to monitor only the HealthChecks listed in HealthChecks. + // +kubebuilder:default:=true + // +optional + DefaultChecks *bool `json:"defaultChecks,omitempty"` + + // HealthChecks is a list of names of HealthCheck instances to monitor in addition + // to the default checks. Those are deployed as they are: selecting only the + // resources deployed by this profile is up to the HealthCheck. + // + // Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + // default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + // report it produces is shared by every profile that references it on a given cluster, with + // no filtering by namespace. Allowing a Profile to reference one would let a tenant name + // a HealthCheck it does not own, and learn about resources outside its namespace through + // the shared report. The default checks are safe for a Profile: they select only the + // resources that profile deployed. + // +listType=set + // +optional + HealthChecks []string `json:"healthChecks,omitempty"` +} + type Spec struct { // ClusterSelector identifies clusters to associate to. // +optional @@ -1110,6 +1159,13 @@ type Spec struct { // +optional ValidateHealths []libsveltosv1beta1.ValidateHealth `json:"validateHealths,omitempty"` + // HealthMonitoring, when enabled, continuously monitors the health of the resources + // this profile deploys in each matching cluster. Unlike ValidateHealths, which are + // evaluated once during deployment, these checks run for as long as the profile + // is deployed. Results are available as HealthCheckReports. + // +optional + HealthMonitoring *HealthMonitoring `json:"healthMonitoring,omitempty"` + // Define additional Kustomize inline Patches applied for all resources on this profile // Within the Patch Spec you can use templating // +listType=atomic diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 0046326f..35a7f67e 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -854,6 +854,31 @@ func (in *HandOverError) DeepCopy() *HandOverError { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HealthMonitoring) DeepCopyInto(out *HealthMonitoring) { + *out = *in + if in.DefaultChecks != nil { + in, out := &in.DefaultChecks, &out.DefaultChecks + *out = new(bool) + **out = **in + } + if in.HealthChecks != nil { + in, out := &in.HealthChecks, &out.HealthChecks + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HealthMonitoring. +func (in *HealthMonitoring) DeepCopy() *HealthMonitoring { + if in == nil { + return nil + } + out := new(HealthMonitoring) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HelmChart) DeepCopyInto(out *HelmChart) { *out = *in @@ -1305,6 +1330,11 @@ func (in *ProfileSpec) DeepCopyInto(out *ProfileSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.HealthMonitoring != nil { + in, out := &in.HealthMonitoring, &out.HealthMonitoring + *out = new(HealthMonitoring) + (*in).DeepCopyInto(*out) + } if in.Patches != nil { in, out := &in.Patches, &out.Patches *out = make([]apiv1beta1.Patch, len(*in)) @@ -1532,6 +1562,11 @@ func (in *Spec) DeepCopyInto(out *Spec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.HealthMonitoring != nil { + in, out := &in.HealthMonitoring, &out.HealthMonitoring + *out = new(HealthMonitoring) + (*in).DeepCopyInto(*out) + } if in.Patches != nil { in, out := &in.Patches, &out.Patches *out = make([]apiv1beta1.Patch, len(*in)) diff --git a/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml b/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml index 43d9a545..38228a56 100644 --- a/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml +++ b/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml @@ -237,6 +237,55 @@ spec: `ExtraLabels`, the value from `ExtraLabels` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is empty: + there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml b/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml index 81720985..7c2d0d5f 100644 --- a/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml +++ b/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml @@ -136,6 +136,55 @@ spec: type: object type: array x-kubernetes-list-type: atomic + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is + empty: there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml index b5bcdebb..a17b9724 100644 --- a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml +++ b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml @@ -273,6 +273,55 @@ spec: `ExtraLabels`, the value from `ExtraLabels` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is + empty: there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/config/crd/bases/config.projectsveltos.io_profiles.yaml b/config/crd/bases/config.projectsveltos.io_profiles.yaml index e33f95e2..f357ecc8 100644 --- a/config/crd/bases/config.projectsveltos.io_profiles.yaml +++ b/config/crd/bases/config.projectsveltos.io_profiles.yaml @@ -237,6 +237,55 @@ spec: `ExtraLabels`, the value from `ExtraLabels` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is empty: + there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 7dd01cb0..eb34aceb 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -171,21 +171,10 @@ rules: - lib.projectsveltos.io resources: - clusterhealthchecks - - configurationgroups/status - - debuggingconfigurations - - eventtriggers - - sveltosclusters - - sveltosclusters/status - verbs: - - get - - list - - watch -- apiGroups: - - lib.projectsveltos.io - resources: - clustersets - configurationbundles - configurationgroups + - healthchecks - reloaders - resourcesummaries - sets @@ -222,6 +211,18 @@ rules: - list - update - watch +- apiGroups: + - lib.projectsveltos.io + resources: + - configurationgroups/status + - debuggingconfigurations + - eventtriggers + - sveltosclusters + - sveltosclusters/status + verbs: + - get + - list + - watch - apiGroups: - lib.projectsveltos.io resources: diff --git a/controllers/clustersummary_controller.go b/controllers/clustersummary_controller.go index b2df69c4..7f540a4c 100644 --- a/controllers/clustersummary_controller.go +++ b/controllers/clustersummary_controller.go @@ -333,6 +333,13 @@ func (r *ClusterSummaryReconciler) reconcileDelete( r.updateDeletedInstancs(clusterSummaryScope) + // HealthCheck and ClusterHealthCheck live in the management cluster: they can be removed + // whatever the state of the managed cluster is. + if err := r.removeHealthMonitoring(ctx, clusterSummaryScope.ClusterSummary, logger); err != nil { + logger.V(logs.LogInfo).Error(err, "failed to remove health monitoring resources") + return reconcile.Result{RequeueAfter: deleteRequeueAfter}, nil + } + isReady, err := r.isReady(ctx, clusterSummaryScope.ClusterSummary, logger) if err != nil { return reconcile.Result{RequeueAfter: deleteRequeueAfter}, nil @@ -507,6 +514,14 @@ func (r *ClusterSummaryReconciler) reconcileNormal(ctx context.Context, } } + // Health monitoring is independent from deployment: it must be reconciled even when + // the ClusterSummary has nothing left to deploy (see shouldReconcile) + if err := r.reconcileHealthMonitoring(ctx, clusterSummaryScope.ClusterSummary, logger); err != nil { + logger.V(logs.LogInfo).Error(err, "failed to reconcile health monitoring") + r.setNextReconcileTime(clusterSummaryScope, normalRequeueAfter) + return reconcile.Result{RequeueAfter: normalRequeueAfter}, nil + } + if !r.shouldReconcile(clusterSummaryScope, logger) { logger.V(logs.LogInfo).Info("ClusterSummary does not need a reconciliation") return reconcile.Result{}, nil diff --git a/controllers/clustersummary_controller_test.go b/controllers/clustersummary_controller_test.go index e83cadbb..f64cb0b5 100644 --- a/controllers/clustersummary_controller_test.go +++ b/controllers/clustersummary_controller_test.go @@ -18,6 +18,7 @@ package controllers_test import ( "context" + "errors" "fmt" "sync" "time" @@ -36,6 +37,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -1116,6 +1118,111 @@ var _ = Describe("ClustersummaryController", func() { Expect(result.RequeueAfter).To(BeZero()) }) + It("reconcileDelete removes the health monitoring resources, even when the cluster is not found", func() { + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + clusterSummary.Spec.ClusterProfileSpec.PolicyRefs = []configv1beta1.PolicyRef{ + {Namespace: randomString(), Name: randomString(), Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind)}, + } + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureResources, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + } + + // No cluster. + initObjects := []client.Object{ + clusterProfile, + clusterSummary, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(initObjects...).WithObjects(initObjects...).Build() + + dep := fakedeployer.GetClient(context.TODO(), textlogger.NewLogger(textlogger.NewConfig()), c) + clusterSummaryReconciler := getClusterSummaryReconciler(c, dep) + + Expect(controllers.ReconcileHealthMonitoring(clusterSummaryReconciler, context.TODO(), clusterSummary, + textlogger.NewLogger(textlogger.NewConfig()))).To(Succeed()) + key := types.NamespacedName{Name: controllers.GetHealthMonitoringName(clusterSummary)} + Expect(c.Get(context.TODO(), key, &libsveltosv1beta1.HealthCheck{})).To(Succeed()) + Expect(c.Get(context.TODO(), key, &libsveltosv1beta1.ClusterHealthCheck{})).To(Succeed()) + + clusterSummaryScope := getClusterSummaryScope(c, textlogger.NewLogger(textlogger.NewConfig()), + clusterProfile, clusterSummary) + _, err := controllers.ReconcileDelete(clusterSummaryReconciler, context.TODO(), clusterSummaryScope, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + + err = c.Get(context.TODO(), key, &libsveltosv1beta1.HealthCheck{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + err = c.Get(context.TODO(), key, &libsveltosv1beta1.ClusterHealthCheck{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("reconcileNormal reconciles health monitoring even when there is nothing to deploy", func() { + // OneTime, with nothing to deploy: shouldReconcile is false. Helm releases are still managed. + clusterSummary.Spec.ClusterProfileSpec.SyncMode = configv1beta1.SyncModeOneTime + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + } + clusterSummary.Status.HelmReleaseSummaries = []configv1beta1.HelmChartSummary{ + {ReleaseNamespace: randomString(), ReleaseName: randomString(), Status: configv1beta1.HelmChartStatusManaging}, + } + + initObjects := []client.Object{ + clusterProfile, + clusterSummary, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(initObjects...).WithObjects(initObjects...).Build() + + dep := fakedeployer.GetClient(context.TODO(), textlogger.NewLogger(textlogger.NewConfig()), c) + clusterSummaryReconciler := getClusterSummaryReconciler(c, dep) + + clusterSummaryScope := getClusterSummaryScope(c, textlogger.NewLogger(textlogger.NewConfig()), + clusterProfile, clusterSummary) + result, err := controllers.ReconcileNormal(clusterSummaryReconciler, context.TODO(), clusterSummaryScope, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + Expect(result.RequeueAfter).To(BeZero()) + + key := types.NamespacedName{Name: controllers.GetHealthMonitoringName(clusterSummary)} + Expect(c.Get(context.TODO(), key, &libsveltosv1beta1.ClusterHealthCheck{})).To(Succeed()) + }) + + It("reconcileNormal requeues, without failing, when health monitoring cannot be reconciled", func() { + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + } + clusterSummary.Status.HelmReleaseSummaries = []configv1beta1.HelmChartSummary{ + {ReleaseNamespace: randomString(), ReleaseName: randomString(), Status: configv1beta1.HelmChartStatusManaging}, + } + + initObjects := []client.Object{ + clusterProfile, + clusterSummary, + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(initObjects...).WithObjects(initObjects...). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*libsveltosv1beta1.HealthCheck); ok { + return errors.New("create failure") + } + return c.Create(ctx, obj, opts...) + }, + }).Build() + + dep := fakedeployer.GetClient(context.TODO(), textlogger.NewLogger(textlogger.NewConfig()), c) + clusterSummaryReconciler := getClusterSummaryReconciler(c, dep) + + clusterSummaryScope := getClusterSummaryScope(c, textlogger.NewLogger(textlogger.NewConfig()), + clusterProfile, clusterSummary) + result, err := controllers.ReconcileNormal(clusterSummaryReconciler, context.TODO(), clusterSummaryScope, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + Expect(result.RequeueAfter).To(Equal(controllers.NormalRequeueAfter)) + }) + It("reconcileDelete removes the DeletedInstances entry once the finalizer is removed", func() { // No cluster. initObjects := []client.Object{ diff --git a/controllers/clustersummary_predicates.go b/controllers/clustersummary_predicates.go index 119a348d..931ff85d 100644 --- a/controllers/clustersummary_predicates.go +++ b/controllers/clustersummary_predicates.go @@ -373,6 +373,14 @@ func (p ClusterSummaryPredicate) Update(e event.UpdateEvent) bool { return true } + // Feature summaries are compared by hash below, so a feature becoming Provisioned does not + // trigger a reconciliation. Health monitoring starts when all features are provisioned and + // depends on the Helm releases deployed: reconcile when any of those changes. + if isHealthMonitoringEnabled(newClusterSummary) && healthMonitoringInputsChanged(oldClusterSummary, newClusterSummary) { + log.V(logs.LogVerbose).Info("ClusterSummary state relevant to health monitoring changed. Reconciling ClusterSummary.") + return true + } + oldFS := getFeatureSummaryForFeatureID(oldClusterSummary, libsveltosv1beta1.FeatureHelm) newFS := getFeatureSummaryForFeatureID(newClusterSummary, libsveltosv1beta1.FeatureHelm) if !isFeatureSummarySame(oldFS, newFS) { diff --git a/controllers/export_test.go b/controllers/export_test.go index efed97d2..0b79821e 100644 --- a/controllers/export_test.go +++ b/controllers/export_test.go @@ -17,6 +17,8 @@ limitations under the License. package controllers import ( + "strings" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/client-go/rest" @@ -34,6 +36,7 @@ func SetEventRecorder(r *ClusterSummaryReconciler, recorder events.EventRecorder const ( PausedClustersRequeueAfter = pausedClustersRequeueAfter DeleteRequeueAfter = deleteRequeueAfter + NormalRequeueAfter = normalRequeueAfter ) var ( @@ -87,6 +90,7 @@ var ( ShouldRedeploy = (*ClusterSummaryReconciler).shouldRedeploy CanRemoveFinalizer = (*ClusterSummaryReconciler).canRemoveFinalizer ReconcileDelete = (*ClusterSummaryReconciler).reconcileDelete + ReconcileNormal = (*ClusterSummaryReconciler).reconcileNormal CleanupBeforeFinalizerRemoval = (*ClusterSummaryReconciler).cleanupBeforeFinalizerRemoval AreDependenciesDeployed = (*ClusterSummaryReconciler).areDependenciesDeployed AreDependentsRemoved = (*ClusterSummaryReconciler).areDependentsRemoved @@ -365,3 +369,32 @@ var ( func NewReleaseInfoForApply(applyHash []byte, skipApply bool) *releaseInfo { return &releaseInfo{applyHash: applyHash, skipApply: skipApply} } + +var ( + GetHealthMonitoringName = getHealthMonitoringName + GetMonitoringResourceSelectors = getMonitoringResourceSelectors + BuildMonitoringHealthCheck = buildMonitoringHealthCheck + BuildMonitoringClusterHealthCheck = buildMonitoringClusterHealthCheck + QuoteLuaString = quoteLuaString + ReconcileHealthMonitoring = (*ClusterSummaryReconciler).reconcileHealthMonitoring + RemoveHealthMonitoring = (*ClusterSummaryReconciler).removeHealthMonitoring + DefaultHealthCheckLivenessName = defaultHealthCheckLivenessName + HealthMonitoringFlappingEvaluation = healthMonitoringFlappingEvaluations +) + +// NewHelmReleases returns helm releases for tests, from "/" strings +func NewHelmReleases(namespaceAndNames ...string) []helmRelease { + releases := make([]helmRelease, 0, len(namespaceAndNames)) + for _, namespaceAndName := range namespaceAndNames { + parts := strings.SplitN(namespaceAndName, "/", 2) + releases = append(releases, helmRelease{namespace: parts[0], name: parts[1]}) + } + return releases +} + +// RenderHealthMonitoringLua renders the health monitoring script for tests +func RenderHealthMonitoringLua(ownerKind, ownerName string, releases []helmRelease) string { + return renderHealthMonitoringLua(&healthMonitoringLuaConfig{ + ownerKind: ownerKind, ownerName: ownerName, helmReleases: releases, + }) +} diff --git a/controllers/health_monitoring.go b/controllers/health_monitoring.go new file mode 100644 index 00000000..75a9fec8 --- /dev/null +++ b/controllers/health_monitoring.go @@ -0,0 +1,534 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "crypto/sha256" + _ "embed" // used to embed the Lua library + "encoding/hex" + "fmt" + "reflect" + "sort" + "strings" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/deployer" + "github.com/projectsveltos/libsveltos/lib/eventreasons" + logs "github.com/projectsveltos/libsveltos/lib/logsettings" +) + +//+kubebuilder:rbac:groups=lib.projectsveltos.io,resources=healthchecks,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=lib.projectsveltos.io,resources=clusterhealthchecks,verbs=get;list;watch;create;update;patch;delete + +const ( + healthMonitoringNamePrefix = "hm-" + + // healthChecksIgnoredReason is the reason of the Warning event emitted when a Profile lists + // HealthChecks, which only a ClusterProfile can do + healthChecksIgnoredReason = "HealthChecksIgnored" + + // defaultHealthCheckLivenessName is the name of the liveness check referencing the + // HealthCheck generated by addon-controller + defaultHealthCheckLivenessName = "deployed-resources" + + // healthMonitoringFlappingEvaluations is the number of consecutive evaluations a + // resource must be found not Healthy before it is reported + healthMonitoringFlappingEvaluations = 3 + + statefulSetKind = "StatefulSet" + daemonSetKind = "DaemonSet" + jobKind = "Job" + persistentVolumeClaimKind = "PersistentVolumeClaim" + + batchGroupName = "batch" + coreGroupName = "" + + // monitoredAPIVersion is the version all the monitored kinds are served from + // (apps/v1, batch/v1 and v1) + monitoredAPIVersion = "v1" +) + +var ( + // healthMonitoringLua is the Lua code evaluating the health of the resources. The + // Lua "config" table is prepended to it, see healthMonitoringLuaConfig. + // + //go:embed healthmonitoring/health.lua + healthMonitoringLua string + + // monitoredKinds are the kinds the default health checks cover + monitoredKinds = []monitoredKind{ + {group: appsGroupName, kind: deploymentKind}, + {group: appsGroupName, kind: statefulSetKind}, + {group: appsGroupName, kind: daemonSetKind}, + {group: batchGroupName, kind: jobKind}, + {group: coreGroupName, kind: persistentVolumeClaimKind}, + } +) + +// monitoredKind identifies a kind (version is always monitoredAPIVersion) covered by the default health checks +type monitoredKind struct { + group string + kind string +} + +// helmRelease identifies a Helm release deployed by a ClusterSummary +type helmRelease struct { + namespace string + name string +} + +// healthMonitoringLuaConfig contains what the Lua code needs to select, among all the +// resources found in the cluster, the ones deployed by a given profile. +type healthMonitoringLuaConfig struct { + ownerKind string + ownerName string + helmReleases []helmRelease +} + +// isHealthMonitoringEnabled returns true if health monitoring has been requested +func isHealthMonitoringEnabled(clusterSummary *configv1beta1.ClusterSummary) bool { + hm := clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring + return hm != nil && hm.Enabled +} + +// useDefaultHealthChecks returns true if the health of the resources deployed by the +// profile must be monitored. Default is true. +func useDefaultHealthChecks(hm *configv1beta1.HealthMonitoring) bool { + return hm.DefaultChecks == nil || *hm.DefaultChecks +} + +// getHealthMonitoringName returns the name of the HealthCheck and of the ClusterHealthCheck +// created for a ClusterSummary. Both are cluster scoped: the name is derived from the +// ClusterSummary UID so that it is unique, short and stable. +func getHealthMonitoringName(clusterSummary *configv1beta1.ClusterSummary) string { + sum := sha256.Sum256([]byte(clusterSummary.UID)) + return healthMonitoringNamePrefix + hex.EncodeToString(sum[:])[:12] +} + +func getHealthMonitoringLabels(clusterSummary *configv1beta1.ClusterSummary) map[string]string { + return map[string]string{ + configv1beta1.HealthMonitoringClusterSummaryUIDLabel: string(clusterSummary.UID), + } +} + +func getHealthMonitoringAnnotations(clusterSummary *configv1beta1.ClusterSummary) map[string]string { + return map[string]string{ + configv1beta1.HealthMonitoringClusterSummaryNamespaceAnnotation: clusterSummary.Namespace, + configv1beta1.HealthMonitoringClusterSummaryNameAnnotation: clusterSummary.Name, + } +} + +// getClusterSummaryClusterRef returns a reference to the cluster a ClusterSummary targets +func getClusterSummaryClusterRef(clusterSummary *configv1beta1.ClusterSummary) corev1.ObjectReference { + clusterRef := corev1.ObjectReference{ + Namespace: clusterSummary.Spec.ClusterNamespace, + Name: clusterSummary.Spec.ClusterName, + } + if clusterSummary.Spec.ClusterType == libsveltosv1beta1.ClusterTypeSveltos { + clusterRef.Kind = libsveltosv1beta1.SveltosClusterKind + clusterRef.APIVersion = libsveltosv1beta1.GroupVersion.String() + } else { + clusterRef.Kind = clusterKind + clusterRef.APIVersion = clusterv1.GroupVersion.String() + } + return clusterRef +} + +// areAllFeaturesProvisioned returns true if the ClusterSummary has features and all of them are provisioned +func areAllFeaturesProvisioned(clusterSummary *configv1beta1.ClusterSummary) bool { + if len(clusterSummary.Status.FeatureSummaries) == 0 { + return false + } + for i := range clusterSummary.Status.FeatureSummaries { + if clusterSummary.Status.FeatureSummaries[i].Status != libsveltosv1beta1.FeatureStatusProvisioned { + return false + } + } + return true +} + +// healthMonitoringInputsChanged returns true if what health monitoring depends on changed between +// two versions of a ClusterSummary: all features becoming (or ceasing to be) provisioned, and the +// Helm releases managed by the ClusterSummary. +func healthMonitoringInputsChanged(oldClusterSummary, newClusterSummary *configv1beta1.ClusterSummary) bool { + if areAllFeaturesProvisioned(oldClusterSummary) != areAllFeaturesProvisioned(newClusterSummary) { + return true + } + + return !reflect.DeepEqual(getManagedHelmReleases(oldClusterSummary), getManagedHelmReleases(newClusterSummary)) +} + +// getManagedHelmReleases returns the Helm releases currently managed by the ClusterSummary, sorted +func getManagedHelmReleases(clusterSummary *configv1beta1.ClusterSummary) []helmRelease { + releases := make([]helmRelease, 0, len(clusterSummary.Status.HelmReleaseSummaries)) + for i := range clusterSummary.Status.HelmReleaseSummaries { + summary := &clusterSummary.Status.HelmReleaseSummaries[i] + if summary.Status != configv1beta1.HelmChartStatusManaging { + continue + } + releases = append(releases, helmRelease{namespace: summary.ReleaseNamespace, name: summary.ReleaseName}) + } + + sort.Slice(releases, func(i, j int) bool { + if releases[i].namespace != releases[j].namespace { + return releases[i].namespace < releases[j].namespace + } + return releases[i].name < releases[j].name + }) + + return releases +} + +// deploysRawResources returns true if the profile deploys anything but Helm charts. +// Resources deployed by Sveltos (everything but Helm charts) carry the ReasonLabel label. +func deploysRawResources(clusterSummary *configv1beta1.ClusterSummary) bool { + spec := &clusterSummary.Spec.ClusterProfileSpec + return len(spec.PolicyRefs) > 0 || len(spec.KustomizationRefs) > 0 +} + +// getMonitoringResourceSelectors returns the selectors for the resources to evaluate: +// - for the resources deployed by Sveltos, every object with the label added by Sveltos; +// - for each Helm release namespace, every object in that namespace (the Lua code +// keeps only the ones belonging to a release of the profile). +func getMonitoringResourceSelectors(deployRaw bool, releases []helmRelease) []libsveltosv1beta1.ResourceSelector { + selectors := make([]libsveltosv1beta1.ResourceSelector, 0) + + if deployRaw { + for _, mk := range monitoredKinds { + selectors = append(selectors, libsveltosv1beta1.ResourceSelector{ + Group: mk.group, Version: monitoredAPIVersion, Kind: mk.kind, + LabelFilters: []libsveltosv1beta1.LabelFilter{ + {Key: deployer.ReasonLabel, Operation: libsveltosv1beta1.OperationHas}, + }, + }) + } + } + + namespaces := make(map[string]bool) + for i := range releases { + namespaces[releases[i].namespace] = true + } + sortedNamespaces := make([]string, 0, len(namespaces)) + for ns := range namespaces { + sortedNamespaces = append(sortedNamespaces, ns) + } + sort.Strings(sortedNamespaces) + + for _, ns := range sortedNamespaces { + for _, mk := range monitoredKinds { + selectors = append(selectors, libsveltosv1beta1.ResourceSelector{ + Group: mk.group, Version: monitoredAPIVersion, Kind: mk.kind, Namespace: ns, + }) + } + } + + return selectors +} + +// quoteLuaString returns s as a Lua string literal (Lua 5.1 syntax: no \x or \u escapes) +func quoteLuaString(s string) string { + var b strings.Builder + b.WriteByte('"') + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c == '\\': + b.WriteString(`\\`) + case c == '"': + b.WriteString(`\"`) + case c == '\n': + b.WriteString(`\n`) + case c == '\r': + b.WriteString(`\r`) + case c < ' ' || c == 0x7f: + fmt.Fprintf(&b, `\%03d`, c) + default: + b.WriteByte(c) + } + } + b.WriteByte('"') + return b.String() +} + +// renderHealthMonitoringLua returns the script evaluating the health of the resources deployed +// by a profile: the Lua "config" table followed by the embedded Lua library. +func renderHealthMonitoringLua(config *healthMonitoringLuaConfig) string { + var b strings.Builder + b.WriteString("local config = {\n") + fmt.Fprintf(&b, " ownerKind = %s,\n", quoteLuaString(config.ownerKind)) + fmt.Fprintf(&b, " ownerName = %s,\n", quoteLuaString(config.ownerName)) + b.WriteString(" helmReleases = {\n") + for _, release := range config.helmReleases { + fmt.Fprintf(&b, " [%s] = true,\n", quoteLuaString(release.namespace+"/"+release.name)) + } + b.WriteString(" },\n}\n\n") + b.WriteString(healthMonitoringLua) + return b.String() +} + +// buildMonitoringHealthCheck returns the HealthCheck evaluating the health of the resources deployed +// by the profile owning the ClusterSummary. Returns nil if there is nothing to monitor. +func buildMonitoringHealthCheck(clusterSummary *configv1beta1.ClusterSummary, profileKind, profileName string, +) *libsveltosv1beta1.HealthCheck { + + releases := getManagedHelmReleases(clusterSummary) + selectors := getMonitoringResourceSelectors(deploysRawResources(clusterSummary), releases) + if len(selectors) == 0 { + return nil + } + + script := renderHealthMonitoringLua(&healthMonitoringLuaConfig{ + ownerKind: profileKind, ownerName: profileName, helmReleases: releases, + }) + + return &libsveltosv1beta1.HealthCheck{ + ObjectMeta: metav1.ObjectMeta{ + Name: getHealthMonitoringName(clusterSummary), + Labels: getHealthMonitoringLabels(clusterSummary), + Annotations: getHealthMonitoringAnnotations(clusterSummary), + }, + Spec: libsveltosv1beta1.HealthCheckSpec{ + ResourceSelectors: selectors, + EvaluateHealth: script, + EvaluateIndependently: true, + Flapping: &libsveltosv1beta1.FlappingPolicy{ + ConsecutiveEvaluations: healthMonitoringFlappingEvaluations, + }, + }, + } +} + +func getHealthCheckLiveness(livenessName, healthCheckName string) libsveltosv1beta1.LivenessCheck { + return libsveltosv1beta1.LivenessCheck{ + Name: livenessName, + Type: libsveltosv1beta1.LivenessTypeHealthCheck, + LivenessSourceRef: &corev1.ObjectReference{ + APIVersion: libsveltosv1beta1.GroupVersion.String(), + Kind: libsveltosv1beta1.HealthCheckKind, + Name: healthCheckName, + }, + } +} + +// buildMonitoringClusterHealthCheck returns the ClusterHealthCheck targeting the cluster of the +// ClusterSummary. Its liveness checks are the generated HealthCheck (when generatedHealthCheck is not +// empty) and the HealthChecks listed by the user. +func buildMonitoringClusterHealthCheck(clusterSummary *configv1beta1.ClusterSummary, + generatedHealthCheck string, userHealthChecks []string) *libsveltosv1beta1.ClusterHealthCheck { + + livenessChecks := make([]libsveltosv1beta1.LivenessCheck, 0, len(userHealthChecks)+1) + if generatedHealthCheck != "" { + livenessChecks = append(livenessChecks, + getHealthCheckLiveness(defaultHealthCheckLivenessName, generatedHealthCheck)) + } + + sortedUserHealthChecks := append([]string{}, userHealthChecks...) + sort.Strings(sortedUserHealthChecks) + for _, name := range sortedUserHealthChecks { + livenessChecks = append(livenessChecks, getHealthCheckLiveness(name, name)) + } + + return &libsveltosv1beta1.ClusterHealthCheck{ + ObjectMeta: metav1.ObjectMeta{ + Name: getHealthMonitoringName(clusterSummary), + Labels: getHealthMonitoringLabels(clusterSummary), + Annotations: getHealthMonitoringAnnotations(clusterSummary), + }, + Spec: libsveltosv1beta1.ClusterHealthCheckSpec{ + ClusterRefs: []corev1.ObjectReference{getClusterSummaryClusterRef(clusterSummary)}, + LivenessChecks: livenessChecks, + // Results are consumed from the HealthCheckReports. No notification is requested. + Notifications: []libsveltosv1beta1.Notification{}, + }, + } +} + +// reconcileHealthMonitoring makes sure the HealthCheck and ClusterHealthCheck needed to monitor the +// ClusterSummary exist (or do not exist when monitoring is not enabled). +// Instances are created only once all features are provisioned, so a deployment still +// in progress is not reported as unhealthy. Once created they are kept, so that the health of an +// update is reported. +func (r *ClusterSummaryReconciler) reconcileHealthMonitoring(ctx context.Context, + clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) error { + + if !clusterSummary.DeletionTimestamp.IsZero() { + return nil + } + + // A DryRun ClusterSummary does not deploy anything: there is nothing to monitor + if clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1beta1.SyncModeDryRun { + return nil + } + + if !isHealthMonitoringEnabled(clusterSummary) { + return r.removeHealthMonitoring(ctx, clusterSummary, logger) + } + + if !areAllFeaturesProvisioned(clusterSummary) { + logger.V(logs.LogDebug).Info("not all features are provisioned yet. Health monitoring is deferred.") + return nil + } + + profileRef, err := configv1beta1.GetProfileRef(clusterSummary) + if err != nil { + return err + } + + hm := clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring + healthChecks := r.getAllowedHealthChecks(clusterSummary, profileRef.Kind, hm) + + generatedHealthCheck := "" + if useDefaultHealthChecks(hm) { + healthCheck := buildMonitoringHealthCheck(clusterSummary, profileRef.Kind, profileRef.Name) + if healthCheck != nil { + if err := createOrUpdateMonitoringHealthCheck(ctx, r.Client, healthCheck); err != nil { + return err + } + generatedHealthCheck = healthCheck.Name + } + } + + if generatedHealthCheck == "" && len(healthChecks) == 0 { + // Nothing to monitor (yet): the profile does not deploy any of the monitored kinds + return r.removeHealthMonitoring(ctx, clusterSummary, logger) + } + + if generatedHealthCheck == "" { + // The generated HealthCheck is not needed anymore + if err := deleteMonitoringHealthCheck(ctx, r.Client, clusterSummary); err != nil { + return err + } + } + + clusterHealthCheck := buildMonitoringClusterHealthCheck(clusterSummary, generatedHealthCheck, healthChecks) + return createOrUpdateMonitoringClusterHealthCheck(ctx, r.Client, clusterHealthCheck) +} + +// getAllowedHealthChecks returns the HealthChecks listed in healthMonitoring that can be monitored. +// Only a ClusterProfile can list HealthChecks: a HealthCheck is cluster scoped and referenced by name, +// and the report it produces is shared by all the profiles referencing it on a cluster, so a (namespaced) +// Profile could otherwise learn about resources outside its namespace. For a Profile the list is +// ignored, and a Warning event says so. +func (r *ClusterSummaryReconciler) getAllowedHealthChecks(clusterSummary *configv1beta1.ClusterSummary, + profileKind string, hm *configv1beta1.HealthMonitoring) []string { + + if profileKind != configv1beta1.ProfileKind || len(hm.HealthChecks) == 0 { + return hm.HealthChecks + } + + recordEventf(r.eventRecorder, clusterSummary, nil, corev1.EventTypeWarning, healthChecksIgnoredReason, + eventreasons.ActionReconciling, + "healthMonitoring.healthChecks is ignored: only a ClusterProfile can reference HealthChecks, "+ + "a Profile gets the default health checks only") + + return nil +} + +// removeHealthMonitoring removes the HealthCheck and ClusterHealthCheck created for the ClusterSummary +func (r *ClusterSummaryReconciler) removeHealthMonitoring(ctx context.Context, + clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) error { + + logger.V(logs.LogDebug).Info("removing health monitoring resources, if any") + + // The ClusterHealthCheck goes first: removing it makes healthcheck-manager remove the + // HealthCheck from the managed cluster. + if err := deleteIfOwned(ctx, r.Client, &libsveltosv1beta1.ClusterHealthCheck{}, clusterSummary); err != nil { + return err + } + + return deleteMonitoringHealthCheck(ctx, r.Client, clusterSummary) +} + +func deleteMonitoringHealthCheck(ctx context.Context, c client.Client, + clusterSummary *configv1beta1.ClusterSummary) error { + + return deleteIfOwned(ctx, c, &libsveltosv1beta1.HealthCheck{}, clusterSummary) +} + +// deleteIfOwned deletes the instance created for the ClusterSummary, if any. It does nothing +// if the instance does not exist, or if the CRD is not installed. +func deleteIfOwned(ctx context.Context, c client.Client, obj client.Object, + clusterSummary *configv1beta1.ClusterSummary) error { + + obj.SetName(getHealthMonitoringName(clusterSummary)) + err := c.Get(ctx, client.ObjectKeyFromObject(obj), obj) + if err != nil { + if apierrors.IsNotFound(err) || meta.IsNoMatchError(err) { + return nil + } + return err + } + + // Never delete an instance this ClusterSummary did not create + if obj.GetLabels()[configv1beta1.HealthMonitoringClusterSummaryUIDLabel] != string(clusterSummary.UID) { + return nil + } + + err = c.Delete(ctx, obj) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +func createOrUpdateMonitoringHealthCheck(ctx context.Context, c client.Client, + desired *libsveltosv1beta1.HealthCheck) error { + + healthCheck := &libsveltosv1beta1.HealthCheck{ObjectMeta: metav1.ObjectMeta{Name: desired.Name}} + _, err := controllerutil.CreateOrUpdate(ctx, c, healthCheck, func() error { + healthCheck.Labels = mergeStringMaps(healthCheck.Labels, desired.Labels) + healthCheck.Annotations = mergeStringMaps(healthCheck.Annotations, desired.Annotations) + healthCheck.Spec = desired.Spec + return nil + }) + return err +} + +func createOrUpdateMonitoringClusterHealthCheck(ctx context.Context, c client.Client, + desired *libsveltosv1beta1.ClusterHealthCheck) error { + + clusterHealthCheck := &libsveltosv1beta1.ClusterHealthCheck{ObjectMeta: metav1.ObjectMeta{Name: desired.Name}} + _, err := controllerutil.CreateOrUpdate(ctx, c, clusterHealthCheck, func() error { + clusterHealthCheck.Labels = mergeStringMaps(clusterHealthCheck.Labels, desired.Labels) + clusterHealthCheck.Annotations = mergeStringMaps(clusterHealthCheck.Annotations, desired.Annotations) + clusterHealthCheck.Spec = desired.Spec + return nil + }) + return err +} + +// mergeStringMaps returns current with all the entries of desired set +func mergeStringMaps(current, desired map[string]string) map[string]string { + if current == nil { + current = make(map[string]string, len(desired)) + } + for k, v := range desired { + current[k] = v + } + return current +} diff --git a/controllers/health_monitoring_test.go b/controllers/health_monitoring_test.go new file mode 100644 index 00000000..168357f9 --- /dev/null +++ b/controllers/health_monitoring_test.go @@ -0,0 +1,1147 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers_test + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "time" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + lua "github.com/yuin/gopher-lua" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + "k8s.io/klog/v2/textlogger" + "k8s.io/utils/ptr" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/event" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + "github.com/projectsveltos/addon-controller/controllers" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/deployer" + fakedeployer "github.com/projectsveltos/libsveltos/lib/deployer/fake" + sveltoslua "github.com/projectsveltos/libsveltos/lib/lua" +) + +const ( + hmOwnerKind = "ClusterProfile" + hmOwnerName = "platform-stack" + + hmNamespace = "web-namespace" + hmUserHealthCheck = "my-cert-expiry-check" + + hmHelmReleaseNameAnnotation = "meta.helm.sh/release-name" + hmHelmReleaseNamespaceAnnotation = "meta.helm.sh/release-namespace" + + hmStatusProgressing = "Progressing" + + hmCertManagerRelease = "cert-manager/cert-manager" + hmGrafanaRelease = "monitoring/grafana" +) + +// healthResult is what the health monitoring Lua code reports for one resource +type healthResult struct { + name string + status string + message string +} + +// luaHealthOutput is the structure returned by the evaluate Lua function +type luaHealthOutput struct { + Resources []struct { + Resource struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + } `json:"resource"` + Status string `json:"status"` + Message string `json:"message"` + } `json:"resources"` +} + +// kindOf returns the Kubernetes kind of a typed object, e.g. Deployment for *appsv1.Deployment +func kindOf(obj interface{}) string { + return reflect.TypeOf(obj).Elem().Name() +} + +// toLuaResource converts a typed object to the table the HealthCheck Lua code receives. Resources reach +// the Lua code decoded from JSON, hence the round trip. +func toLuaResource(obj client.Object) lua.LValue { + gvk, err := apiutil.GVKForObject(obj, scheme) + Expect(err).To(BeNil()) + obj.GetObjectKind().SetGroupVersionKind(gvk) + + jsonResource, err := json.Marshal(obj) + Expect(err).To(BeNil()) + decoded := map[string]interface{}{} + Expect(json.Unmarshal(jsonResource, &decoded)).To(Succeed()) + + return sveltoslua.MapToTable(decoded) +} + +// evaluateHealthLua runs the health monitoring script against resources +func evaluateHealthLua(script string, resources ...client.Object) []healthResult { + l := lua.NewState() + defer l.Close() + + Expect(l.DoString(script)).To(Succeed()) + + resourcesTable := l.NewTable() + for i := range resources { + resourcesTable.Append(toLuaResource(resources[i])) + } + l.SetGlobal("resources", resourcesTable) + + Expect(l.CallByParam(lua.P{Fn: l.GetGlobal("evaluate"), NRet: 1, Protect: true})).To(Succeed()) + + tbl, ok := l.Get(-1).(*lua.LTable) + Expect(ok).To(BeTrue()) + + jsonOutput, err := json.Marshal(sveltoslua.ToGoValue(tbl)) + Expect(err).To(BeNil()) + var output luaHealthOutput + Expect(json.Unmarshal(jsonOutput, &output)).To(Succeed()) + + var results []healthResult + for i := range output.Resources { + results = append(results, healthResult{ + name: output.Resources[i].Resource.Metadata.Name, + status: output.Resources[i].Status, + message: output.Resources[i].Message, + }) + } + return results +} + +func ownerAnnotations() map[string]string { + return map[string]string{ + deployer.OwnerKind: hmOwnerKind, + deployer.OwnerName: hmOwnerName, + } +} + +func helmAnnotations(namespace, release string) map[string]string { + return map[string]string{ + hmHelmReleaseNameAnnotation: release, + hmHelmReleaseNamespaceAnnotation: namespace, + } +} + +func hmMetadata(name, uid string, annotations map[string]string, generation int64) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Name: name, Namespace: hmNamespace, UID: types.UID(uid), Generation: generation, Annotations: annotations, + } +} + +func hmDeployment(name, uid string, annotations map[string]string, generation int64, + spec *appsv1.DeploymentSpec, status *appsv1.DeploymentStatus) *appsv1.Deployment { + + return &appsv1.Deployment{ObjectMeta: hmMetadata(name, uid, annotations, generation), Spec: *spec, Status: *status} +} + +func hmStatefulSet(name, uid string, annotations map[string]string, generation int64, + spec *appsv1.StatefulSetSpec, status *appsv1.StatefulSetStatus) *appsv1.StatefulSet { + + return &appsv1.StatefulSet{ObjectMeta: hmMetadata(name, uid, annotations, generation), Spec: *spec, Status: *status} +} + +func hmDaemonSet(name, uid string, annotations map[string]string, + spec *appsv1.DaemonSetSpec, status *appsv1.DaemonSetStatus) *appsv1.DaemonSet { + + return &appsv1.DaemonSet{ObjectMeta: hmMetadata(name, uid, annotations, 1), Spec: *spec, Status: *status} +} + +func hmJob(name, uid string, conditions ...batchv1.JobCondition) *batchv1.Job { + return &batchv1.Job{ + ObjectMeta: hmMetadata(name, uid, ownerAnnotations(), 1), + Status: batchv1.JobStatus{Conditions: conditions}, + } +} + +func jobCondition(conditionType batchv1.JobConditionType, reason string) batchv1.JobCondition { + return batchv1.JobCondition{Type: conditionType, Status: corev1.ConditionTrue, Reason: reason} +} + +func rollingUpdateStrategy(partition int32) appsv1.StatefulSetUpdateStrategy { + return appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{Partition: ptr.To(partition)}, + } +} + +var _ = Describe("Health monitoring: Lua health rules", func() { + var script string + + BeforeEach(func() { + script = controllers.RenderHealthMonitoringLua(hmOwnerKind, hmOwnerName, + controllers.NewHelmReleases(hmCertManagerRelease, hmGrafanaRelease)) + }) + + It("reports nothing for a Deployment that completed its rollout", func() { + deployment := hmDeployment("web", "uid-1", ownerAnnotations(), 2, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)}, + &appsv1.DeploymentStatus{ObservedGeneration: 2, Replicas: 3, UpdatedReplicas: 3, AvailableReplicas: 3}) + + Expect(evaluateHealthLua(script, deployment)).To(BeEmpty()) + }) + + It("reports a Deployment in the middle of a rollout as Progressing", func() { + deployment := hmDeployment("web", "uid-1", ownerAnnotations(), 2, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)}, + &appsv1.DeploymentStatus{ObservedGeneration: 2, Replicas: 4, UpdatedReplicas: 2, AvailableReplicas: 3}) + + results := evaluateHealthLua(script, deployment) + Expect(results).To(HaveLen(1)) + Expect(results[0].status).To(Equal("Progressing")) + Expect(results[0].message).To(ContainSubstring("2 out of 3 new replicas have been updated")) + }) + + It("reports a Deployment whose new generation was not observed yet as Progressing", func() { + deployment := hmDeployment("web", "uid-1", ownerAnnotations(), 3, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)}, + &appsv1.DeploymentStatus{ObservedGeneration: 2, Replicas: 3, UpdatedReplicas: 3, AvailableReplicas: 3}) + + results := evaluateHealthLua(script, deployment) + Expect(results).To(HaveLen(1)) + Expect(results[0].status).To(Equal("Progressing")) + Expect(results[0].message).To(ContainSubstring("observed deployment generation less than desired generation")) + }) + + It("reports a Deployment with updated replicas not available as Progressing", func() { + deployment := hmDeployment("web", "uid-1", ownerAnnotations(), 2, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)}, + &appsv1.DeploymentStatus{ObservedGeneration: 2, Replicas: 3, UpdatedReplicas: 3, AvailableReplicas: 1}) + + results := evaluateHealthLua(script, deployment) + Expect(results).To(HaveLen(1)) + Expect(results[0].status).To(Equal("Progressing")) + Expect(results[0].message).To(ContainSubstring("1 of 3 updated replicas are available")) + }) + + It("reports a Deployment that exceeded its progress deadline as Degraded", func() { + deployment := hmDeployment("web", "uid-1", ownerAnnotations(), 2, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)}, + &appsv1.DeploymentStatus{ + ObservedGeneration: 2, Replicas: 3, UpdatedReplicas: 3, + Conditions: []appsv1.DeploymentCondition{ + { + Type: appsv1.DeploymentProgressing, Status: corev1.ConditionFalse, + Reason: "ProgressDeadlineExceeded", + }, + }, + }) + + results := evaluateHealthLua(script, deployment) + Expect(results).To(HaveLen(1)) + Expect(results[0].status).To(Equal("Degraded")) + }) + + It("reports a paused Deployment as Suspended", func() { + deployment := hmDeployment("web", "uid-1", ownerAnnotations(), 2, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3), Paused: true}, + &appsv1.DeploymentStatus{ObservedGeneration: 2}) + + results := evaluateHealthLua(script, deployment) + Expect(results).To(HaveLen(1)) + Expect(results[0].status).To(Equal("Suspended")) + }) + + It("ignores resources not deployed by the profile", func() { + broken := &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)} + notObserved := &appsv1.DeploymentStatus{ObservedGeneration: 1} + + // Deployed by another profile + otherProfile := ownerAnnotations() + otherProfile[deployer.OwnerName] = "another-profile" + // Same name, deployed by a Profile instead of a ClusterProfile + otherKind := ownerAnnotations() + otherKind[deployer.OwnerKind] = "Profile" + + results := evaluateHealthLua(script, + hmDeployment("no-annotation", "uid-1", nil, 2, broken, notObserved), + hmDeployment("other-profile", "uid-2", otherProfile, 2, broken, notObserved), + hmDeployment("other-kind", "uid-3", otherKind, 2, broken, notObserved), + // Helm release not deployed by this profile + hmDeployment("other-release", "uid-4", helmAnnotations(testReleaseNameKyverno, testReleaseNameKyverno), 2, + broken, notObserved), + // Same release name in a different namespace + hmDeployment("other-namespace", "uid-5", helmAnnotations(hmNamespace, "cert-manager"), 2, broken, notObserved), + ) + + Expect(results).To(BeEmpty()) + }) + + It("evaluates resources belonging to a Helm release of the profile", func() { + broken := &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)} + notObserved := &appsv1.DeploymentStatus{ObservedGeneration: 1} + + results := evaluateHealthLua(script, + hmDeployment("cert-manager", "uid-1", helmAnnotations("cert-manager", "cert-manager"), 2, broken, notObserved), + hmDeployment("grafana", "uid-2", helmAnnotations("monitoring", "grafana"), 2, broken, notObserved), + ) + + Expect(results).To(HaveLen(2)) + Expect(results[0].name).To(Equal("cert-manager")) + Expect(results[1].name).To(Equal("grafana")) + }) + + It("evaluates once a resource selected by more than one selector", func() { + annotations := ownerAnnotations() + for k, v := range helmAnnotations("cert-manager", "cert-manager") { + annotations[k] = v + } + deployment := hmDeployment("web", "same-uid", annotations, 2, + &appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)}, &appsv1.DeploymentStatus{ObservedGeneration: 1}) + + Expect(evaluateHealthLua(script, deployment, deployment)).To(HaveLen(1)) + }) + + It("ignores kinds it has no rule for", func() { + replicaSet := &appsv1.ReplicaSet{ObjectMeta: hmMetadata("rs", "uid-1", ownerAnnotations(), 1)} + Expect(evaluateHealthLua(script, replicaSet)).To(BeEmpty()) + }) + + It("evaluates StatefulSets", func() { + healthy := hmStatefulSet("healthy", "uid-1", ownerAnnotations(), 1, + &appsv1.StatefulSetSpec{Replicas: ptr.To[int32](3), UpdateStrategy: rollingUpdateStrategy(0)}, + &appsv1.StatefulSetStatus{ObservedGeneration: 1, ReadyReplicas: 3, UpdatedReplicas: 3}) + notReady := hmStatefulSet("not-ready", "uid-2", ownerAnnotations(), 1, + &appsv1.StatefulSetSpec{Replicas: ptr.To[int32](3)}, + &appsv1.StatefulSetStatus{ObservedGeneration: 1, ReadyReplicas: 1}) + notObserved := hmStatefulSet("not-observed", "uid-3", ownerAnnotations(), 2, + &appsv1.StatefulSetSpec{Replicas: ptr.To[int32](3)}, + &appsv1.StatefulSetStatus{ObservedGeneration: 1, ReadyReplicas: 3}) + rolling := hmStatefulSet("rolling", "uid-4", ownerAnnotations(), 1, + &appsv1.StatefulSetSpec{Replicas: ptr.To[int32](3), UpdateStrategy: rollingUpdateStrategy(0)}, + &appsv1.StatefulSetStatus{ObservedGeneration: 1, ReadyReplicas: 3, UpdatedReplicas: 1}) + onDelete := hmStatefulSet("on-delete", "uid-5", ownerAnnotations(), 1, + &appsv1.StatefulSetSpec{ + Replicas: ptr.To[int32](3), + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{Type: appsv1.OnDeleteStatefulSetStrategyType}, + }, + &appsv1.StatefulSetStatus{ObservedGeneration: 1, ReadyReplicas: 3, UpdatedReplicas: 0}) + + results := evaluateHealthLua(script, healthy, notReady, notObserved, rolling, onDelete) + + Expect(results).To(HaveLen(3)) + Expect(results[0].name).To(Equal("not-ready")) + Expect(results[0].message).To(ContainSubstring("Waiting for 2 pods to be ready")) + Expect(results[1].name).To(Equal("not-observed")) + Expect(results[2].name).To(Equal("rolling")) + Expect(results[2].message).To(ContainSubstring("1 out of 3 new pods have been updated")) + for i := range results { + Expect(results[i].status).To(Equal("Progressing")) + } + }) + + It("evaluates DaemonSets", func() { + healthy := hmDaemonSet("healthy", "uid-1", ownerAnnotations(), &appsv1.DaemonSetSpec{}, + &appsv1.DaemonSetStatus{ + ObservedGeneration: 1, DesiredNumberScheduled: 3, UpdatedNumberScheduled: 3, NumberAvailable: 3, + }) + updating := hmDaemonSet("updating", "uid-2", ownerAnnotations(), &appsv1.DaemonSetSpec{}, + &appsv1.DaemonSetStatus{ + ObservedGeneration: 1, DesiredNumberScheduled: 3, UpdatedNumberScheduled: 1, NumberAvailable: 3, + }) + unavailable := hmDaemonSet("unavailable", "uid-3", ownerAnnotations(), &appsv1.DaemonSetSpec{}, + &appsv1.DaemonSetStatus{ + ObservedGeneration: 1, DesiredNumberScheduled: 3, UpdatedNumberScheduled: 3, NumberAvailable: 2, + }) + onDelete := hmDaemonSet("on-delete", "uid-4", ownerAnnotations(), + &appsv1.DaemonSetSpec{UpdateStrategy: appsv1.DaemonSetUpdateStrategy{Type: appsv1.OnDeleteDaemonSetStrategyType}}, + &appsv1.DaemonSetStatus{ObservedGeneration: 1, DesiredNumberScheduled: 3}) + + results := evaluateHealthLua(script, healthy, updating, unavailable, onDelete) + + Expect(results).To(HaveLen(2)) + Expect(results[0].name).To(Equal("updating")) + Expect(results[0].message).To(ContainSubstring("1 out of 3 new pods have been updated")) + Expect(results[1].name).To(Equal("unavailable")) + Expect(results[1].message).To(ContainSubstring("2 of 3 updated pods are available")) + }) + + It("evaluates Jobs", func() { + completed := hmJob("completed", "uid-1", jobCondition(batchv1.JobComplete, "")) + running := hmJob("running", "uid-2") + failed := hmJob("failed", "uid-3", jobCondition(batchv1.JobFailed, "BackoffLimitExceeded")) + suspended := hmJob("suspended", "uid-4", jobCondition(batchv1.JobSuspended, "")) + // A Failed condition wins over a Suspended one + failedAndSuspended := hmJob("failed-and-suspended", "uid-5", + jobCondition(batchv1.JobSuspended, ""), jobCondition(batchv1.JobFailed, "DeadlineExceeded")) + // A condition that is not true does not count + resumed := hmJob("resumed", "uid-6", batchv1.JobCondition{Type: batchv1.JobSuspended, Status: corev1.ConditionFalse}) + + results := evaluateHealthLua(script, completed, running, failed, suspended, failedAndSuspended, resumed) + + Expect(results).To(HaveLen(5)) + Expect(results[0]).To(Equal(healthResult{name: "running", status: hmStatusProgressing, message: "Job has not completed yet"})) + Expect(results[1].name).To(Equal("failed")) + Expect(results[1].status).To(Equal("Degraded")) + Expect(results[1].message).To(ContainSubstring("BackoffLimitExceeded")) + Expect(results[2]).To(Equal(healthResult{name: "suspended", status: "Suspended", message: "Job is suspended"})) + Expect(results[3].name).To(Equal("failed-and-suspended")) + Expect(results[3].status).To(Equal("Degraded")) + Expect(results[4].name).To(Equal("resumed")) + Expect(results[4].status).To(Equal("Progressing")) + }) + + It("evaluates PersistentVolumeClaims", func() { + pvc := func(name, uid string, phase corev1.PersistentVolumeClaimPhase) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: hmMetadata(name, uid, ownerAnnotations(), 1), + Status: corev1.PersistentVolumeClaimStatus{Phase: phase}, + } + } + + results := evaluateHealthLua(script, + pvc("bound", "uid-1", corev1.ClaimBound), + pvc("pending", "uid-2", corev1.ClaimPending), + pvc("lost", "uid-3", corev1.ClaimLost), + pvc("no-phase", "uid-4", ""), + ) + + Expect(results).To(HaveLen(3)) + Expect(results[0]).To(Equal(healthResult{name: "pending", status: hmStatusProgressing, message: "PersistentVolumeClaim is not bound yet"})) + Expect(results[1].name).To(Equal("lost")) + Expect(results[1].status).To(Equal("Degraded")) + Expect(results[2].name).To(Equal("no-phase")) + Expect(results[2].status).To(Equal("Progressing")) + }) + + It("quotes values placed in the Lua configuration", func() { + Expect(controllers.QuoteLuaString(`plain`)).To(Equal(`"plain"`)) + Expect(controllers.QuoteLuaString(`a"b\c`)).To(Equal(`"a\"b\\c"`)) + Expect(controllers.QuoteLuaString("a\nb\rc")).To(Equal(`"a\nb\rc"`)) + Expect(controllers.QuoteLuaString("a\x01b")).To(Equal(`"a\001b"`)) + + // A malicious value stays a string and cannot break out of the configuration + malicious := `"}} os.exit(1) --` + script := controllers.RenderHealthMonitoringLua(malicious, malicious, + controllers.NewHelmReleases("ns/"+malicious)) + Expect(evaluateHealthLua(script)).To(BeEmpty()) + }) +}) + +func newHealthMonitoringClusterSummary(clusterType libsveltosv1beta1.ClusterType) *configv1beta1.ClusterSummary { + return &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: randomString(), + Name: randomString(), + UID: types.UID(randomString()), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: configv1beta1.GroupVersion.String(), Kind: configv1beta1.ClusterProfileKind, + Name: hmOwnerName, UID: types.UID(randomString()), + }, + }, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: randomString(), + ClusterName: randomString(), + ClusterType: clusterType, + }, + } +} + +func provisionAllFeatures(clusterSummary *configv1beta1.ClusterSummary) { + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + {FeatureID: libsveltosv1beta1.FeatureResources, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + } +} + +func addHelmReleases(clusterSummary *configv1beta1.ClusterSummary, namespaceAndNames ...string) { + for _, namespaceAndName := range namespaceAndNames { + parts := strings.SplitN(namespaceAndName, "/", 2) + clusterSummary.Status.HelmReleaseSummaries = append(clusterSummary.Status.HelmReleaseSummaries, + configv1beta1.HelmChartSummary{ + ReleaseNamespace: parts[0], ReleaseName: parts[1], Status: configv1beta1.HelmChartStatusManaging, + }) + } +} + +var _ = Describe("Health monitoring: resources generated for a ClusterSummary", func() { + It("selects the resources deployed by Sveltos by label and the Helm ones by namespace", func() { + selectors := controllers.GetMonitoringResourceSelectors(true, + controllers.NewHelmReleases("monitoring/grafana", "cert-manager/cert-manager", "monitoring/loki")) + + // 5 kinds with the label, then 5 kinds in each of the two (sorted) namespaces + type kindGroup struct{ kind, group string } + kinds := []kindGroup{ + {kindOf(&appsv1.Deployment{}), appsv1.GroupName}, + {kindOf(&appsv1.StatefulSet{}), appsv1.GroupName}, + {kindOf(&appsv1.DaemonSet{}), appsv1.GroupName}, + {kindOf(&batchv1.Job{}), batchv1.GroupName}, + {kindOf(&corev1.PersistentVolumeClaim{}), corev1.GroupName}, + } + Expect(selectors).To(HaveLen(len(kinds) * 3)) + + for i, kg := range kinds { + raw := selectors[i] + Expect(raw.Kind).To(Equal(kg.kind)) + Expect(raw.Group).To(Equal(kg.group)) + Expect(raw.Namespace).To(BeEmpty()) + Expect(raw.LabelFilters).To(Equal([]libsveltosv1beta1.LabelFilter{ + {Key: "projectsveltos.io/reason", Operation: libsveltosv1beta1.OperationHas}, + })) + + for j, namespace := range []string{"cert-manager", "monitoring"} { + helm := selectors[len(kinds)*(j+1)+i] + Expect(helm.Kind).To(Equal(kg.kind)) + Expect(helm.Group).To(Equal(kg.group)) + Expect(helm.Namespace).To(Equal(namespace)) + Expect(helm.LabelFilters).To(BeEmpty()) + } + } + for i := range selectors { + Expect(selectors[i].Version).To(Equal(appsv1.SchemeGroupVersion.Version)) + } + + Expect(controllers.GetMonitoringResourceSelectors(false, nil)).To(BeEmpty()) + }) + + It("builds a HealthCheck for the resources and Helm releases of the ClusterSummary", func() { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + clusterSummary.Spec.ClusterProfileSpec.PolicyRefs = []configv1beta1.PolicyRef{ + {Kind: "ConfigMap", Namespace: hmNamespace, Name: "policies"}, + } + addHelmReleases(clusterSummary, hmCertManagerRelease) + // A release managed by another ClusterSummary must not be monitored + clusterSummary.Status.HelmReleaseSummaries = append(clusterSummary.Status.HelmReleaseSummaries, + configv1beta1.HelmChartSummary{ + ReleaseNamespace: testReleaseNameKyverno, ReleaseName: testReleaseNameKyverno, Status: configv1beta1.HelmChartStatusConflict, + }) + + healthCheck := controllers.BuildMonitoringHealthCheck(clusterSummary, hmOwnerKind, hmOwnerName) + Expect(healthCheck).ToNot(BeNil()) + + Expect(healthCheck.Name).To(Equal(controllers.GetHealthMonitoringName(clusterSummary))) + Expect(healthCheck.Name).To(HavePrefix("hm-")) + Expect(healthCheck.Labels).To(HaveKeyWithValue( + configv1beta1.HealthMonitoringClusterSummaryUIDLabel, string(clusterSummary.UID))) + Expect(healthCheck.Annotations).To(HaveKeyWithValue( + configv1beta1.HealthMonitoringClusterSummaryNamespaceAnnotation, clusterSummary.Namespace)) + Expect(healthCheck.Annotations).To(HaveKeyWithValue( + configv1beta1.HealthMonitoringClusterSummaryNameAnnotation, clusterSummary.Name)) + + Expect(healthCheck.Spec.EvaluateIndependently).To(BeTrue()) + Expect(healthCheck.Spec.Flapping).ToNot(BeNil()) + Expect(healthCheck.Spec.Flapping.ConsecutiveEvaluations).To(Equal(int32(controllers.HealthMonitoringFlappingEvaluation))) + // 5 kinds for resources, 5 kinds for the one Helm namespace + Expect(healthCheck.Spec.ResourceSelectors).To(HaveLen(10)) + Expect(healthCheck.Spec.EvaluateHealth).To(ContainSubstring(`ownerName = "platform-stack"`)) + Expect(healthCheck.Spec.EvaluateHealth).To(ContainSubstring(`["cert-manager/cert-manager"] = true`)) + Expect(healthCheck.Spec.EvaluateHealth).ToNot(ContainSubstring(testReleaseNameKyverno)) + }) + + It("builds no HealthCheck when the ClusterSummary deploys nothing to monitor", func() { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + Expect(controllers.BuildMonitoringHealthCheck(clusterSummary, hmOwnerKind, hmOwnerName)).To(BeNil()) + }) + + It("builds a ClusterHealthCheck targeting only the cluster of the ClusterSummary", func() { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + + clusterHealthCheck := controllers.BuildMonitoringClusterHealthCheck(clusterSummary, "hm-123", + []string{"zeta-check", "alpha-check"}) + + Expect(clusterHealthCheck.Name).To(Equal(controllers.GetHealthMonitoringName(clusterSummary))) + Expect(clusterHealthCheck.Spec.ClusterRefs).To(Equal([]corev1.ObjectReference{ + { + APIVersion: libsveltosv1beta1.GroupVersion.String(), Kind: libsveltosv1beta1.SveltosClusterKind, + Namespace: clusterSummary.Spec.ClusterNamespace, Name: clusterSummary.Spec.ClusterName, + }, + })) + // An empty selector matches no cluster: only ClusterRefs apply + Expect(clusterHealthCheck.Spec.ClusterSelector.MatchLabels).To(BeEmpty()) + Expect(clusterHealthCheck.Spec.ClusterSelector.MatchExpressions).To(BeEmpty()) + // Required by the CRD: must be an empty list, not nil (nil is serialized as null) + Expect(clusterHealthCheck.Spec.Notifications).ToNot(BeNil()) + Expect(clusterHealthCheck.Spec.Notifications).To(BeEmpty()) + + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(3)) + Expect(clusterHealthCheck.Spec.LivenessChecks[0].Name).To(Equal(controllers.DefaultHealthCheckLivenessName)) + Expect(clusterHealthCheck.Spec.LivenessChecks[0].LivenessSourceRef.Name).To(Equal("hm-123")) + Expect(clusterHealthCheck.Spec.LivenessChecks[1].LivenessSourceRef.Name).To(Equal("alpha-check")) + Expect(clusterHealthCheck.Spec.LivenessChecks[2].LivenessSourceRef.Name).To(Equal("zeta-check")) + for i := range clusterHealthCheck.Spec.LivenessChecks { + Expect(clusterHealthCheck.Spec.LivenessChecks[i].Type).To(Equal(libsveltosv1beta1.LivenessTypeHealthCheck)) + Expect(clusterHealthCheck.Spec.LivenessChecks[i].LivenessSourceRef.Kind).To(Equal(libsveltosv1beta1.HealthCheckKind)) + } + }) + + It("targets a CAPI cluster with a CAPI cluster reference", func() { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeCapi) + + clusterHealthCheck := controllers.BuildMonitoringClusterHealthCheck(clusterSummary, "hm-123", nil) + + Expect(clusterHealthCheck.Spec.ClusterRefs).To(Equal([]corev1.ObjectReference{ + { + APIVersion: clusterv1.GroupVersion.String(), Kind: "Cluster", + Namespace: clusterSummary.Spec.ClusterNamespace, Name: clusterSummary.Spec.ClusterName, + }, + })) + }) +}) + +var _ = Describe("Health monitoring: reconciliation", func() { + var ( + logger logr.Logger + c client.Client + r *controllers.ClusterSummaryReconciler + ) + + BeforeEach(func() { + logger = textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1))) + c = fake.NewClientBuilder().WithScheme(scheme).Build() + r = getClusterSummaryReconciler(c, fakedeployer.GetClient(context.TODO(), logger, c)) + }) + + monitoredClusterSummary := func() *configv1beta1.ClusterSummary { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + addHelmReleases(clusterSummary, hmCertManagerRelease) + return clusterSummary + } + + getHealthCheck := func(clusterSummary *configv1beta1.ClusterSummary) (*libsveltosv1beta1.HealthCheck, error) { + healthCheck := &libsveltosv1beta1.HealthCheck{} + err := c.Get(context.TODO(), types.NamespacedName{Name: controllers.GetHealthMonitoringName(clusterSummary)}, healthCheck) + return healthCheck, err + } + + getClusterHealthCheck := func(clusterSummary *configv1beta1.ClusterSummary) (*libsveltosv1beta1.ClusterHealthCheck, error) { + clusterHealthCheck := &libsveltosv1beta1.ClusterHealthCheck{} + err := c.Get(context.TODO(), + types.NamespacedName{Name: controllers.GetHealthMonitoringName(clusterSummary)}, clusterHealthCheck) + return clusterHealthCheck, err + } + + It("creates nothing while features are not all provisioned", func() { + clusterSummary := monitoredClusterSummary() + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioned}, + {FeatureID: libsveltosv1beta1.FeatureResources, Status: libsveltosv1beta1.FeatureStatusProvisioning}, + } + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + _, err = getClusterHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("creates the HealthCheck and the ClusterHealthCheck once all features are provisioned", func() { + clusterSummary := monitoredClusterSummary() + provisionAllFeatures(clusterSummary) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + healthCheck, err := getHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(healthCheck.Spec.ResourceSelectors).To(HaveLen(5)) + + clusterHealthCheck, err := getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(1)) + Expect(clusterHealthCheck.Spec.LivenessChecks[0].LivenessSourceRef.Name).To(Equal(healthCheck.Name)) + }) + + It("keeps the resources, and updates the HealthCheck, when the deployed Helm releases change", func() { + clusterSummary := monitoredClusterSummary() + provisionAllFeatures(clusterSummary) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + // The ClusterSummary is deploying again: nothing is touched + clusterSummary.Status.FeatureSummaries[0].Status = libsveltosv1beta1.FeatureStatusProvisioning + addHelmReleases(clusterSummary, testReleaseNameKyverno+"/"+testReleaseNameKyverno) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + healthCheck, err := getHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(healthCheck.Spec.ResourceSelectors).To(HaveLen(5)) + + // Provisioned again: the new release is monitored + provisionAllFeatures(clusterSummary) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + healthCheck, err = getHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(healthCheck.Spec.ResourceSelectors).To(HaveLen(10)) + Expect(healthCheck.Spec.EvaluateHealth).To(ContainSubstring(`["` + testReleaseNameKyverno + `/` + testReleaseNameKyverno + `"] = true`)) + }) + + It("monitors only the listed HealthChecks when defaultChecks is false", func() { + clusterSummary := monitoredClusterSummary() + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.DefaultChecks = ptr.To(false) + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.HealthChecks = []string{hmUserHealthCheck} + provisionAllFeatures(clusterSummary) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + + clusterHealthCheck, err := getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(1)) + Expect(clusterHealthCheck.Spec.LivenessChecks[0].LivenessSourceRef.Name).To(Equal(hmUserHealthCheck)) + }) + + It("monitors the default and the listed HealthChecks, and removes the generated one when not needed anymore", func() { + clusterSummary := monitoredClusterSummary() + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.HealthChecks = []string{hmUserHealthCheck} + provisionAllFeatures(clusterSummary) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + clusterHealthCheck, err := getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(2)) + _, err = getHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.DefaultChecks = ptr.To(false) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + clusterHealthCheck, err = getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(1)) + _, err = getHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("creates nothing when there is nothing to monitor", func() { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + provisionAllFeatures(clusterSummary) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getClusterHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("removes the resources when monitoring is disabled", func() { + clusterSummary := monitoredClusterSummary() + provisionAllFeatures(clusterSummary) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + // A HealthCheck written by the user must not be touched + userHealthCheck := &libsveltosv1beta1.HealthCheck{ObjectMeta: metav1.ObjectMeta{Name: hmUserHealthCheck}} + Expect(c.Create(context.TODO(), userHealthCheck)).To(Succeed()) + + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.Enabled = false + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + _, err = getClusterHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + Expect(c.Get(context.TODO(), client.ObjectKeyFromObject(userHealthCheck), userHealthCheck)).To(Succeed()) + }) + + It("removeHealthMonitoring deletes only what was created for the ClusterSummary", func() { + clusterSummary := monitoredClusterSummary() + provisionAllFeatures(clusterSummary) + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + // Same name, but created for another ClusterSummary (different UID label) + other := clusterSummary.DeepCopy() + other.UID = types.UID(randomString()) + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), other, logger)).To(Succeed()) + _, err := getHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + _, err = getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + _, err = getHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + _, err = getClusterHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + + // Nothing left to remove is not an error + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + }) + + It("does nothing for a DryRun ClusterSummary", func() { + clusterSummary := monitoredClusterSummary() + clusterSummary.Spec.ClusterProfileSpec.SyncMode = configv1beta1.SyncModeDryRun + provisionAllFeatures(clusterSummary) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getClusterHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) +}) + +var _ = Describe("Health monitoring: HealthChecks of a Profile", func() { + var ( + logger logr.Logger + c client.Client + r *controllers.ClusterSummaryReconciler + recorder *events.FakeRecorder + ) + + BeforeEach(func() { + logger = textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1))) + c = fake.NewClientBuilder().WithScheme(scheme).Build() + r = getClusterSummaryReconciler(c, fakedeployer.GetClient(context.TODO(), logger, c)) + recorder = events.NewFakeRecorder(10) + controllers.SetEventRecorder(r, recorder) + }) + + // newProvisionedClusterSummary returns a provisioned ClusterSummary, owned by a profile of kind + // profileKind, listing a HealthCheck + newProvisionedClusterSummary := func(profileKind string) *configv1beta1.ClusterSummary { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + clusterSummary.OwnerReferences[0].Kind = profileKind + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{ + Enabled: true, HealthChecks: []string{hmUserHealthCheck}, + } + addHelmReleases(clusterSummary, hmCertManagerRelease) + provisionAllFeatures(clusterSummary) + return clusterSummary + } + + getClusterHealthCheck := func(clusterSummary *configv1beta1.ClusterSummary) (*libsveltosv1beta1.ClusterHealthCheck, error) { + clusterHealthCheck := &libsveltosv1beta1.ClusterHealthCheck{} + err := c.Get(context.TODO(), + types.NamespacedName{Name: controllers.GetHealthMonitoringName(clusterSummary)}, clusterHealthCheck) + return clusterHealthCheck, err + } + + It("a Profile gets the default checks only, and a warning explains why the HealthChecks are ignored", func() { + clusterSummary := newProvisionedClusterSummary(configv1beta1.ProfileKind) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + clusterHealthCheck, err := getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(1)) + Expect(clusterHealthCheck.Spec.LivenessChecks[0].Name).To(Equal(controllers.DefaultHealthCheckLivenessName)) + Expect(clusterHealthCheck.Spec.LivenessChecks[0].LivenessSourceRef.Name).ToNot(Equal(hmUserHealthCheck)) + + Expect(recorder.Events).To(Receive(And( + ContainSubstring("Warning HealthChecksIgnored"), ContainSubstring("only a ClusterProfile")))) + }) + + It("a Profile listing HealthChecks, with defaultChecks false, monitors nothing", func() { + clusterSummary := newProvisionedClusterSummary(configv1beta1.ProfileKind) + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.DefaultChecks = ptr.To(false) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getClusterHealthCheck(clusterSummary) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + Expect(recorder.Events).To(Receive(ContainSubstring("Warning HealthChecksIgnored"))) + }) + + It("a Profile listing no HealthChecks gets no warning", func() { + clusterSummary := newProvisionedClusterSummary(configv1beta1.ProfileKind) + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.HealthChecks = nil + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + _, err := getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(recorder.Events).ToNot(Receive()) + }) + + It("a ClusterProfile keeps its HealthChecks and gets no warning", func() { + clusterSummary := newProvisionedClusterSummary(configv1beta1.ClusterProfileKind) + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + clusterHealthCheck, err := getClusterHealthCheck(clusterSummary) + Expect(err).To(BeNil()) + Expect(clusterHealthCheck.Spec.LivenessChecks).To(HaveLen(2)) + Expect(recorder.Events).ToNot(Receive()) + }) +}) + +var _ = Describe("Health monitoring: ownership and failures", func() { + var ( + logger logr.Logger + c client.Client + r *controllers.ClusterSummaryReconciler + ) + + BeforeEach(func() { + logger = textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1))) + }) + + newReconciler := func(funcs interceptor.Funcs) { + c = fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(funcs).Build() + r = getClusterSummaryReconciler(c, fakedeployer.GetClient(context.TODO(), logger, c)) + } + + newProvisionedClusterSummary := func() *configv1beta1.ClusterSummary { + clusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + addHelmReleases(clusterSummary, hmCertManagerRelease) + provisionAllFeatures(clusterSummary) + return clusterSummary + } + + nameOf := func(clusterSummary *configv1beta1.ClusterSummary) types.NamespacedName { + return types.NamespacedName{Name: controllers.GetHealthMonitoringName(clusterSummary)} + } + + It("never deletes an object with the generated name which another ClusterSummary created", func() { + newReconciler(interceptor.Funcs{}) + clusterSummary := newProvisionedClusterSummary() + + // No label at all, and the label of another ClusterSummary + for _, labels := range []map[string]string{ + nil, + {configv1beta1.HealthMonitoringClusterSummaryUIDLabel: randomString()}, + } { + healthCheck := &libsveltosv1beta1.HealthCheck{ + ObjectMeta: metav1.ObjectMeta{Name: nameOf(clusterSummary).Name, Labels: labels}} + clusterHealthCheck := &libsveltosv1beta1.ClusterHealthCheck{ + ObjectMeta: metav1.ObjectMeta{Name: nameOf(clusterSummary).Name, Labels: labels}} + Expect(c.Create(context.TODO(), healthCheck)).To(Succeed()) + Expect(c.Create(context.TODO(), clusterHealthCheck)).To(Succeed()) + + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + Expect(c.Get(context.TODO(), nameOf(clusterSummary), &libsveltosv1beta1.HealthCheck{})).To(Succeed()) + Expect(c.Get(context.TODO(), nameOf(clusterSummary), &libsveltosv1beta1.ClusterHealthCheck{})).To(Succeed()) + + Expect(c.Delete(context.TODO(), healthCheck)).To(Succeed()) + Expect(c.Delete(context.TODO(), clusterHealthCheck)).To(Succeed()) + } + }) + + It("does not touch the objects of another ClusterSummary when monitoring is disabled", func() { + newReconciler(interceptor.Funcs{}) + clusterSummary := newProvisionedClusterSummary() + clusterHealthCheck := &libsveltosv1beta1.ClusterHealthCheck{ObjectMeta: metav1.ObjectMeta{ + Name: nameOf(clusterSummary).Name, + Labels: map[string]string{configv1beta1.HealthMonitoringClusterSummaryUIDLabel: randomString()}, + }} + Expect(c.Create(context.TODO(), clusterHealthCheck)).To(Succeed()) + + clusterSummary.Spec.ClusterProfileSpec.HealthMonitoring.Enabled = false + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + Expect(c.Get(context.TODO(), nameOf(clusterSummary), &libsveltosv1beta1.ClusterHealthCheck{})).To(Succeed()) + }) + + It("creates nothing for a ClusterSummary being deleted", func() { + newReconciler(interceptor.Funcs{}) + clusterSummary := newProvisionedClusterSummary() + clusterSummary.DeletionTimestamp = &metav1.Time{Time: time.Now()} + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + err := c.Get(context.TODO(), nameOf(clusterSummary), &libsveltosv1beta1.ClusterHealthCheck{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("creates nothing for a ClusterSummary with no feature summaries", func() { + newReconciler(interceptor.Funcs{}) + clusterSummary := newProvisionedClusterSummary() + clusterSummary.Status.FeatureSummaries = nil + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + err := c.Get(context.TODO(), nameOf(clusterSummary), &libsveltosv1beta1.ClusterHealthCheck{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("fails, creating nothing, when the profile owning the ClusterSummary cannot be found", func() { + newReconciler(interceptor.Funcs{}) + clusterSummary := newProvisionedClusterSummary() + clusterSummary.OwnerReferences = nil + + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).ToNot(Succeed()) + + err := c.Get(context.TODO(), nameOf(clusterSummary), &libsveltosv1beta1.ClusterHealthCheck{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("builds the same HealthCheck whatever the order of the Helm releases", func() { + first := newProvisionedClusterSummary() + first.Status.HelmReleaseSummaries = nil + addHelmReleases(first, hmCertManagerRelease, hmGrafanaRelease, "monitoring/loki") + + second := first.DeepCopy() + second.Status.HelmReleaseSummaries = nil + addHelmReleases(second, "monitoring/loki", hmGrafanaRelease, hmCertManagerRelease) + + firstHealthCheck := controllers.BuildMonitoringHealthCheck(first, hmOwnerKind, hmOwnerName) + secondHealthCheck := controllers.BuildMonitoringHealthCheck(second, hmOwnerKind, hmOwnerName) + Expect(firstHealthCheck.Spec).To(Equal(secondHealthCheck.Spec)) + }) + + It("removing is not an error when the HealthCheck CRDs are not installed", func() { + newReconciler(interceptor.Funcs{ + Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, _ client.Object, + _ ...client.GetOption) error { + + return &meta.NoKindMatchError{} + }, + }) + + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), newProvisionedClusterSummary(), logger)).To(Succeed()) + }) + + It("removing fails when the object cannot be read", func() { + newReconciler(interceptor.Funcs{ + Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, _ client.Object, + _ ...client.GetOption) error { + + return errors.New("read failure") + }, + }) + + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), newProvisionedClusterSummary(), logger)).ToNot(Succeed()) + }) + + It("removing fails when an object cannot be deleted, and ignores one deleted in the meantime", func() { + deleteError := errors.New("delete failure") + newReconciler(interceptor.Funcs{ + Delete: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.DeleteOption) error { + return deleteError + }, + }) + clusterSummary := newProvisionedClusterSummary() + Expect(controllers.ReconcileHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), clusterSummary, logger)).ToNot(Succeed()) + + deleteError = apierrors.NewNotFound(schema.GroupResource{}, "deleted already") + Expect(controllers.RemoveHealthMonitoring(r, context.TODO(), clusterSummary, logger)).To(Succeed()) + }) +}) + +var _ = Describe("Health monitoring: validation of healthMonitoring", func() { + // createClusterProfile creates a ClusterProfile in the API server, so the validation rules of the CRD apply + createClusterProfile := func(healthMonitoring *configv1beta1.HealthMonitoring) error { + clusterProfile := &configv1beta1.ClusterProfile{ + ObjectMeta: metav1.ObjectMeta{Name: randomString()}, + Spec: configv1beta1.Spec{HealthMonitoring: healthMonitoring}, + } + err := testEnv.Create(context.TODO(), clusterProfile) + if err == nil { + DeferCleanup(func() { Expect(testEnv.Delete(context.TODO(), clusterProfile)).To(Succeed()) }) + } + return err + } + + It("accepts valid combinations", func() { + Expect(createClusterProfile(nil)).To(Succeed()) + Expect(createClusterProfile(&configv1beta1.HealthMonitoring{Enabled: false})).To(Succeed()) + Expect(createClusterProfile(&configv1beta1.HealthMonitoring{Enabled: true})).To(Succeed()) + Expect(createClusterProfile(&configv1beta1.HealthMonitoring{ + Enabled: true, HealthChecks: []string{hmUserHealthCheck}})).To(Succeed()) + Expect(createClusterProfile(&configv1beta1.HealthMonitoring{ + Enabled: true, DefaultChecks: ptr.To(false), HealthChecks: []string{hmUserHealthCheck}})).To(Succeed()) + }) + + It("rejects healthChecks when monitoring is not enabled", func() { + err := createClusterProfile(&configv1beta1.HealthMonitoring{ + Enabled: false, HealthChecks: []string{hmUserHealthCheck}}) + Expect(err).ToNot(BeNil()) + Expect(err.Error()).To(ContainSubstring("healthChecks can only be set when enabled is true")) + }) + + It("rejects defaultChecks false without healthChecks, as nothing would be monitored", func() { + err := createClusterProfile(&configv1beta1.HealthMonitoring{Enabled: true, DefaultChecks: ptr.To(false)}) + Expect(err).ToNot(BeNil()) + Expect(err.Error()).To(ContainSubstring("defaultChecks cannot be false when healthChecks is empty")) + }) +}) + +var _ = Describe("Health monitoring: ClusterSummary predicate", func() { + var predicate controllers.ClusterSummaryPredicate + + BeforeEach(func() { + predicate = controllers.ClusterSummaryPredicate{ + Logger: textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1))), + } + }) + + It("reconciles when all features become provisioned and health monitoring is enabled", func() { + oldClusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + oldClusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + oldClusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioning}, + } + + newClusterSummary := oldClusterSummary.DeepCopy() + newClusterSummary.Status.FeatureSummaries[0].Status = libsveltosv1beta1.FeatureStatusProvisioned + + Expect(predicate.Update(event.UpdateEvent{ObjectOld: oldClusterSummary, ObjectNew: newClusterSummary})).To(BeTrue()) + }) + + It("reconciles when the Helm releases managed by the ClusterSummary change", func() { + oldClusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + oldClusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + provisionAllFeatures(oldClusterSummary) + addHelmReleases(oldClusterSummary, hmCertManagerRelease) + + newClusterSummary := oldClusterSummary.DeepCopy() + addHelmReleases(newClusterSummary, testReleaseNameKyverno+"/"+testReleaseNameKyverno) + + Expect(predicate.Update(event.UpdateEvent{ObjectOld: oldClusterSummary, ObjectNew: newClusterSummary})).To(BeTrue()) + }) + + It("does not reconcile for a status change that does not matter to health monitoring", func() { + oldClusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + oldClusterSummary.Spec.ClusterProfileSpec.HealthMonitoring = &configv1beta1.HealthMonitoring{Enabled: true} + provisionAllFeatures(oldClusterSummary) + + newClusterSummary := oldClusterSummary.DeepCopy() + message := "something" + newClusterSummary.Status.Dependencies = &message + + Expect(predicate.Update(event.UpdateEvent{ObjectOld: oldClusterSummary, ObjectNew: newClusterSummary})).To(BeFalse()) + }) + + It("keeps ignoring features becoming provisioned when health monitoring is not enabled", func() { + oldClusterSummary := newHealthMonitoringClusterSummary(libsveltosv1beta1.ClusterTypeSveltos) + oldClusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureHelm, Status: libsveltosv1beta1.FeatureStatusProvisioning}, + } + + newClusterSummary := oldClusterSummary.DeepCopy() + newClusterSummary.Status.FeatureSummaries[0].Status = libsveltosv1beta1.FeatureStatusProvisioned + + Expect(predicate.Update(event.UpdateEvent{ObjectOld: oldClusterSummary, ObjectNew: newClusterSummary})).To(BeFalse()) + }) +}) diff --git a/controllers/healthmonitoring/health.lua b/controllers/healthmonitoring/health.lua new file mode 100644 index 00000000..4dc995eb --- /dev/null +++ b/controllers/healthmonitoring/health.lua @@ -0,0 +1,224 @@ +-- Sveltos health monitoring: health of the Deployments, StatefulSets, DaemonSets, Jobs and +-- PersistentVolumeClaims deployed by a profile. +-- +-- addon-controller prepends a "config" table to this script: +-- config.ownerKind ClusterProfile or Profile +-- config.ownerName name of the profile +-- config.helmReleases set of "/" for the Helm charts of the profile +-- +-- The health rules follow the ones of Argo CD (gitops-engine, pkg/health, Apache-2.0). + +local function isOwned(resource) + local annotations = (resource.metadata or {}).annotations or {} + + -- Resources deployed by Sveltos (raw YAML, Kustomize, ...) carry the owner annotations + if annotations["projectsveltos.io/owner-kind"] == config.ownerKind and + annotations["projectsveltos.io/owner-name"] == config.ownerName then + return true + end + + -- Resources deployed by a Helm chart carry the Helm release annotations + local release = annotations["meta.helm.sh/release-name"] + local namespace = annotations["meta.helm.sh/release-namespace"] + if release ~= nil and namespace ~= nil then + return config.helmReleases[namespace .. "/" .. release] == true + end + + return false +end + +local function deploymentHealth(resource) + local metadata = resource.metadata or {} + local spec = resource.spec or {} + local status = resource.status or {} + + if spec.paused then + return "Suspended", "Deployment is paused" + end + + if (metadata.generation or 0) > (status.observedGeneration or 0) then + return "Progressing", "Waiting for rollout to finish: observed deployment generation less than desired generation" + end + + for _, condition in ipairs(status.conditions or {}) do + if condition.type == "Progressing" and condition.reason == "ProgressDeadlineExceeded" then + return "Degraded", string.format("Deployment %q exceeded its progress deadline", metadata.name or "") + end + end + + local desired = spec.replicas or 1 + local updated = status.updatedReplicas or 0 + local current = status.replicas or 0 + local available = status.availableReplicas or 0 + + if updated < desired then + return "Progressing", string.format( + "Waiting for rollout to finish: %d out of %d new replicas have been updated", updated, desired) + end + if current > updated then + return "Progressing", string.format( + "Waiting for rollout to finish: %d old replicas are pending termination", current - updated) + end + if available < updated then + return "Progressing", string.format( + "Waiting for rollout to finish: %d of %d updated replicas are available", available, updated) + end + + return "Healthy", "" +end + +local function statefulSetHealth(resource) + local metadata = resource.metadata or {} + local spec = resource.spec or {} + local status = resource.status or {} + + local observed = status.observedGeneration or 0 + if observed == 0 or (metadata.generation or 0) > observed then + return "Progressing", "Waiting for statefulset spec update to be observed" + end + + local ready = status.readyReplicas or 0 + if spec.replicas ~= nil and ready < spec.replicas then + return "Progressing", string.format("Waiting for %d pods to be ready", spec.replicas - ready) + end + + local strategy = spec.updateStrategy or {} + if strategy.type == "OnDelete" then + return "Healthy", "" + end + + if strategy.type == "RollingUpdate" and strategy.rollingUpdate ~= nil then + local partition = strategy.rollingUpdate.partition + if partition ~= nil and spec.replicas ~= nil then + local expected = spec.replicas - partition + local updated = status.updatedReplicas or 0 + if updated < expected then + return "Progressing", string.format( + "Waiting for partitioned roll out to finish: %d out of %d new pods have been updated", updated, expected) + end + end + return "Healthy", "" + end + + if status.updateRevision ~= status.currentRevision then + return "Progressing", string.format("Waiting for rolling update to finish: current revision %s, update revision %s", + tostring(status.currentRevision), tostring(status.updateRevision)) + end + + return "Healthy", "" +end + +local function daemonSetHealth(resource) + local metadata = resource.metadata or {} + local spec = resource.spec or {} + local status = resource.status or {} + + if (metadata.generation or 0) > (status.observedGeneration or 0) then + return "Progressing", "Waiting for rollout to finish: observed daemon set generation less than desired generation" + end + + local strategy = spec.updateStrategy or {} + if strategy.type == "OnDelete" then + return "Healthy", "" + end + + local desired = status.desiredNumberScheduled or 0 + local updated = status.updatedNumberScheduled or 0 + local available = status.numberAvailable or 0 + + if updated < desired then + return "Progressing", string.format( + "Waiting for rollout to finish: %d out of %d new pods have been updated", updated, desired) + end + if available < desired then + return "Progressing", string.format( + "Waiting for rollout to finish: %d of %d updated pods are available", available, desired) + end + + return "Healthy", "" +end + +-- A Job is finished once the Job controller adds a Failed or a Complete condition. Until then it is +-- Progressing, a Job that is running is reported like Argo CD does. +local function jobHealth(resource) + local status = resource.status or {} + + local complete = false + local suspended = false + for _, condition in ipairs(status.conditions or {}) do + if condition.status == "True" then + if condition.type == "Failed" then + return "Degraded", string.format("Job failed: %s", condition.message or condition.reason or "unknown reason") + elseif condition.type == "Complete" then + complete = true + elseif condition.type == "Suspended" then + suspended = true + end + end + end + + if complete then + return "Healthy", "" + end + if suspended then + return "Suspended", "Job is suspended" + end + + return "Progressing", "Job has not completed yet" +end + +-- A PersistentVolumeClaim whose storage class waits for the first consumer stays Pending until a Pod +-- uses it: the phase alone cannot tell it from a claim that cannot be bound. +local function persistentVolumeClaimHealth(resource) + local phase = (resource.status or {}).phase + + if phase == "Bound" then + return "Healthy", "" + elseif phase == "Lost" then + return "Degraded", "PersistentVolumeClaim lost its PersistentVolume" + elseif phase == "Pending" then + return "Progressing", "PersistentVolumeClaim is not bound yet" + end + + return "Progressing", string.format("PersistentVolumeClaim has no known phase yet (%s)", tostring(phase)) +end + +local healthByKind = { + Deployment = deploymentHealth, + StatefulSet = statefulSetHealth, + DaemonSet = daemonSetHealth, + Job = jobHealth, + PersistentVolumeClaim = persistentVolumeClaimHealth, +} + +function evaluate() + local hs = {} + hs.status = "Healthy" + hs.message = "" + + local unhealthy = {} + -- The same object can be selected twice: by label (deployed by Sveltos) and by namespace (Helm) + local seen = {} + + for _, resource in ipairs(resources) do + local uid = (resource.metadata or {}).uid + local health = healthByKind[resource.kind] + + if health ~= nil and (uid == nil or not seen[uid]) and isOwned(resource) then + if uid ~= nil then + seen[uid] = true + end + + local status, message = health(resource) + if status ~= "Healthy" then + table.insert(unhealthy, {resource = resource, status = status, message = message}) + end + end + end + + if #unhealthy > 0 then + hs.resources = unhealthy + end + + return hs +end diff --git a/go.mod b/go.mod index 06380685..9a10cfc1 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/onsi/gomega v1.44.0 github.com/opencontainers/image-spec v1.1.1 github.com/pkg/errors v0.9.1 - github.com/projectsveltos/libsveltos v1.16.1-0.20261011133909-b577d6e6fe69 + github.com/projectsveltos/libsveltos v1.16.1-0.20261014064744-50f7977def53 github.com/prometheus/client_golang v1.25.0 github.com/sigstore/cosign/v3 v3.1.3 github.com/sigstore/sigstore v1.11.0 diff --git a/go.sum b/go.sum index c1c4d0c0..516bff9c 100644 --- a/go.sum +++ b/go.sum @@ -643,8 +643,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= -github.com/projectsveltos/libsveltos v1.16.1-0.20261011133909-b577d6e6fe69 h1:rg2W4If7Ejm/3OOLeGiu8GywpK8nFEOfbTsFdVHG+Ho= -github.com/projectsveltos/libsveltos v1.16.1-0.20261011133909-b577d6e6fe69/go.mod h1:wup25BAf1SHuG6FsY8D+tnm1U3/3NiuzcKec/In/fB4= +github.com/projectsveltos/libsveltos v1.16.1-0.20261014064744-50f7977def53 h1:7RH3CeioAs44oednP7fME8cDn51tNVXJIZHWLaUYhdw= +github.com/projectsveltos/libsveltos v1.16.1-0.20261014064744-50f7977def53/go.mod h1:wup25BAf1SHuG6FsY8D+tnm1U3/3NiuzcKec/In/fB4= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5 h1:khnc+994UszxZYu69J+R5FKiLA/Nk1JQj0EYAkwTWz0= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5/go.mod h1:yVL8KQFa9tmcxgwl9nwIMtKgtmIVC1zaFRSCfOwYvPY= github.com/projectsveltos/lua-utils/glua-runes v0.0.0-20251212200258-2b3cdcb7c0f5 h1:YbsebwRwTRhV8QacvEAdFqxcxHdeu7JTVtsBovbkgos= diff --git a/lib/crd/clusterprofiles.go b/lib/crd/clusterprofiles.go index be9c350f..7e32a1bc 100644 --- a/lib/crd/clusterprofiles.go +++ b/lib/crd/clusterprofiles.go @@ -256,6 +256,55 @@ spec: ` + "`" + `ExtraLabels` + "`" + `, the value from ` + "`" + `ExtraLabels` + "`" + ` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is empty: + there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/lib/crd/clusterpromotions.go b/lib/crd/clusterpromotions.go index b41a067a..09c24c74 100644 --- a/lib/crd/clusterpromotions.go +++ b/lib/crd/clusterpromotions.go @@ -155,6 +155,55 @@ spec: type: object type: array x-kubernetes-list-type: atomic + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is + empty: there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/lib/crd/clustersummaries.go b/lib/crd/clustersummaries.go index 99b608b2..145251b9 100644 --- a/lib/crd/clustersummaries.go +++ b/lib/crd/clustersummaries.go @@ -292,6 +292,55 @@ spec: ` + "`" + `ExtraLabels` + "`" + `, the value from ` + "`" + `ExtraLabels` + "`" + ` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is + empty: there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/lib/crd/profiles.go b/lib/crd/profiles.go index b31ba277..e8012088 100644 --- a/lib/crd/profiles.go +++ b/lib/crd/profiles.go @@ -256,6 +256,55 @@ spec: ` + "`" + `ExtraLabels` + "`" + `, the value from ` + "`" + `ExtraLabels` + "`" + ` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is empty: + there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml index 51c3b551..41151bc5 100644 --- a/manifest/manifest.yaml +++ b/manifest/manifest.yaml @@ -546,6 +546,55 @@ spec: `ExtraLabels`, the value from `ExtraLabels` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is empty: + there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed @@ -3041,6 +3090,55 @@ spec: type: object type: array x-kubernetes-list-type: atomic + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is + empty: there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed @@ -6712,6 +6810,55 @@ spec: `ExtraLabels`, the value from `ExtraLabels` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is + empty: there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed @@ -9378,6 +9525,55 @@ spec: `ExtraLabels`, the value from `ExtraLabels` will override the existing value. (Deprecated use Patches instead) type: object + healthMonitoring: + description: |- + HealthMonitoring, when enabled, continuously monitors the health of the resources + this profile deploys in each matching cluster. Unlike ValidateHealths, which are + evaluated once during deployment, these checks run for as long as the profile + is deployed. Results are available as HealthCheckReports. + properties: + defaultChecks: + default: true + description: |- + DefaultChecks, when true (default), monitors the Deployments, StatefulSets, DaemonSets, + Jobs and PersistentVolumeClaims deployed by this profile (raw resources and Helm charts). + Set it to false to monitor only the HealthChecks listed in HealthChecks. + type: boolean + enabled: + description: |- + Enabled turns health monitoring on for the resources deployed by this profile. + When true, Sveltos creates a HealthCheck and a ClusterHealthCheck for each + matching cluster once the profile is provisioned there, and removes them + when monitoring is disabled or the cluster does not match anymore. + type: boolean + healthChecks: + description: |- + HealthChecks is a list of names of HealthCheck instances to monitor in addition + to the default checks. Those are deployed as they are: selecting only the + resources deployed by this profile is up to the HealthCheck. + + Only a ClusterProfile can set it. It is ignored for a (namespaced) Profile, which gets the + default checks only. A HealthCheck is a cluster-scoped resource referenced by name, and the + report it produces is shared by every profile that references it on a given cluster, with + no filtering by namespace. Allowing a Profile to reference one would let a tenant name + a HealthCheck it does not own, and learn about resources outside its namespace through + the shared report. The default checks are safe for a Profile: they select only the + resources that profile deployed. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - enabled + type: object + x-kubernetes-validations: + - message: healthChecks can only be set when enabled is true + rule: self.enabled || !has(self.healthChecks) || size(self.healthChecks) + == 0 + - message: 'defaultChecks cannot be false when healthChecks is empty: + there would be nothing to monitor' + rule: '!self.enabled || !has(self.defaultChecks) || self.defaultChecks + || (has(self.healthChecks) && size(self.healthChecks) > 0)' helmCharts: description: Helm charts is a list of helm charts that need to be deployed @@ -11930,21 +12126,10 @@ rules: - lib.projectsveltos.io resources: - clusterhealthchecks - - configurationgroups/status - - debuggingconfigurations - - eventtriggers - - sveltosclusters - - sveltosclusters/status - verbs: - - get - - list - - watch -- apiGroups: - - lib.projectsveltos.io - resources: - clustersets - configurationbundles - configurationgroups + - healthchecks - reloaders - resourcesummaries - sets @@ -11981,6 +12166,18 @@ rules: - list - update - watch +- apiGroups: + - lib.projectsveltos.io + resources: + - configurationgroups/status + - debuggingconfigurations + - eventtriggers + - sveltosclusters + - sveltosclusters/status + verbs: + - get + - list + - watch - apiGroups: - lib.projectsveltos.io resources: