Handle AWS/Azure/GCP CSI differences, and get rid of global.datacenter variable

Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.3
+version: 0.3.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

README.md

@@ -1,11 +1,18 @@
 # openshift-data-foundations
 
-![Version: 0.2.3](https://img.shields.io/badge/Version-0.2.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square)
+![Version: 0.3.0](https://img.shields.io/badge/Version-0.3.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square)
 
 A Helm chart to install ODF on Openshift
 
 ### Notable changes
 
+* v0.3.0: Replace `global.datacenter.storageClassName` with `odf.osd.pvc.storageClassName`.
+          When unset, the OSD storage class is derived from `global.clusterPlatform`
+          (AWS: `gp3-csi`, Azure: `managed-csi`, GCP: `standard-csi`).
+          This is a backwards-incompatible change.
+          Also add a CronJob that periodically re-runs the same node-labeling
+          logic as the bootstrap Job (same useSpecificNodes / selector conditions).
+
 * v0.2.3: Allow passing of label selector to labelling job to avoid, for example, labelling submariner nodes.
           Also ensure at least 3 nodes are labelled when using label selector.
 
@@ -23,15 +30,18 @@ A Helm chart to install ODF on Openshift
 
 ## Notes
 
-This branch currently tracks the v0.2.x releases which use the host as a
+This branch currently tracks the v0.3.x releases which use the host as a
 default failure domain for objectStorage.
 
 ## Values
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| global.datacenter.storageClassName | string | `"gp3-csi"` |  |
+| global.clusterPlatform | string | `""` | OpenShift cluster platform (AWS, Azure, GCP). Used to select the default OSD storage class when odf.osd.pvc.storageClassName is empty. |
+| job.failedJobsHistoryLimit | int | `1` | failedJobsHistoryLimit for the label-storage-nodes CronJob. |
 | job.image | string | `"image-registry.openshift-image-registry.svc:5000/openshift/cli:latest"` |  |
+| job.schedule | string | `"*/15 * * * *"` | Cron schedule for re-labeling storage nodes after the initial Job (UTC). |
+| job.successfulJobsHistoryLimit | int | `3` | successfulJobsHistoryLimit for the label-storage-nodes CronJob. |
 | objectStorage.dataPool.failureDomain | string | `"host"` | Failuredomain for the dataPool |
 | objectStorage.dataPool.replicas | int | `3` |  |
 | objectStorage.enable | bool | `true` |  |
@@ -55,6 +65,7 @@ default failure domain for objectStorage.
 | odf.noobaadb.requests.cpu | int | `1` |  |
 | odf.noobaadb.requests.memory | string | `"4Gi"` |  |
 | odf.osd.pvc.storage | string | `"2Ti"` |  |
+| odf.osd.pvc.storageClassName | string | `""` | Storage class for ODF OSD volumes. Empty selects a platform default from global.clusterPlatform (AWS: gp3-csi, Azure: managed-csi, GCP: standard-csi). |
 | odf.osd.requests.cpu | int | `2` |  |
 | odf.osd.requests.memory | string | `"5Gi"` |  |
 | odf.serviceUrl | string | `"http://rook-ceph-rgw-ocs-storagecluster-cephobjectstore.openshift-storage.svc.cluster.local"` |  |

README.md.gotmpl

@@ -7,6 +7,14 @@
 
 ### Notable changes
 
+* v0.3.0: Replace `global.datacenter.storageClassName` with `odf.osd.pvc.storageClassName`.
+          When unset, the OSD storage class is derived from `global.clusterPlatform`
+          (AWS: `gp3-csi`, Azure: `managed-csi`, GCP: `standard-csi`).
+          This is a backwards-incompatible change.
+          Also add a CronJob that periodically re-runs the same node-labeling
+          logic as the bootstrap Job (same useSpecificNodes / selector conditions).
+
+
 * v0.2.3: Allow passing of label selector to labelling job to avoid, for example, labelling submariner nodes.
           Also ensure at least 3 nodes are labelled when using label selector.
 
@@ -20,7 +28,7 @@
 
 ## Notes
 
-This branch currently tracks the v0.2.x releases which use the host as a
+This branch currently tracks the v0.3.x releases which use the host as a
 default failure domain for objectStorage.
 
 {{ template "chart.sourcesSection" . }}

templates/_helpers.tpl

@@ -0,0 +1,64 @@
+{{/*
+Return the storage class used for ODF OSD volume PVCs.
+
+Uses odf.osd.pvc.storageClassName when set; otherwise selects a platform
+default from global.clusterPlatform:
+
+  AWS (or unset) -> gp3-csi
+  Azure          -> managed-csi
+  GCP            -> standard-csi
+*/}}
+{{- define "odf.osd.storageClassName" -}}
+{{- if .Values.odf.osd.pvc.storageClassName -}}
+{{- .Values.odf.osd.pvc.storageClassName -}}
+{{- else -}}
+{{- $platform := .Values.global.clusterPlatform | default "" | lower -}}
+{{- if or (eq $platform "") (eq $platform "aws") -}}
+gp3-csi
+{{- else if eq $platform "azure" -}}
+managed-csi
+{{- else if eq $platform "gcp" -}}
+standard-csi
+{{- else -}}
+{{- fail (printf "Set odf.osd.pvc.storageClassName or use a supported global.clusterPlatform (AWS, Azure, GCP); got %q" .Values.global.clusterPlatform) -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Bash script used by the label-storage-nodes Job and CronJob.
+Respects storageSystem.inventory.useSpecificNodes vs nodeJobLabelSelector.
+*/}}
+{{- define "odf.labelStorageNodes.script" -}}
+{{- if .Values.storageSystem.inventory.useSpecificNodes }}
+{{- range .Values.storageSystem.inventory.nodes }}
+oc label node {{ . }} cluster.ocs.openshift.io/openshift-storage='' --overwrite
+{{- end }}
+{{- else }}
+oc label nodes -l {{ .Values.storageSystem.inventory.nodeJobLabelSelector | squote }} cluster.ocs.openshift.io/openshift-storage='' --overwrite
+LABELED_COUNT=$(oc get nodes -l cluster.ocs.openshift.io/openshift-storage --no-headers 2>/dev/null | wc -l)
+if [ "$LABELED_COUNT" -lt 3 ]; then
+  echo "Error: Only $LABELED_COUNT node(s) were labeled. At least 3 nodes must be labeled."
+  exit 1
+fi
+echo "Successfully labeled $LABELED_COUNT node(s)"
+{{- end }}
+{{- end }}
+
+{{/*
+Pod spec shared by the label-storage-nodes Job and CronJob.
+*/}}
+{{- define "odf.labelStorageNodes.podSpec" -}}
+containers:
+- image: {{ .Values.job.image }}
+  command:
+  - /bin/bash
+  - -c
+  - |
+    {{- include "odf.labelStorageNodes.script" . | nindent 4 }}
+  name: label-storage-nodes
+dnsPolicy: ClusterFirst
+restartPolicy: Never
+serviceAccountName: {{ .Values.serviceAccountName }}
+terminationGracePeriodSeconds: 400
+{{- end }}

templates/cronjob-labelNodes.yaml

@@ -0,0 +1,17 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+  name: cronjob-label-storage-nodes
+  namespace: {{ .Values.odf.namespace }}
+spec:
+  schedule: {{ .Values.job.schedule | quote }}
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: {{ .Values.job.successfulJobsHistoryLimit }}
+  failedJobsHistoryLimit: {{ .Values.job.failedJobsHistoryLimit }}
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          {{- include "odf.labelStorageNodes.podSpec" . | nindent 10 }}

templates/job-labelNodes.yaml

@@ -8,27 +8,4 @@ metadata:
 spec:
   template:
     spec:
-      containers:
-      - image: {{ .Values.job.image }}
-        command:
-        - /bin/bash
-        - -c
-        - |
-          {{- if .Values.storageSystem.inventory.useSpecificNodes }}
-          {{- range .Values.storageSystem.inventory.nodes }}
-          oc label node {{ . }} cluster.ocs.openshift.io/openshift-storage='' --overwrite
-          {{- end }}
-          {{- else }}
-          oc label nodes -l {{ $.Values.storageSystem.inventory.nodeJobLabelSelector | squote }} cluster.ocs.openshift.io/openshift-storage='' --overwrite
-          LABELED_COUNT=$(oc get nodes -l cluster.ocs.openshift.io/openshift-storage --no-headers 2>/dev/null | wc -l)
-          if [ "$LABELED_COUNT" -lt 3 ]; then
-            echo "Error: Only $LABELED_COUNT node(s) were labeled. At least 3 nodes must be labeled."
-            exit 1
-          fi
-          echo "Successfully labeled $LABELED_COUNT node(s)"
-          {{- end }}
-        name: label-storage-nodes
-      dnsPolicy: ClusterFirst
-      restartPolicy: Never
-      serviceAccountName: {{ .Values.serviceAccountName }}
-      terminationGracePeriodSeconds: 400
+      {{- include "odf.labelStorageNodes.podSpec" . | nindent 6 }}

templates/odf-storagecluster.yaml

@@ -43,7 +43,7 @@ spec:
     dataPVCTemplate:
       metadata: {}
       spec:
-        storageClassName: {{ .Values.global.datacenter.storageClassName }}
+        storageClassName: {{ include "odf.osd.storageClassName" . }}
         accessModes:
         - ReadWriteOnce
         volumeMode: Block

tests/cronjob_labelNodes_test.yaml

@@ -0,0 +1,59 @@
+suite: Test label-storage-nodes CronJob
+templates:
+  - templates/cronjob-labelNodes.yaml
+release:
+  name: release-test
+tests:
+  - it: Should render CronJob with default schedule
+    asserts:
+      - isKind:
+          of: CronJob
+      - equal:
+          path: metadata.name
+          value: cronjob-label-storage-nodes
+      - equal:
+          path: spec.schedule
+          value: "*/15 * * * *"
+      - equal:
+          path: spec.concurrencyPolicy
+          value: Forbid
+      - equal:
+          path: spec.jobTemplate.spec.template.spec.serviceAccountName
+          value: odf-node-label-sa
+
+  - it: Should honor custom schedule
+    set:
+      job:
+        schedule: "0 * * * *"
+    asserts:
+      - equal:
+          path: spec.schedule
+          value: "0 * * * *"
+
+  - it: Should label specific nodes when useSpecificNodes is true
+    set:
+      storageSystem:
+        inventory:
+          useSpecificNodes: true
+          nodes:
+            - nodeA
+            - nodeB
+            - nodeC
+    asserts:
+      - matchRegex:
+          path: spec.jobTemplate.spec.template.spec.containers[0].command[2]
+          pattern: oc label node nodeA
+
+  - it: Should use label selector when useSpecificNodes is false
+    set:
+      storageSystem:
+        inventory:
+          useSpecificNodes: false
+          nodeJobLabelSelector: "node-role.kubernetes.io/worker="
+    asserts:
+      - matchRegex:
+          path: spec.jobTemplate.spec.template.spec.containers[0].command[2]
+          pattern: oc label nodes -l 'node-role.kubernetes.io/worker='
+      - matchRegex:
+          path: spec.jobTemplate.spec.template.spec.containers[0].command[2]
+          pattern: LABELED_COUNT=

tests/odf_storagecluster_storageclass_test.yaml

@@ -0,0 +1,59 @@
+suite: Test ODF StorageCluster OSD storage class
+templates:
+  - templates/odf-storagecluster.yaml
+release:
+  name: release-test
+tests:
+  - it: Should default to gp3-csi when clusterPlatform is unset
+    asserts:
+      - equal:
+          path: spec.storageDeviceSets[0].dataPVCTemplate.spec.storageClassName
+          value: gp3-csi
+
+  - it: Should default to gp3-csi on AWS
+    set:
+      global:
+        clusterPlatform: AWS
+    asserts:
+      - equal:
+          path: spec.storageDeviceSets[0].dataPVCTemplate.spec.storageClassName
+          value: gp3-csi
+
+  - it: Should default to managed-csi on Azure
+    set:
+      global:
+        clusterPlatform: Azure
+    asserts:
+      - equal:
+          path: spec.storageDeviceSets[0].dataPVCTemplate.spec.storageClassName
+          value: managed-csi
+
+  - it: Should default to standard-csi on GCP
+    set:
+      global:
+        clusterPlatform: GCP
+    asserts:
+      - equal:
+          path: spec.storageDeviceSets[0].dataPVCTemplate.spec.storageClassName
+          value: standard-csi
+
+  - it: Should allow overriding the platform default
+    set:
+      global:
+        clusterPlatform: AWS
+      odf:
+        osd:
+          pvc:
+            storageClassName: custom-csi
+    asserts:
+      - equal:
+          path: spec.storageDeviceSets[0].dataPVCTemplate.spec.storageClassName
+          value: custom-csi
+
+  - it: Should fail when platform is unsupported and no override is set
+    set:
+      global:
+        clusterPlatform: BareMetal
+    asserts:
+      - failedTemplate:
+          errorMessage: 'Set odf.osd.pvc.storageClassName or use a supported global.clusterPlatform (AWS, Azure, GCP); got "BareMetal"'

values.yaml

@@ -1,6 +1,6 @@
 global:
-  datacenter:
-    storageClassName: gp3-csi
+  # -- OpenShift cluster platform (AWS, Azure, GCP). Used to select the default OSD storage class when odf.osd.pvc.storageClassName is empty.
+  clusterPlatform: ""
 
 odf:
   namespace: openshift-storage
@@ -33,6 +33,8 @@ odf:
       memory: 5Gi
     pvc:
       storage: 2Ti
+      # -- Storage class for ODF OSD volumes. Empty selects a platform default from global.clusterPlatform (AWS: gp3-csi, Azure: managed-csi, GCP: standard-csi).
+      storageClassName: ""
   storageClass:
     name: ocs-storagecluster-ceph-rgw
     # name: openshift-storage.noobaa.io
@@ -119,3 +121,9 @@ rbac:
 
 job:
   image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest
+  # -- Cron schedule for re-labeling storage nodes after the initial Job (UTC).
+  schedule: "*/15 * * * *"
+  # -- successfulJobsHistoryLimit for the label-storage-nodes CronJob.
+  successfulJobsHistoryLimit: 3
+  # -- failedJobsHistoryLimit for the label-storage-nodes CronJob.
+  failedJobsHistoryLimit: 1