diff --git a/Chart.yaml b/Chart.yaml index 374694b..7b2574f 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.3 +version: 0.3.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/README.md b/README.md index cd430e9..9161a04 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,18 @@ # openshift-data-foundations -![Version: 0.2.3](https://img.shields.io/badge/Version-0.2.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) +![Version: 0.3.0](https://img.shields.io/badge/Version-0.3.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) A Helm chart to install ODF on Openshift ### Notable changes +* v0.3.0: Replace `global.datacenter.storageClassName` with `odf.osd.pvc.storageClassName`. + When unset, the OSD storage class is derived from `global.clusterPlatform` + (AWS: `gp3-csi`, Azure: `managed-csi`, GCP: `standard-csi`). + This is a backwards-incompatible change. + Also add a CronJob that periodically re-runs the same node-labeling + logic as the bootstrap Job (same useSpecificNodes / selector conditions). + * v0.2.3: Allow passing of label selector to labelling job to avoid, for example, labelling submariner nodes. Also ensure at least 3 nodes are labelled when using label selector. @@ -23,15 +30,18 @@ A Helm chart to install ODF on Openshift ## Notes -This branch currently tracks the v0.2.x releases which use the host as a +This branch currently tracks the v0.3.x releases which use the host as a default failure domain for objectStorage. ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| -| global.datacenter.storageClassName | string | `"gp3-csi"` | | +| global.clusterPlatform | string | `""` | OpenShift cluster platform (AWS, Azure, GCP). Used to select the default OSD storage class when odf.osd.pvc.storageClassName is empty. | +| job.failedJobsHistoryLimit | int | `1` | failedJobsHistoryLimit for the label-storage-nodes CronJob. | | job.image | string | `"image-registry.openshift-image-registry.svc:5000/openshift/cli:latest"` | | +| job.schedule | string | `"*/15 * * * *"` | Cron schedule for re-labeling storage nodes after the initial Job (UTC). | +| job.successfulJobsHistoryLimit | int | `3` | successfulJobsHistoryLimit for the label-storage-nodes CronJob. | | objectStorage.dataPool.failureDomain | string | `"host"` | Failuredomain for the dataPool | | objectStorage.dataPool.replicas | int | `3` | | | objectStorage.enable | bool | `true` | | @@ -55,6 +65,7 @@ default failure domain for objectStorage. | odf.noobaadb.requests.cpu | int | `1` | | | odf.noobaadb.requests.memory | string | `"4Gi"` | | | odf.osd.pvc.storage | string | `"2Ti"` | | +| odf.osd.pvc.storageClassName | string | `""` | Storage class for ODF OSD volumes. Empty selects a platform default from global.clusterPlatform (AWS: gp3-csi, Azure: managed-csi, GCP: standard-csi). | | odf.osd.requests.cpu | int | `2` | | | odf.osd.requests.memory | string | `"5Gi"` | | | odf.serviceUrl | string | `"http://rook-ceph-rgw-ocs-storagecluster-cephobjectstore.openshift-storage.svc.cluster.local"` | | diff --git a/README.md.gotmpl b/README.md.gotmpl index df1fbf7..59eb063 100644 --- a/README.md.gotmpl +++ b/README.md.gotmpl @@ -7,6 +7,14 @@ ### Notable changes +* v0.3.0: Replace `global.datacenter.storageClassName` with `odf.osd.pvc.storageClassName`. + When unset, the OSD storage class is derived from `global.clusterPlatform` + (AWS: `gp3-csi`, Azure: `managed-csi`, GCP: `standard-csi`). + This is a backwards-incompatible change. + Also add a CronJob that periodically re-runs the same node-labeling + logic as the bootstrap Job (same useSpecificNodes / selector conditions). + + * v0.2.3: Allow passing of label selector to labelling job to avoid, for example, labelling submariner nodes. Also ensure at least 3 nodes are labelled when using label selector. @@ -20,7 +28,7 @@ ## Notes -This branch currently tracks the v0.2.x releases which use the host as a +This branch currently tracks the v0.3.x releases which use the host as a default failure domain for objectStorage. {{ template "chart.sourcesSection" . }} diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl new file mode 100644 index 0000000..ee35561 --- /dev/null +++ b/templates/_helpers.tpl @@ -0,0 +1,64 @@ +{{/* +Return the storage class used for ODF OSD volume PVCs. + +Uses odf.osd.pvc.storageClassName when set; otherwise selects a platform +default from global.clusterPlatform: + + AWS (or unset) -> gp3-csi + Azure -> managed-csi + GCP -> standard-csi +*/}} +{{- define "odf.osd.storageClassName" -}} +{{- if .Values.odf.osd.pvc.storageClassName -}} +{{- .Values.odf.osd.pvc.storageClassName -}} +{{- else -}} +{{- $platform := .Values.global.clusterPlatform | default "" | lower -}} +{{- if or (eq $platform "") (eq $platform "aws") -}} +gp3-csi +{{- else if eq $platform "azure" -}} +managed-csi +{{- else if eq $platform "gcp" -}} +standard-csi +{{- else -}} +{{- fail (printf "Set odf.osd.pvc.storageClassName or use a supported global.clusterPlatform (AWS, Azure, GCP); got %q" .Values.global.clusterPlatform) -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Bash script used by the label-storage-nodes Job and CronJob. +Respects storageSystem.inventory.useSpecificNodes vs nodeJobLabelSelector. +*/}} +{{- define "odf.labelStorageNodes.script" -}} +{{- if .Values.storageSystem.inventory.useSpecificNodes }} +{{- range .Values.storageSystem.inventory.nodes }} +oc label node {{ . }} cluster.ocs.openshift.io/openshift-storage='' --overwrite +{{- end }} +{{- else }} +oc label nodes -l {{ .Values.storageSystem.inventory.nodeJobLabelSelector | squote }} cluster.ocs.openshift.io/openshift-storage='' --overwrite +LABELED_COUNT=$(oc get nodes -l cluster.ocs.openshift.io/openshift-storage --no-headers 2>/dev/null | wc -l) +if [ "$LABELED_COUNT" -lt 3 ]; then + echo "Error: Only $LABELED_COUNT node(s) were labeled. At least 3 nodes must be labeled." + exit 1 +fi +echo "Successfully labeled $LABELED_COUNT node(s)" +{{- end }} +{{- end }} + +{{/* +Pod spec shared by the label-storage-nodes Job and CronJob. +*/}} +{{- define "odf.labelStorageNodes.podSpec" -}} +containers: +- image: {{ .Values.job.image }} + command: + - /bin/bash + - -c + - | + {{- include "odf.labelStorageNodes.script" . | nindent 4 }} + name: label-storage-nodes +dnsPolicy: ClusterFirst +restartPolicy: Never +serviceAccountName: {{ .Values.serviceAccountName }} +terminationGracePeriodSeconds: 400 +{{- end }} diff --git a/templates/cronjob-labelNodes.yaml b/templates/cronjob-labelNodes.yaml new file mode 100644 index 0000000..af7d270 --- /dev/null +++ b/templates/cronjob-labelNodes.yaml @@ -0,0 +1,17 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + annotations: + argocd.argoproj.io/sync-wave: "0" + name: cronjob-label-storage-nodes + namespace: {{ .Values.odf.namespace }} +spec: + schedule: {{ .Values.job.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: {{ .Values.job.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ .Values.job.failedJobsHistoryLimit }} + jobTemplate: + spec: + template: + spec: + {{- include "odf.labelStorageNodes.podSpec" . | nindent 10 }} diff --git a/templates/job-labelNodes.yaml b/templates/job-labelNodes.yaml index b3d5781..b90d409 100644 --- a/templates/job-labelNodes.yaml +++ b/templates/job-labelNodes.yaml @@ -8,27 +8,4 @@ metadata: spec: template: spec: - containers: - - image: {{ .Values.job.image }} - command: - - /bin/bash - - -c - - | - {{- if .Values.storageSystem.inventory.useSpecificNodes }} - {{- range .Values.storageSystem.inventory.nodes }} - oc label node {{ . }} cluster.ocs.openshift.io/openshift-storage='' --overwrite - {{- end }} - {{- else }} - oc label nodes -l {{ $.Values.storageSystem.inventory.nodeJobLabelSelector | squote }} cluster.ocs.openshift.io/openshift-storage='' --overwrite - LABELED_COUNT=$(oc get nodes -l cluster.ocs.openshift.io/openshift-storage --no-headers 2>/dev/null | wc -l) - if [ "$LABELED_COUNT" -lt 3 ]; then - echo "Error: Only $LABELED_COUNT node(s) were labeled. At least 3 nodes must be labeled." - exit 1 - fi - echo "Successfully labeled $LABELED_COUNT node(s)" - {{- end }} - name: label-storage-nodes - dnsPolicy: ClusterFirst - restartPolicy: Never - serviceAccountName: {{ .Values.serviceAccountName }} - terminationGracePeriodSeconds: 400 + {{- include "odf.labelStorageNodes.podSpec" . | nindent 6 }} diff --git a/templates/odf-storagecluster.yaml b/templates/odf-storagecluster.yaml index 5aec5cb..358e34e 100644 --- a/templates/odf-storagecluster.yaml +++ b/templates/odf-storagecluster.yaml @@ -43,7 +43,7 @@ spec: dataPVCTemplate: metadata: {} spec: - storageClassName: {{ .Values.global.datacenter.storageClassName }} + storageClassName: {{ include "odf.osd.storageClassName" . }} accessModes: - ReadWriteOnce volumeMode: Block diff --git a/tests/cronjob_labelNodes_test.yaml b/tests/cronjob_labelNodes_test.yaml new file mode 100644 index 0000000..36ea049 --- /dev/null +++ b/tests/cronjob_labelNodes_test.yaml @@ -0,0 +1,59 @@ +suite: Test label-storage-nodes CronJob +templates: + - templates/cronjob-labelNodes.yaml +release: + name: release-test +tests: + - it: Should render CronJob with default schedule + asserts: + - isKind: + of: CronJob + - equal: + path: metadata.name + value: cronjob-label-storage-nodes + - equal: + path: spec.schedule + value: "*/15 * * * *" + - equal: + path: spec.concurrencyPolicy + value: Forbid + - equal: + path: spec.jobTemplate.spec.template.spec.serviceAccountName + value: odf-node-label-sa + + - it: Should honor custom schedule + set: + job: + schedule: "0 * * * *" + asserts: + - equal: + path: spec.schedule + value: "0 * * * *" + + - it: Should label specific nodes when useSpecificNodes is true + set: + storageSystem: + inventory: + useSpecificNodes: true + nodes: + - nodeA + - nodeB + - nodeC + asserts: + - matchRegex: + path: spec.jobTemplate.spec.template.spec.containers[0].command[2] + pattern: oc label node nodeA + + - it: Should use label selector when useSpecificNodes is false + set: + storageSystem: + inventory: + useSpecificNodes: false + nodeJobLabelSelector: "node-role.kubernetes.io/worker=" + asserts: + - matchRegex: + path: spec.jobTemplate.spec.template.spec.containers[0].command[2] + pattern: oc label nodes -l 'node-role.kubernetes.io/worker=' + - matchRegex: + path: spec.jobTemplate.spec.template.spec.containers[0].command[2] + pattern: LABELED_COUNT= diff --git a/tests/odf_storagecluster_storageclass_test.yaml b/tests/odf_storagecluster_storageclass_test.yaml new file mode 100644 index 0000000..5fb79d9 --- /dev/null +++ b/tests/odf_storagecluster_storageclass_test.yaml @@ -0,0 +1,59 @@ +suite: Test ODF StorageCluster OSD storage class +templates: + - templates/odf-storagecluster.yaml +release: + name: release-test +tests: + - it: Should default to gp3-csi when clusterPlatform is unset + asserts: + - equal: + path: spec.storageDeviceSets[0].dataPVCTemplate.spec.storageClassName + value: gp3-csi + + - it: Should default to gp3-csi on AWS + set: + global: + clusterPlatform: AWS + asserts: + - equal: + path: spec.storageDeviceSets[0].dataPVCTemplate.spec.storageClassName + value: gp3-csi + + - it: Should default to managed-csi on Azure + set: + global: + clusterPlatform: Azure + asserts: + - equal: + path: spec.storageDeviceSets[0].dataPVCTemplate.spec.storageClassName + value: managed-csi + + - it: Should default to standard-csi on GCP + set: + global: + clusterPlatform: GCP + asserts: + - equal: + path: spec.storageDeviceSets[0].dataPVCTemplate.spec.storageClassName + value: standard-csi + + - it: Should allow overriding the platform default + set: + global: + clusterPlatform: AWS + odf: + osd: + pvc: + storageClassName: custom-csi + asserts: + - equal: + path: spec.storageDeviceSets[0].dataPVCTemplate.spec.storageClassName + value: custom-csi + + - it: Should fail when platform is unsupported and no override is set + set: + global: + clusterPlatform: BareMetal + asserts: + - failedTemplate: + errorMessage: 'Set odf.osd.pvc.storageClassName or use a supported global.clusterPlatform (AWS, Azure, GCP); got "BareMetal"' diff --git a/values.yaml b/values.yaml index 648dcf1..7970ffe 100644 --- a/values.yaml +++ b/values.yaml @@ -1,6 +1,6 @@ global: - datacenter: - storageClassName: gp3-csi + # -- OpenShift cluster platform (AWS, Azure, GCP). Used to select the default OSD storage class when odf.osd.pvc.storageClassName is empty. + clusterPlatform: "" odf: namespace: openshift-storage @@ -33,6 +33,8 @@ odf: memory: 5Gi pvc: storage: 2Ti + # -- Storage class for ODF OSD volumes. Empty selects a platform default from global.clusterPlatform (AWS: gp3-csi, Azure: managed-csi, GCP: standard-csi). + storageClassName: "" storageClass: name: ocs-storagecluster-ceph-rgw # name: openshift-storage.noobaa.io @@ -119,3 +121,9 @@ rbac: job: image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest + # -- Cron schedule for re-labeling storage nodes after the initial Job (UTC). + schedule: "*/15 * * * *" + # -- successfulJobsHistoryLimit for the label-storage-nodes CronJob. + successfulJobsHistoryLimit: 3 + # -- failedJobsHistoryLimit for the label-storage-nodes CronJob. + failedJobsHistoryLimit: 1