diff --git a/Chart.yaml b/Chart.yaml index 7a5c239..6519a18 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: edge-gitops-vms description: Edge GitOps VMs type: application -version: 0.5.2 +version: 0.5.3 dependencies: [ ] diff --git a/README.md b/README.md index 241c0fb..cbb3731 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # edge-gitops-vms -![Version: 0.5.2](https://img.shields.io/badge/Version-0.5.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.5.3](https://img.shields.io/badge/Version-0.5.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) Edge GitOps VMs @@ -8,6 +8,7 @@ This chart is used to set up Edge GitOps VMs in conjunction with OpenShift Virtu ### Notable changes +* v0.5.3: Extend set-default-virt-storageclass job to wait until VM DataSources are Ready and their CSI driver/provisioner matches vmDefaults.storageClassName (prevents CDI host-assisted clone fallback onto the wrong StorageClass) * v0.5.2: Add optional additionalPvcDisks with default PVC creation; set create: false to attach existing PVCs * v0.5.1: Add optional job to set the default KubeVirt storage class from vmDefaults.storageClassName * v0.5.0: Change default VM type to rhel9 and workload type to server; change other defaults to "gitops-vms" from "edge-gitops-vms" diff --git a/README.md.gotmpl b/README.md.gotmpl index 62f4ec2..0c42592 100644 --- a/README.md.gotmpl +++ b/README.md.gotmpl @@ -9,6 +9,7 @@ This chart is used to set up Edge GitOps VMs in conjunction with OpenShift Virtu ### Notable changes +* v0.5.3: Extend set-default-virt-storageclass job to wait until VM DataSources are Ready and their CSI driver/provisioner matches vmDefaults.storageClassName (prevents CDI host-assisted clone fallback onto the wrong StorageClass) * v0.5.2: Add optional additionalPvcDisks with default PVC creation; set create: false to attach existing PVCs * v0.5.1: Add optional job to set the default KubeVirt storage class from vmDefaults.storageClassName * v0.5.0: Change default VM type to rhel9 and workload type to server; change other defaults to "gitops-vms" from "edge-gitops-vms" diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl new file mode 100644 index 0000000..9bc457c --- /dev/null +++ b/templates/_helpers.tpl @@ -0,0 +1,21 @@ +{{/* +Unique DataSource names referenced by VMs (root disks and additional disks that clone). +*/}} +{{- define "edge-gitops-vms.dataSourceNames" -}} +{{- $datasources := dict -}} +{{- $def := .Values.vmDefaults -}} +{{- range $_, $vmr := .Values.vms -}} + {{- if $vmr -}} + {{- $ds := coalesce $vmr.dataVolume $def.dataVolume $vmr.os $def.os -}} + {{- if $ds -}} + {{- $_ := set $datasources $ds "true" -}} + {{- end -}} + {{- range $disk := (coalesce $vmr.additionalDisks $def.additionalDisks (list)) -}} + {{- if and $disk $disk.dataVolume -}} + {{- $_ := set $datasources $disk.dataVolume "true" -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- keys $datasources | sortAlpha | join " " -}} +{{- end -}} diff --git a/templates/job-setDefaultVirtStorageClass.yaml b/templates/job-setDefaultVirtStorageClass.yaml index bcf2047..a178367 100644 --- a/templates/job-setDefaultVirtStorageClass.yaml +++ b/templates/job-setDefaultVirtStorageClass.yaml @@ -1,4 +1,5 @@ {{- if .Values.setDefaultVirtStorageClass }} +{{- $dataSources := include "edge-gitops-vms.dataSourceNames" . | trim }} apiVersion: batch/v1 kind: Job metadata: @@ -17,10 +18,15 @@ spec: - | set -euo pipefail TARGET_SC="{{ .Values.vmDefaults.storageClassName }}" + DS_NAMESPACE="{{ .Values.vmDefaults.externalDataSourceNamespace }}" + # Space-separated DataSource names derived from chart VMs (may be empty) + DATASOURCES="{{ $dataSources }}" + SLEEP_SECS=15 check_other_default_virt() { local sc annotation while IFS= read -r line; do + [[ -z "${line}" ]] && continue sc="${line%%:*}" annotation="${line#*:}" if [ "$annotation" = "true" ] && [ "$sc" != "$TARGET_SC" ]; then @@ -31,26 +37,146 @@ spec: return 0 } + ensure_default_virt_sc() { + if ! check_other_default_virt; then + return 1 + fi + local current + current=$(oc get storageclass "${TARGET_SC}" -o jsonpath='{.metadata.annotations.storageclass\.kubevirt\.io/is-default-virt-class}') + if [ "${current}" = "true" ]; then + echo "StorageClass ${TARGET_SC} is already the default virt storage class" + return 0 + fi + oc patch storageclass "${TARGET_SC}" -p '{"metadata":{"annotations":{"storageclass.kubevirt.io/is-default-virt-class":"true"}}}' + echo "Set ${TARGET_SC} as default virt storage class" + return 0 + } + + # Returns 0 when DataSource is Ready and its source CSI driver/provisioner + # matches TARGET_SC.provisioner (avoids CDI host-assisted clone fallback). + datasource_clone_ready() { + local ds=$1 + local ready snap pvc pvc_ns deleting vsc content driver pvc_sc pvc_prov + + if ! oc get datasource "${ds}" -n "${DS_NAMESPACE}" >/dev/null 2>&1; then + echo "DataSource ${DS_NAMESPACE}/${ds} not found yet" + return 1 + fi + + ready=$(oc get datasource "${ds}" -n "${DS_NAMESPACE}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) + if [ "${ready}" != "True" ]; then + echo "DataSource ${DS_NAMESPACE}/${ds} Ready=${ready:-Unknown}" + return 1 + fi + + snap=$(oc get datasource "${ds}" -n "${DS_NAMESPACE}" -o jsonpath='{.spec.source.snapshot.name}' 2>/dev/null || true) + pvc=$(oc get datasource "${ds}" -n "${DS_NAMESPACE}" -o jsonpath='{.spec.source.pvc.name}' 2>/dev/null || true) + pvc_ns=$(oc get datasource "${ds}" -n "${DS_NAMESPACE}" -o jsonpath='{.spec.source.pvc.namespace}' 2>/dev/null || true) + + if [ -n "${snap}" ]; then + if ! oc get volumesnapshot "${snap}" -n "${DS_NAMESPACE}" >/dev/null 2>&1; then + echo "VolumeSnapshot ${DS_NAMESPACE}/${snap} for DataSource ${ds} not found" + return 1 + fi + deleting=$(oc get volumesnapshot "${snap}" -n "${DS_NAMESPACE}" -o jsonpath='{.metadata.deletionTimestamp}' 2>/dev/null || true) + if [ -n "${deleting}" ]; then + echo "VolumeSnapshot ${DS_NAMESPACE}/${snap} is being deleted" + return 1 + fi + vsc=$(oc get volumesnapshot "${snap}" -n "${DS_NAMESPACE}" -o jsonpath='{.spec.volumeSnapshotClassName}' 2>/dev/null || true) + if [ -n "${vsc}" ]; then + driver=$(oc get volumesnapshotclass "${vsc}" -o jsonpath='{.driver}' 2>/dev/null || true) + else + content=$(oc get volumesnapshot "${snap}" -n "${DS_NAMESPACE}" -o jsonpath='{.status.boundVolumeSnapshotContentName}' 2>/dev/null || true) + if [ -z "${content}" ]; then + echo "VolumeSnapshot ${snap} has no VolumeSnapshotClass or bound content yet" + return 1 + fi + driver=$(oc get volumesnapshotcontent "${content}" -o jsonpath='{.spec.driver}' 2>/dev/null || true) + fi + if [ -z "${driver}" ]; then + echo "Could not resolve CSI driver for VolumeSnapshot ${snap}" + return 1 + fi + if [ "${driver}" != "${TARGET_PROVISIONER}" ]; then + echo "DataSource ${ds}: snapshot driver '${driver}' != StorageClass ${TARGET_SC} provisioner '${TARGET_PROVISIONER}'" + return 1 + fi + echo "DataSource ${ds}: Ready, snapshot driver matches ${TARGET_PROVISIONER}" + return 0 + fi + + if [ -n "${pvc}" ]; then + pvc_ns="${pvc_ns:-${DS_NAMESPACE}}" + if ! oc get pvc "${pvc}" -n "${pvc_ns}" >/dev/null 2>&1; then + echo "PVC ${pvc_ns}/${pvc} for DataSource ${ds} not found" + return 1 + fi + pvc_sc=$(oc get pvc "${pvc}" -n "${pvc_ns}" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true) + if [ -z "${pvc_sc}" ]; then + echo "PVC ${pvc_ns}/${pvc} has no storageClassName" + return 1 + fi + pvc_prov=$(oc get storageclass "${pvc_sc}" -o jsonpath='{.provisioner}' 2>/dev/null || true) + if [ -z "${pvc_prov}" ]; then + echo "Could not resolve provisioner for StorageClass ${pvc_sc}" + return 1 + fi + if [ "${pvc_prov}" != "${TARGET_PROVISIONER}" ]; then + echo "DataSource ${ds}: PVC provisioner '${pvc_prov}' != StorageClass ${TARGET_SC} provisioner '${TARGET_PROVISIONER}'" + return 1 + fi + echo "DataSource ${ds}: Ready, PVC provisioner matches ${TARGET_PROVISIONER}" + return 0 + fi + + echo "DataSource ${ds} has neither snapshot nor pvc source" + return 1 + } + + all_datasources_clone_ready() { + local ds + if [ -z "${DATASOURCES}" ]; then + echo "No VM DataSources to wait for" + return 0 + fi + for ds in ${DATASOURCES}; do + if ! datasource_clone_ready "${ds}"; then + return 1 + fi + done + return 0 + } + + echo "Target virt StorageClass: ${TARGET_SC}" + echo "DataSource namespace: ${DS_NAMESPACE}" + echo "Required DataSources: ${DATASOURCES:-}" + while true; do if ! oc get storageclass "${TARGET_SC}" >/dev/null 2>&1; then echo "StorageClass ${TARGET_SC} not found, waiting..." - sleep 15 + sleep "${SLEEP_SECS}" continue fi - if ! check_other_default_virt; then + TARGET_PROVISIONER=$(oc get storageclass "${TARGET_SC}" -o jsonpath='{.provisioner}') + if [ -z "${TARGET_PROVISIONER}" ]; then + echo "StorageClass ${TARGET_SC} has empty provisioner, waiting..." + sleep "${SLEEP_SECS}" + continue + fi + + if ! ensure_default_virt_sc; then exit 1 fi - current=$(oc get storageclass "${TARGET_SC}" -o jsonpath='{.metadata.annotations.storageclass\.kubevirt\.io/is-default-virt-class}') - if [ "${current}" = "true" ]; then - echo "StorageClass ${TARGET_SC} is already the default virt storage class" + if all_datasources_clone_ready; then + echo "Default virt StorageClass set and required DataSources are clone-ready" exit 0 fi - oc patch storageclass "${TARGET_SC}" -p '{"metadata":{"annotations":{"storageclass.kubevirt.io/is-default-virt-class":"true"}}}' - echo "Set ${TARGET_SC} as default virt storage class" - exit 0 + echo "Waiting for DataSources to become clone-ready..." + sleep "${SLEEP_SECS}" done name: set-default-virt-storageclass dnsPolicy: ClusterFirst diff --git a/templates/rbac/set-default-virt-storageclass.yaml b/templates/rbac/set-default-virt-storageclass.yaml index b7bf61e..239d077 100644 --- a/templates/rbac/set-default-virt-storageclass.yaml +++ b/templates/rbac/set-default-virt-storageclass.yaml @@ -17,6 +17,32 @@ rules: - watch - patch - update + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + - volumesnapshotcontents + - volumesnapshots + verbs: + - get + - list + - watch + - apiGroups: + - cdi.kubevirt.io + resources: + - datasources + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/tests/set-default-virt-storageclass_test.yaml b/tests/set-default-virt-storageclass_test.yaml index 94e6d2d..3b0d408 100644 --- a/tests/set-default-virt-storageclass_test.yaml +++ b/tests/set-default-virt-storageclass_test.yaml @@ -18,6 +18,7 @@ tests: setDefaultVirtStorageClass: true vmDefaults: storageClassName: ocs-storagecluster-ceph-rbd-virtualization + externalDataSourceNamespace: openshift-virtualization-os-images asserts: - hasDocuments: count: 1 @@ -32,6 +33,40 @@ tests: - matchRegex: path: spec.template.spec.containers[0].command[2] pattern: ocs-storagecluster-ceph-rbd-virtualization + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: datasource_clone_ready + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: openshift-virtualization-os-images + + - it: embeds DataSource names from VMs + set: + setDefaultVirtStorageClass: true + vmDefaults: + storageClassName: my-virt-sc + externalDataSourceNamespace: openshift-virtualization-os-images + os: rhel9 + vms: + kiosk: + role: kiosk + os: rhel8 + count: 1 + server: + role: server + os: rhel10 + count: 1 + additionalDisks: + - name: data + storage: 10Gi + dataVolume: fedora + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: DATASOURCES="fedora rhel10 rhel8" + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: TARGET_SC="my-virt-sc" --- suite: set-default-virt-storageclass-rbac @@ -61,6 +96,18 @@ tests: path: rules[0].resources content: storageclasses documentIndex: 0 + - contains: + path: rules[1].resources + content: volumesnapshots + documentIndex: 0 + - contains: + path: rules[2].resources + content: datasources + documentIndex: 0 + - contains: + path: rules[3].resources + content: persistentvolumeclaims + documentIndex: 0 - isKind: of: ClusterRoleBinding documentIndex: 1