From 8db5b1d08bbfe9cdc90a691c2ef419f17b201992 Mon Sep 17 00:00:00 2001 From: orion Date: Wed, 28 Oct 2026 11:51:22 +0000 Subject: [PATCH 1/8] feat(helm): retain cluster bootstrap topology Separate retained initialization counts from desired node replicas. Import live chart-managed bootstrap inputs and reject conflicting history. Document PD compatibility, controlled membership and startup limits. --- helm/hugegraph/README.md | 6 + helm/hugegraph/docs/operations.md | 84 +++++++ helm/hugegraph/templates/_bootstrap.tpl | 149 ++++++++++++ helm/hugegraph/templates/_helpers.tpl | 20 +- .../templates/bootstrap-configmap.yaml | 34 +++ helm/hugegraph/templates/pd-statefulset.yaml | 2 +- .../templates/store-statefulset.yaml | 2 +- .../bootstrap_consumer_mismatch_test.yaml | 119 ++++++++++ .../tests/bootstrap_corrupt_record_test.yaml | 132 +++++++++++ .../tests/bootstrap_custom_legacy_test.yaml | 116 ++++++++++ .../hugegraph/tests/bootstrap_fresh_test.yaml | 219 ++++++++++++++++++ .../bootstrap_incomplete_legacy_test.yaml | 123 ++++++++++ .../tests/bootstrap_legacy_test.yaml | 170 ++++++++++++++ .../tests/bootstrap_pd_growth_test.yaml | 194 ++++++++++++++++ .../tests/bootstrap_pd_shrink_test.yaml | 187 +++++++++++++++ .../tests/bootstrap_record_test.yaml | 203 ++++++++++++++++ .../tests/bootstrap_record_version_test.yaml | 126 ++++++++++ .../tests/bootstrap_reinstall_test.yaml | 127 ++++++++++ helm/hugegraph/values.schema.json | 45 ++++ helm/hugegraph/values.yaml | 10 + helm/test_helm_bootstrap.py | 105 +++++++++ 21 files changed, 2164 insertions(+), 9 deletions(-) create mode 100644 helm/hugegraph/templates/_bootstrap.tpl create mode 100644 helm/hugegraph/templates/bootstrap-configmap.yaml create mode 100644 helm/hugegraph/tests/bootstrap_consumer_mismatch_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_corrupt_record_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_custom_legacy_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_fresh_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_incomplete_legacy_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_legacy_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_pd_growth_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_record_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_record_version_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_reinstall_test.yaml create mode 100644 helm/test_helm_bootstrap.py diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 37351b141d..c4ebd8af92 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -34,6 +34,12 @@ use `latest` images; pin component `image.digest` values when needed. The source installation and configuration examples below assume a repository checkout and commands run from its root. +The [stable bootstrap configuration](docs/operations.md#stable-bootstrap-topology) +separates initialization inputs from node replicas. It is disabled by default +and requires PD images with separate bootstrap and runtime quorum checks before +using it for node operations. It does not implement safe node admission or +retirement by itself. + ## Install from source and open Hubble ```bash diff --git a/helm/hugegraph/docs/operations.md b/helm/hugegraph/docs/operations.md index 46c8c489bc..aabd39e7cb 100644 --- a/helm/hugegraph/docs/operations.md +++ b/helm/hugegraph/docs/operations.md @@ -226,6 +226,90 @@ Server replicas can be changed routinely; CPU-utilization HPA requires positive CPU requests. Store growth still needs volume and shard planning. Use chart updates for Store scale changes so Hubble's endpoint list is updated too. +## Stable bootstrap topology + +This feature is disabled by default. Enable it with `bootstrap.enabled=true`. It separates +initial PD peers, initial Store allowlist and startup Store threshold from +current `pd.replicas` and `store.replicas`. It does not perform database member +changes, approve Pending Stores or prove that a Store is safe to delete. + +Before using it for node operations, deploy a PD image that treats +`pd.initial-store-count` as an admission threshold while no shard groups exist +and checks existing groups against their known Store voting majorities. Older PD images can still apply the +initial count during normal operation even when a runtime quorum is healthy. +Pin the compatible PD image version or digest; this chart cannot infer that +capability from an image tag. See the [PD configuration guide](../../../hugegraph-pd/docs/configuration.md). + +On a fresh install, empty `bootstrap.pdReplicas`, `bootstrap.storeReplicas` and +`bootstrap.initialStoreCount` inherit the requested PD count, Store count and +Store count, respectively. A fresh five-PD installation therefore starts with +five initial voters, not three. The chart records these counts in a kept, +immutable `-bootstrap` ConfigMap (with a hash for long names). PD startup inputs, client PD +seeds, Store startup seed wait, and an unspecified initial shard count use the +recorded topology. PDB validation still uses the current desired PD majority. +Hubble's Store REST allowlist can change with Store count as before. +PD changes between three and five replicas preserve these Pod startup inputs. +Crossing the one-PD boundary can still change the inherited liveness path; +use a compatible PD image and explicitly pin `pd.livenessPath` before such a +controlled operation when the existing Pod template must remain unchanged. + +For an existing release without this record, the first online upgrade imports +the live PD's `HG_PD_RAFT_PEERS_LIST`, `HG_PD_INITIAL_STORE_LIST` and +`HG_PD_INITIAL_STORE_COUNT`. It does not infer history from live replica counts, +which a controlled operation may already have changed. Import supports the +chart's contiguous ordinal endpoints; customized endpoint lists require a +separate reviewed migration. Import preserves a Store allowlist of four and a +startup threshold of three as distinct inputs. No bootstrap checksum is added +to Pod templates, so enabling the feature does not itself request a rollout. +An incomplete live environment is rejected even when all value pins are given; +pins cannot reconstruct unknown online history. +Existing Store and Server seed inputs must agree with that original PD peer +count; a known inconsistent consumer is rejected instead of silently changing +its Pod template during import. + +Recorded bootstrap counts are initialization inputs, not the current voting +configuration or a list of all admitted Stores. A new Store outside the +recorded allowlist remains Pending until a controlled management action admits +it after identity, disk and capacity checks. Plan migration and verify actual +Raft members before changing Pod counts. Existing live guards still reject PD +growth/shrink and Store shrink until the documented database operation and +resource steps are complete. Matching live replicas alone remains insufficient +proof of completion. + +Keep the feature enabled once a record exists. An online render rejects pins +that conflict with the record, or disabling it while the record remains. +Bootstrap records are kept on uninstall like data PVCs. An install that finds +an existing record or live PD is rejected rather than misrepresenting recovery +as a fresh cluster. Recovery with kept data requires a separate review of the +actual membership, original bootstrap inputs and volume identities; this +feature does not add an automatic restore mode. +A controlled rollback must retain the same bootstrap record and use a chart +version that understands it. Rolling back to a chart or release configuration +that predates stable bootstrap can rederive Pod startup inputs from replica +counts; it is not a supported automatic rollback procedure. + +Offline upgrades cannot discover this history. Render them with `--is-upgrade` +and pin all three counts from the retained record, for example: + +```bash +helm template hugegraph ./helm/hugegraph -n hugegraph --is-upgrade \ + --set bootstrap.enabled=true \ + --set bootstrap.pdReplicas=3 --set bootstrap.storeReplicas=3 \ + --set bootstrap.initialStoreCount=3 +``` + +An offline upgrade without all three pins fails. A renderer that always uses +install semantics cannot safely infer an existing topology during replica +changes; unattended GitOps topology reconciliation is not supported by this feature. +Suspend competing reconciliation during controlled resource steps. Static +bootstrap seed majority checks are conservative and are not a replacement for +an authenticated read of current PD membership and quorum health. For example, +after an original five-PD cluster shrinks to three, Store startup still requires +three of the five recorded seeds. If one of the remaining PDs then fails, +current PD quorum may be healthy while a newly created Store remains blocked +by this static startup check. Dynamic membership-aware startup is a separate +follow-up; unchanged Pod templates do not prove every combined failure case. + ## Rolling Store updates and recovery Use `store.updateStrategy.type=OnDelete` for a controlled rollout. Replace one diff --git a/helm/hugegraph/templates/_bootstrap.tpl b/helm/hugegraph/templates/_bootstrap.tpl new file mode 100644 index 0000000000..2cd0f67856 --- /dev/null +++ b/helm/hugegraph/templates/_bootstrap.tpl @@ -0,0 +1,149 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{/* +Retain only initialization counts, not a second membership view. Explicit pins may reproduce history offline but cannot change +an online record. A legacy release is imported from its actual PD environment, +never from StatefulSet replicas that a controlled operation may already change. +*/}} +{{- define "hugegraph.bootstrap.name" -}} +{{- $name := include "hugegraph.fullname" . -}} +{{- if gt (len $name) 53 -}} +{{- printf "%s-%s-bootstrap" ($name | trunc 44 | trimSuffix "-") ($name | sha256sum | trunc 8) -}} +{{- else -}} +{{- printf "%s-bootstrap" $name -}} +{{- end -}} +{{- end }} + +{{- define "hugegraph.bootstrap.resolve" -}} +{{- $config := get .Values "bootstrap" | default dict -}} +{{- $enabled := get $config "enabled" | default false -}} +{{- $record := lookup "v1" "ConfigMap" .Release.Namespace (include "hugegraph.bootstrap.name" .) -}} +{{- $history := dict -}} +{{- $legacyEnv := dict -}} +{{- if not $enabled -}} +{{- if $record -}} +{{- fail "bootstrap.enabled cannot be disabled while the retained bootstrap ConfigMap exists; keep initialization topology pinned" -}} +{{- end -}} +{{- $_ := set .Values "__hugegraphBootstrap" (dict "pdReplicas" (int .Values.pd.replicas) "storeReplicas" (int .Values.store.replicas) "initialStoreCount" (int .Values.store.replicas)) -}} +{{- else -}} +{{- if $record -}} +{{- if not .Release.IsUpgrade -}} +{{- fail "bootstrap history already exists on an install; use a reviewed recovery procedure rather than treating kept data as a fresh cluster" -}} +{{- end -}} +{{- $owner := dig "metadata" "labels" "app.kubernetes.io/instance" "" $record -}} +{{- if and $owner (ne $owner .Release.Name) -}} +{{- fail "bootstrap ConfigMap belongs to another release; inspect the retained initialization topology" -}} +{{- end -}} +{{- $data := get $record "data" | default dict -}} +{{- if ne (get $data "schemaVersion" | default "") "1" -}} +{{- fail "bootstrap ConfigMap has an unsupported schemaVersion; inspect it before continuing" -}} +{{- end -}} +{{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}} +{{- $_ := set $history $key (get $data $key | default "") -}} +{{- end -}} +{{- else -}} +{{- $live := lookup "apps/v1" "StatefulSet" .Release.Namespace (include "hugegraph.pd.name" .) -}} +{{- if $live -}} +{{- if not .Release.IsUpgrade -}} +{{- fail "bootstrap import requires an upgrade of the existing release; inspect live PD and retained data before recovery" -}} +{{- end -}} +{{- $env := dict -}} +{{- range $container := dig "spec" "template" "spec" "containers" list $live -}} +{{- if eq (get $container "name" | default "") "pd" -}} +{{- range $item := get $container "env" | default list -}} +{{- $_ := set $env (get $item "name") (get $item "value" | default "") -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- $pdPeers := get $env "HG_PD_RAFT_PEERS_LIST" | default "" -}} +{{- $stores := get $env "HG_PD_INITIAL_STORE_LIST" | default "" -}} +{{- $count := get $env "HG_PD_INITIAL_STORE_COUNT" | default "" -}} +{{- if or (not $pdPeers) (not $stores) (not $count) -}} +{{- fail "bootstrap import requires the existing PD peer list, initial Store list and initial Store count; inspect the live environment instead of guessing from replicas" -}} +{{- end -}} +{{- $_ := set $history "pdReplicas" (len (splitList "," $pdPeers) | toString) -}} +{{- $_ := set $history "storeReplicas" (len (splitList "," $stores) | toString) -}} +{{- $_ := set $history "initialStoreCount" $count -}} +{{- $legacyEnv = $env -}} +{{- else if .Release.IsUpgrade -}} +{{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}} +{{- if eq (include "hugegraph.optionalScalar" (get $config $key)) "" -}} +{{- fail "bootstrap.pdReplicas, bootstrap.storeReplicas and bootstrap.initialStoreCount must all be pinned for an upgrade without a live bootstrap record" -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- if not (empty $history) -}} +{{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}} +{{- if not (regexMatch "^[1-9][0-9]*$" (get $history $key | toString)) -}} +{{- fail (printf "bootstrap record is missing a valid %s; inspect it before continuing" $key) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- $resolved := dict -}} +{{- $defaults := dict "pdReplicas" .Values.pd.replicas "storeReplicas" .Values.store.replicas "initialStoreCount" .Values.store.replicas -}} +{{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}} +{{- $pin := include "hugegraph.optionalScalar" (get $config $key) -}} +{{- $old := get $history $key | default "" | toString -}} +{{- if and (ne $old "") (ne $pin "") (ne $pin $old) -}} +{{- fail (printf "bootstrap.%s cannot change from %s to %s on an initialized release" $key $old $pin) -}} +{{- end -}} +{{- $value := ternary $pin $old (ne $pin "") -}} +{{- if eq $value "" -}}{{- $value = get $defaults $key | toString -}}{{- end -}} +{{- if or (not (regexMatch "^[1-9][0-9]*$" $value)) (lt (int $value) 1) -}} +{{- fail (printf "bootstrap.%s must be a positive integer in the retained bootstrap record" $key) -}} +{{- end -}} +{{- $_ := set $resolved $key (int $value) -}} +{{- end -}} +{{- if and (not .Release.IsUpgrade) (empty $history) -}} +{{- if ne (int $resolved.pdReplicas) (int .Values.pd.replicas) -}} +{{- fail "bootstrap.pdReplicas must equal pd.replicas on a fresh install; use --is-upgrade with pinned history for offline upgrades" -}} +{{- end -}} +{{- if gt (int $resolved.initialStoreCount) (int .Values.store.replicas) -}} +{{- fail "bootstrap.initialStoreCount exceeds the Stores started by a fresh install" -}} +{{- end -}} +{{- if gt (int $resolved.initialStoreCount) (int $resolved.storeReplicas) -}} +{{- fail "bootstrap.initialStoreCount cannot exceed bootstrap.storeReplicas on a fresh install: the initial allowlist could not satisfy startup" -}} +{{- end -}} +{{- end -}} +{{- $_ := set .Values "__hugegraphBootstrap" $resolved -}} +{{- if not (empty $legacyEnv) -}} +{{- if or (ne (get $legacyEnv "HG_PD_RAFT_PEERS_LIST") (include "hugegraph.pd.raftPeersList" .)) (ne (get $legacyEnv "HG_PD_INITIAL_STORE_LIST") (include "hugegraph.store.initialStoreList" .)) -}} +{{- fail "bootstrap import only supports the chart's contiguous ordinal endpoint lists; inspect customized live endpoints before migration" -}} +{{- end -}} +{{- $seed := include "hugegraph.pd.grpcPeersList" . -}} +{{- range $component := list "store" "server" -}} +{{- $kind := ternary "StatefulSet" "Deployment" (eq $component "store") -}} +{{- $name := ternary (include "hugegraph.store.name" $) (include "hugegraph.server.name" $) (eq $component "store") -}} +{{- $consumer := lookup "apps/v1" $kind $.Release.Namespace $name -}} +{{- $key := ternary "HG_STORE_PD_ADDRESS" "HG_SERVER_PD_PEERS" (eq $component "store") -}} +{{- if $consumer -}} +{{- range $container := dig "spec" "template" "spec" "containers" list $consumer -}} +{{- if eq (get $container "name" | default "") $component -}} +{{- range $item := get $container "env" | default list -}} +{{- if and (eq (get $item "name" | default "") $key) (ne (get $item "value" | default "") $seed) -}} +{{- fail (printf "bootstrap import found %s seeds that differ from the PD initial peers; reconcile the existing templates before migration" $component) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end }} diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index bd0c8770c8..e7e7ce5696 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -117,6 +117,7 @@ Fail before dereferencing missing blocks; keep values.yaml the sole defaults sou {{- $budget := add (int $server.storageWaitTimeoutSeconds) 150 -}} {{- $_ := set $startup "failureThreshold" (add (div (add (sub $budget 1) $period) $period) 1) -}} {{- end -}} +{{- include "hugegraph.bootstrap.resolve" . -}} {{- end -}} {{- end }} @@ -447,7 +448,7 @@ Uses short headless DNS (cluster.local optional) resolvable inside the namespace */}} {{- define "hugegraph.pd.raftPeersList" -}} {{- $peers := list -}} -{{- $replicas := int .Values.pd.replicas -}} +{{- $replicas := int .Values.__hugegraphBootstrap.pdReplicas -}} {{- $name := include "hugegraph.pd.name" . -}} {{- $ns := .Release.Namespace -}} {{- $port := int .Values.pd.ports.raft -}} @@ -462,7 +463,7 @@ PD gRPC peers for Store/Server. */}} {{- define "hugegraph.pd.grpcPeersList" -}} {{- $peers := list -}} -{{- $replicas := int .Values.pd.replicas -}} +{{- $replicas := int .Values.__hugegraphBootstrap.pdReplicas -}} {{- $name := include "hugegraph.pd.name" . -}} {{- $ns := .Release.Namespace -}} {{- $port := int .Values.pd.ports.grpc -}} @@ -477,7 +478,7 @@ PD REST endpoints for Server storage-readiness checks. */}} {{- define "hugegraph.pd.restPeersList" -}} {{- $peers := list -}} -{{- $replicas := int .Values.pd.replicas -}} +{{- $replicas := int .Values.__hugegraphBootstrap.pdReplicas -}} {{- $name := include "hugegraph.pd.name" . -}} {{- $ns := .Release.Namespace -}} {{- $port := int .Values.pd.ports.rest -}} @@ -531,7 +532,7 @@ Initial store list for PD bootstrap: store-0.svc.ns.svc:8500,... */}} {{- define "hugegraph.store.initialStoreList" -}} {{- $peers := list -}} -{{- $replicas := int .Values.store.replicas -}} +{{- $replicas := int .Values.__hugegraphBootstrap.storeReplicas -}} {{- $name := include "hugegraph.store.name" . -}} {{- $ns := .Release.Namespace -}} {{- $port := int .Values.store.ports.grpc -}} @@ -600,6 +601,11 @@ Quorum size: floor(replicas/2)+1 {{- add (div (int .Values.pd.replicas) 2) 1 -}} {{- end }} +{{/* The static seed wait is independent of the current voting/PDB majority. */}} +{{- define "hugegraph.pd.bootstrapQuorum" -}} +{{- add (div (int .Values.__hugegraphBootstrap.pdReplicas) 2) 1 -}} +{{- end }} + {{/* Render JAVA_OPTS only when explicitly configured. An empty value preserves the image entrypoint's existing automatic JVM sizing behavior. @@ -688,7 +694,7 @@ well, which is what a leader election waited on. {{- $flags := list -}} {{- $shardCount := include "hugegraph.optionalScalar" (get $partition "defaultShardCount") -}} {{- if eq $shardCount "" -}} -{{- $shardCount = ternary "3" "1" (ge (int .Values.store.replicas) 3) -}} +{{- $shardCount = ternary "3" "1" (ge (int .Values.__hugegraphBootstrap.storeReplicas) 3) -}} {{- end -}} {{- $flags = append $flags (printf "-Dpartition.default-shard-count=%s" $shardCount) -}} {{- $maxShard := include "hugegraph.optionalScalar" (get $partition "storeMaxShardCount") -}} @@ -881,8 +887,8 @@ the chart renders reaches it only as NodeOptions.setInitialConf, which jraft applies when bootstrapping a node that has no configuration of its own. On an initialized group, adding Pods adds non-voting strangers: the extra PD starts, the peer list changes, and the voting configuration does not. PD exposes the -change through RaftEngine.changePeerList, reachable from the PD client API but -from no REST route, so the chart cannot perform it and does not pretend to. +change through RaftEngine.changePeerList and its REST management route, but +the chart does not coordinate node preparation, catch-up and member completion. Growing Store is ordinary scale-out and stays allowed. */}} {{- if and (eq $comp "pd") (gt $liveReplicas 0) (gt $desired $liveReplicas) -}} diff --git a/helm/hugegraph/templates/bootstrap-configmap.yaml b/helm/hugegraph/templates/bootstrap-configmap.yaml new file mode 100644 index 0000000000..3d20a8cd88 --- /dev/null +++ b/helm/hugegraph/templates/bootstrap-configmap.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{ include "hugegraph.resolveValues" . }} +{{- if get (.Values.bootstrap | default dict) "enabled" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hugegraph.bootstrap.name" . }} + annotations: + helm.sh/resource-policy: keep + labels: + {{- include "hugegraph.labels" . | nindent 4 }} +immutable: true +data: + schemaVersion: "1" + pdReplicas: {{ .Values.__hugegraphBootstrap.pdReplicas | quote }} + storeReplicas: {{ .Values.__hugegraphBootstrap.storeReplicas | quote }} + initialStoreCount: {{ .Values.__hugegraphBootstrap.initialStoreCount | quote }} +{{- end }} diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index d1603e319d..00d57efd64 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -122,7 +122,7 @@ spec: - name: HG_PD_INITIAL_STORE_LIST value: {{ include "hugegraph.store.initialStoreList" . | quote }} - name: HG_PD_INITIAL_STORE_COUNT - value: {{ .Values.store.replicas | quote }} + value: {{ .Values.__hugegraphBootstrap.initialStoreCount | quote }} - name: HG_PD_DATA_PATH value: {{ .Values.pd.dataPath | quote }} - name: HG_PD_AUTH_SECRET_KEY diff --git a/helm/hugegraph/templates/store-statefulset.yaml b/helm/hugegraph/templates/store-statefulset.yaml index e76832df75..0907cf2f2f 100644 --- a/helm/hugegraph/templates/store-statefulset.yaml +++ b/helm/hugegraph/templates/store-statefulset.yaml @@ -101,7 +101,7 @@ spec: - -c - | set -eu - REQUIRED={{ include "hugegraph.pd.quorum" . }} + REQUIRED={{ include "hugegraph.pd.bootstrapQuorum" . }} HEALTH_PEERS=$(echo "{{ include "hugegraph.pd.restPeersList" . }}" | tr ',' ' ') TIMEOUT={{ .Values.store.waitTimeoutSeconds | default 900 }} DEADLINE=$(( $(date +%s) + TIMEOUT )) diff --git a/helm/hugegraph/tests/bootstrap_consumer_mismatch_test.yaml b/helm/hugegraph/tests/bootstrap_consumer_mismatch_test.yaml new file mode 100644 index 0000000000..31cd00405a --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_consumer_mismatch_test.yaml @@ -0,0 +1,119 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap refuses inconsistent existing consumers +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500,cluster-store-3.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' + env: + - name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686 +tests: +- it: does not silently rewrite an existing Store seed template + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap import found store seeds that differ diff --git a/helm/hugegraph/tests/bootstrap_corrupt_record_test.yaml b/helm/hugegraph/tests/bootstrap_corrupt_record_test.yaml new file mode 100644 index 0000000000..8f9528f543 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_corrupt_record_test.yaml @@ -0,0 +1,132 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap rejects corrupt records +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: refuses a missing count rather than deriving from desired + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap record is missing a valid initialStoreCount +- it: refuses to repair incomplete history through a value pin + template: store-statefulset.yaml + set: + bootstrap.initialStoreCount: 3 + asserts: + - failedTemplate: + errorPattern: bootstrap record is missing a valid initialStoreCount diff --git a/helm/hugegraph/tests/bootstrap_custom_legacy_test.yaml b/helm/hugegraph/tests/bootstrap_custom_legacy_test.yaml new file mode 100644 index 0000000000..b352f59e90 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_custom_legacy_test.yaml @@ -0,0 +1,116 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap rejects customized legacy endpoints +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: custom-store:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: requires review instead of rewriting custom Store endpoints + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap import only supports the chart diff --git a/helm/hugegraph/tests/bootstrap_fresh_test.yaml b/helm/hugegraph/tests/bootstrap_fresh_test.yaml new file mode 100644 index 0000000000..b6b4d9e538 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_fresh_test.yaml @@ -0,0 +1,219 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap fresh installs +templates: +- bootstrap-configmap.yaml +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token +tests: +- it: records fresh 1 PD and Store topology + template: bootstrap-configmap.yaml + set: &id001 + pd.replicas: 1 + store.replicas: 1 + pd.pdb.minAvailable: 1 + store.pdb.minAvailable: 1 + asserts: + - equal: + path: data.pdReplicas + value: '1' + - equal: + path: data.storeReplicas + value: '1' + - equal: + path: data.initialStoreCount + value: '1' + - equal: + path: immutable + value: true +- it: initializes all 1 fresh PD voters + template: pd-statefulset.yaml + set: *id001 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '1' +- it: gives fresh 1 topology matching client seeds + template: store-statefulset.yaml + set: *id001 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686 +- it: records fresh 3 PD and Store topology + template: bootstrap-configmap.yaml + set: &id002 + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 + asserts: + - equal: + path: data.pdReplicas + value: '3' + - equal: + path: data.storeReplicas + value: '3' + - equal: + path: data.initialStoreCount + value: '3' + - equal: + path: immutable + value: true +- it: initializes all 3 fresh PD voters + template: pd-statefulset.yaml + set: *id002 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '3' +- it: gives fresh 3 topology matching client seeds + template: store-statefulset.yaml + set: *id002 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 +- it: records fresh 5 PD and Store topology + template: bootstrap-configmap.yaml + set: &id003 + pd.replicas: 5 + store.replicas: 5 + pd.pdb.minAvailable: 3 + store.pdb.minAvailable: 4 + asserts: + - equal: + path: data.pdReplicas + value: '5' + - equal: + path: data.storeReplicas + value: '5' + - equal: + path: data.initialStoreCount + value: '5' + - equal: + path: immutable + value: true +- it: initializes all 5 fresh PD voters + template: pd-statefulset.yaml + set: *id003 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610,cluster-pd-3.cluster-pd.NAMESPACE.svc:8610,cluster-pd-4.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500,cluster-store-3.cluster-store.NAMESPACE.svc:8500,cluster-store-4.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '5' +- it: gives fresh 5 topology matching client seeds + template: store-statefulset.yaml + set: *id003 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686,cluster-pd-3.cluster-pd.NAMESPACE.svc:8686,cluster-pd-4.cluster-pd.NAMESPACE.svc:8686 +- it: rejects fresh PD5 with a three-voter bootstrap + template: store-statefulset.yaml + set: + pd.replicas: 5 + pd.pdb.minAvailable: 3 + bootstrap.pdReplicas: 3 + asserts: + - failedTemplate: + errorPattern: bootstrap.pdReplicas must equal pd.replicas on a fresh install +- it: rejects unreachable startup Store threshold + template: store-statefulset.yaml + set: + bootstrap.initialStoreCount: 4 + asserts: + - failedTemplate: + errorPattern: bootstrap.initialStoreCount exceeds the Stores started by a fresh + install +- it: requires offline upgrades to pin all history + release: + upgrade: true + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: must all be pinned for an upgrade without a live bootstrap record +- it: allows an offline upgrade with explicitly pinned history + release: + upgrade: true + template: pd-statefulset.yaml + set: + store.replicas: 4 + store.pdb.minAvailable: 3 + bootstrap.pdReplicas: 3 + bootstrap.storeReplicas: 3 + bootstrap.initialStoreCount: 3 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 diff --git a/helm/hugegraph/tests/bootstrap_incomplete_legacy_test.yaml b/helm/hugegraph/tests/bootstrap_incomplete_legacy_test.yaml new file mode 100644 index 0000000000..a25be70ed8 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_incomplete_legacy_test.yaml @@ -0,0 +1,123 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap rejects incomplete live initialization +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500,cluster-store-3.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + valueFrom: + configMapKeyRef: + name: custom + key: count + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: does not guess a live valueFrom count even with pins + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 3 + bootstrap.storeReplicas: 4 + bootstrap.initialStoreCount: 3 + asserts: + - failedTemplate: + errorPattern: bootstrap import requires the existing PD peer list diff --git a/helm/hugegraph/tests/bootstrap_legacy_test.yaml b/helm/hugegraph/tests/bootstrap_legacy_test.yaml new file mode 100644 index 0000000000..64c040e4d7 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_legacy_test.yaml @@ -0,0 +1,170 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap bootstrap_legacy_test.yaml +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500,cluster-store-3.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: retains PD bootstrap inputs + template: pd-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500,cluster-store-3.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '3' +- it: retains Store seeds and bootstrap wait quorum + template: store-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: REQUIRED=2 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: cluster-pd-0\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-1\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-2\.cluster-pd\.NAMESPACE\.svc:8620 +- it: retains Server seeds + template: server-deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_PD_PEERS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 +- it: records the original topology + template: bootstrap-configmap.yaml + asserts: + - equal: + path: data.pdReplicas + value: '3' + - equal: + path: data.storeReplicas + value: '4' + - equal: + path: data.initialStoreCount + value: '3' +- it: refuses a bootstrap pin that conflicts with history + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 5 + asserts: + - failedTemplate: + errorPattern: bootstrap.pdReplicas cannot change diff --git a/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml b/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml new file mode 100644 index 0000000000..c91320d90d --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml @@ -0,0 +1,194 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap bootstrap_pd_growth_test.yaml +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 5 + store.replicas: 3 + pd.pdb.minAvailable: 3 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '5' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: retains PD bootstrap inputs + template: pd-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '3' +- it: retains Store seeds and bootstrap wait quorum + template: store-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: REQUIRED=2 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: cluster-pd-0\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-1\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-2\.cluster-pd\.NAMESPACE\.svc:8620 +- it: retains Server seeds + template: server-deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_PD_PEERS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 +- it: records the original topology + template: bootstrap-configmap.yaml + asserts: + - equal: + path: data.pdReplicas + value: '3' + - equal: + path: data.storeReplicas + value: '3' + - equal: + path: data.initialStoreCount + value: '3' +- it: refuses a bootstrap pin that conflicts with history + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 5 + asserts: + - failedTemplate: + errorPattern: bootstrap.pdReplicas cannot change +- it: refuses disabling the retained record + template: store-statefulset.yaml + set: + bootstrap.enabled: false + asserts: + - failedTemplate: + errorPattern: bootstrap.enabled cannot be disabled +- it: validates current PD5 quorum for PDB rather than old PD3 seeds + template: server-deployment.yaml + set: + pd.pdb.minAvailable: 2 + asserts: + - failedTemplate: + errorPattern: pd.pdb.minAvailable must be at least the PD Raft majority diff --git a/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml b/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml new file mode 100644 index 0000000000..b9999a8715 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml @@ -0,0 +1,187 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap bootstrap_pd_shrink_test.yaml +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + data: + schemaVersion: '1' + pdReplicas: '5' + storeReplicas: '3' + initialStoreCount: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610,cluster-pd-3.cluster-pd.NAMESPACE.svc:8610,cluster-pd-4.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: retains PD bootstrap inputs + template: pd-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610,cluster-pd-3.cluster-pd.NAMESPACE.svc:8610,cluster-pd-4.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '3' +- it: retains Store seeds and bootstrap wait quorum + template: store-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686,cluster-pd-3.cluster-pd.NAMESPACE.svc:8686,cluster-pd-4.cluster-pd.NAMESPACE.svc:8686 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: REQUIRED=3 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: cluster-pd-0\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-1\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-2\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-3\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-4\.cluster-pd\.NAMESPACE\.svc:8620 +- it: retains Server seeds + template: server-deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_PD_PEERS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686,cluster-pd-3.cluster-pd.NAMESPACE.svc:8686,cluster-pd-4.cluster-pd.NAMESPACE.svc:8686 +- it: records the original topology + template: bootstrap-configmap.yaml + asserts: + - equal: + path: data.pdReplicas + value: '5' + - equal: + path: data.storeReplicas + value: '3' + - equal: + path: data.initialStoreCount + value: '3' +- it: refuses a bootstrap pin that conflicts with history + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 7 + asserts: + - failedTemplate: + errorPattern: bootstrap.pdReplicas cannot change +- it: refuses disabling the retained record + template: store-statefulset.yaml + set: + bootstrap.enabled: false + asserts: + - failedTemplate: + errorPattern: bootstrap.enabled cannot be disabled diff --git a/helm/hugegraph/tests/bootstrap_record_test.yaml b/helm/hugegraph/tests/bootstrap_record_test.yaml new file mode 100644 index 0000000000..366eec47e3 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_record_test.yaml @@ -0,0 +1,203 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap bootstrap_record_test.yaml +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: retains PD bootstrap inputs + template: pd-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '3' +- it: retains Store seeds and bootstrap wait quorum + template: store-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: REQUIRED=2 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: cluster-pd-0\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-1\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-2\.cluster-pd\.NAMESPACE\.svc:8620 +- it: retains Server seeds + template: server-deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_PD_PEERS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 +- it: records the original topology + template: bootstrap-configmap.yaml + asserts: + - equal: + path: data.pdReplicas + value: '3' + - equal: + path: data.storeReplicas + value: '3' + - equal: + path: data.initialStoreCount + value: '3' +- it: refuses a bootstrap pin that conflicts with history + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 5 + asserts: + - failedTemplate: + errorPattern: bootstrap.pdReplicas cannot change +- it: refuses disabling the retained record + template: store-statefulset.yaml + set: + bootstrap.enabled: false + asserts: + - failedTemplate: + errorPattern: bootstrap.enabled cannot be disabled +- it: retains original PD grow guard + template: server-deployment.yaml + set: + pd.replicas: 5 + pd.pdb.minAvailable: 3 + asserts: + - failedTemplate: + errorPattern: pd.replicas cannot grow from 3 to 5 +- it: retains original Store shrink guard + template: server-deployment.yaml + set: + store.replicas: 3 + store.pdb.minAvailable: 2 + asserts: + - failedTemplate: + errorPattern: store.replicas cannot shrink from 4 to 3 diff --git a/helm/hugegraph/tests/bootstrap_record_version_test.yaml b/helm/hugegraph/tests/bootstrap_record_version_test.yaml new file mode 100644 index 0000000000..037b15e5b6 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_record_version_test.yaml @@ -0,0 +1,126 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap rejects unsupported record versions +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + data: + schemaVersion: '2' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: rejects a record schema it cannot interpret + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: unsupported schemaVersion diff --git a/helm/hugegraph/tests/bootstrap_reinstall_test.yaml b/helm/hugegraph/tests/bootstrap_reinstall_test.yaml new file mode 100644 index 0000000000..5f5099eac0 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_reinstall_test.yaml @@ -0,0 +1,127 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap refuses implicit recovery +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: refuses treating retained topology as a fresh install + template: store-statefulset.yaml + set: + pd.replicas: 5 + pd.pdb.minAvailable: 3 + asserts: + - failedTemplate: + errorPattern: bootstrap history already exists on an install diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index d1a134cb05..27912a3110 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -122,6 +122,51 @@ "minLength": 1, "pattern": "^[A-Za-z0-9_.-]+$", "description": "Shared Server/Store PD metadata namespace. Preserve the initialized Server value; changing it does not migrate data." + }, + "bootstrap": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "pdReplicas": { + "oneOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "string", + "pattern": "^$|^[1-9][0-9]*$" + } + ] + }, + "storeReplicas": { + "oneOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "string", + "pattern": "^$|^[1-9][0-9]*$" + } + ] + }, + "initialStoreCount": { + "oneOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "string", + "pattern": "^$|^[1-9][0-9]*$" + } + ] + } + } } }, "definitions": { diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 3e6070a5d4..1e3c4b73fa 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -29,6 +29,16 @@ global: persistence: enabled: true +# Opt-in stable initialization topology, separate from desired node replicas. +# Online renders capture the first topology in a kept ConfigMap, or import the +# existing PD environment on the first upgrade. Offline upgrades must pin all +# three counts and render with --is-upgrade. This does not change membership. +bootstrap: + enabled: false + pdReplicas: "" + storeReplicas: "" + initialStoreCount: "" + fullnameOverride: "" nameOverride: "" diff --git a/helm/test_helm_bootstrap.py b/helm/test_helm_bootstrap.py new file mode 100644 index 0000000000..668fdcc26d --- /dev/null +++ b/helm/test_helm_bootstrap.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compare whole rendered Pod templates across controlled topology changes.""" + +from pathlib import Path +import re +import subprocess +import unittest + + +CHART = Path(__file__).resolve().parent / "hugegraph" + + +class BootstrapLifecycleTest(unittest.TestCase): + def render(self, pd=3, stores=3, initial_pd=3, initial_stores=3, initial_count=3, + upgrade=True): + values = { + "fullnameOverride": "cluster", + "bootstrap.enabled": "true", + "pd.replicas": pd, + "store.replicas": stores, + "pd.pdb.minAvailable": pd // 2 + 1, + "store.pdb.minAvailable": max(stores - 1, 1), + "pd.auth.existingSecret": "fixture-pd", + "server.auth.admin.existingSecret": "fixture-admin", + "server.auth.token.existingSecret": "fixture-token", + } + if upgrade: + values.update({ + "bootstrap.pdReplicas": initial_pd, + "bootstrap.storeReplicas": initial_stores, + "bootstrap.initialStoreCount": initial_count, + }) + command = ["helm", "template", "bootstrap-test", str(CHART), + "--namespace", "bootstrap-test"] + if upgrade: + command.append("--is-upgrade") + for key, value in values.items(): + command.extend(["--set", f"{key}={value}"]) + result = subprocess.run(command, check=True, capture_output=True, text=True) + documents = {} + for document in result.stdout.split("---\n"): + source = re.search(r"^# Source: [^\n]+/templates/([^\n]+)$", document, re.M) + if source: + documents[source.group(1)] = document + return documents + + def template(self, documents, component): + name = "server-deployment.yaml" if component == "server" else component + "-statefulset.yaml" + lines = documents[name].splitlines(keepends=True) + start = next(i for i, line in enumerate(lines) if line == " template:\n") + end = start + 1 + while end < len(lines) and (not lines[end].strip() or lines[end].startswith(" ")): + end += 1 + return "".join(lines[start:end]) + + def assert_templates_unchanged(self, before, after): + for component in ("pd", "store", "server"): + with self.subTest(component=component): + self.assertEqual(self.template(before, component), self.template(after, component)) + + def test_store_growth_only_changes_capacity_resources(self): + self.assert_templates_unchanged(self.render(stores=3), self.render(stores=4)) + + def test_store_growth_across_default_rf_boundary_preserves_bootstrap(self): + before = self.render(stores=1, initial_stores=1, initial_count=1) + after = self.render(stores=3, initial_stores=1, initial_count=1) + self.assert_templates_unchanged(before, after) + self.assertIn("-Dpartition.default-shard-count=1", self.template(after, "pd")) + + def test_pd_growth_keeps_initial_peers_and_consumers_stable(self): + self.assert_templates_unchanged(self.render(pd=3), self.render(pd=5)) + + def test_pd_shrink_keeps_recorded_five_peer_bootstrap(self): + before = self.render(pd=5, initial_pd=5) + after = self.render(pd=3, initial_pd=5) + self.assert_templates_unchanged(before, after) + + def test_fresh_topologies_bootstrap_the_requested_members(self): + for count in (1, 3, 5): + with self.subTest(replicas=count): + documents = self.render(pd=count, stores=count, upgrade=False) + pd = documents["pd-statefulset.yaml"] + value = re.search(r'- name: HG_PD_RAFT_PEERS_LIST\n\s+value: "([^"]+)"', pd).group(1) + self.assertEqual(count, len(value.split(","))) + self.assertIn(f' pdReplicas: "{count}"', documents["bootstrap-configmap.yaml"]) + self.assertIn(f' initialStoreCount: "{count}"', documents["bootstrap-configmap.yaml"]) + + +if __name__ == "__main__": + unittest.main() From 97b1a6c14f39bf0241696b33bc3a8ccc6f1c6c65 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 29 Oct 2026 00:10:36 +0800 Subject: [PATCH 2/8] fix(helm): reject unowned bootstrap history --- helm/hugegraph/templates/_bootstrap.tpl | 7 ++- .../tests/bootstrap_corrupt_record_test.yaml | 2 + .../tests/bootstrap_pd_growth_test.yaml | 2 + .../tests/bootstrap_pd_shrink_test.yaml | 2 + .../bootstrap_record_missing_owner_test.yaml | 56 ++++++++++++++++++ .../tests/bootstrap_record_test.yaml | 2 + .../tests/bootstrap_record_version_test.yaml | 2 + .../bootstrap_record_wrong_owner_test.yaml | 58 +++++++++++++++++++ 8 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 helm/hugegraph/tests/bootstrap_record_missing_owner_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_record_wrong_owner_test.yaml diff --git a/helm/hugegraph/templates/_bootstrap.tpl b/helm/hugegraph/templates/_bootstrap.tpl index 2cd0f67856..393343fe79 100644 --- a/helm/hugegraph/templates/_bootstrap.tpl +++ b/helm/hugegraph/templates/_bootstrap.tpl @@ -46,7 +46,10 @@ never from StatefulSet replicas that a controlled operation may already change. {{- fail "bootstrap history already exists on an install; use a reviewed recovery procedure rather than treating kept data as a fresh cluster" -}} {{- end -}} {{- $owner := dig "metadata" "labels" "app.kubernetes.io/instance" "" $record -}} -{{- if and $owner (ne $owner .Release.Name) -}} +{{- if not $owner -}} +{{- fail "bootstrap ConfigMap is missing the app.kubernetes.io/instance owner label; inspect the retained initialization topology before continuing" -}} +{{- end -}} +{{- if ne $owner .Release.Name -}} {{- fail "bootstrap ConfigMap belongs to another release; inspect the retained initialization topology" -}} {{- end -}} {{- $data := get $record "data" | default dict -}} @@ -146,4 +149,4 @@ never from StatefulSet replicas that a controlled operation may already change. {{- end -}} {{- end -}} {{- end -}} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/hugegraph/tests/bootstrap_corrupt_record_test.yaml b/helm/hugegraph/tests/bootstrap_corrupt_record_test.yaml index 8f9528f543..04da1daa0e 100644 --- a/helm/hugegraph/tests/bootstrap_corrupt_record_test.yaml +++ b/helm/hugegraph/tests/bootstrap_corrupt_record_test.yaml @@ -52,6 +52,8 @@ kubernetesProvider: metadata: name: cluster-bootstrap namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME data: schemaVersion: '1' pdReplicas: '3' diff --git a/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml b/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml index c91320d90d..9f857423f3 100644 --- a/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml +++ b/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml @@ -52,6 +52,8 @@ kubernetesProvider: metadata: name: cluster-bootstrap namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME data: schemaVersion: '1' pdReplicas: '3' diff --git a/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml b/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml index b9999a8715..c3c13829a4 100644 --- a/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml +++ b/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml @@ -52,6 +52,8 @@ kubernetesProvider: metadata: name: cluster-bootstrap namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME data: schemaVersion: '1' pdReplicas: '5' diff --git a/helm/hugegraph/tests/bootstrap_record_missing_owner_test.yaml b/helm/hugegraph/tests/bootstrap_record_missing_owner_test.yaml new file mode 100644 index 0000000000..ee9f32cd6e --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_record_missing_owner_test.yaml @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Bootstrap rejects an ownerless retained record +templates: +- store-statefulset.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' +tests: +- it: fails closed when retained history has no release owner + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: missing the app.kubernetes.io/instance owner label diff --git a/helm/hugegraph/tests/bootstrap_record_test.yaml b/helm/hugegraph/tests/bootstrap_record_test.yaml index 366eec47e3..e2181362bb 100644 --- a/helm/hugegraph/tests/bootstrap_record_test.yaml +++ b/helm/hugegraph/tests/bootstrap_record_test.yaml @@ -52,6 +52,8 @@ kubernetesProvider: metadata: name: cluster-bootstrap namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME data: schemaVersion: '1' pdReplicas: '3' diff --git a/helm/hugegraph/tests/bootstrap_record_version_test.yaml b/helm/hugegraph/tests/bootstrap_record_version_test.yaml index 037b15e5b6..38d54aa2ad 100644 --- a/helm/hugegraph/tests/bootstrap_record_version_test.yaml +++ b/helm/hugegraph/tests/bootstrap_record_version_test.yaml @@ -52,6 +52,8 @@ kubernetesProvider: metadata: name: cluster-bootstrap namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME data: schemaVersion: '2' pdReplicas: '3' diff --git a/helm/hugegraph/tests/bootstrap_record_wrong_owner_test.yaml b/helm/hugegraph/tests/bootstrap_record_wrong_owner_test.yaml new file mode 100644 index 0000000000..c9e38de3a0 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_record_wrong_owner_test.yaml @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Bootstrap rejects another release's retained record +templates: +- store-statefulset.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + labels: + app.kubernetes.io/instance: another-release + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' +tests: +- it: fails closed when retained history belongs to another release + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap ConfigMap belongs to another release From bc2c897e032179a4f40ae45b48ffab354ef3f78c Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 29 Oct 2026 00:14:15 +0800 Subject: [PATCH 3/8] style(helm): preserve template newline --- helm/hugegraph/templates/_bootstrap.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/hugegraph/templates/_bootstrap.tpl b/helm/hugegraph/templates/_bootstrap.tpl index 393343fe79..e33ceebf34 100644 --- a/helm/hugegraph/templates/_bootstrap.tpl +++ b/helm/hugegraph/templates/_bootstrap.tpl @@ -149,4 +149,4 @@ never from StatefulSet replicas that a controlled operation may already change. {{- end -}} {{- end -}} {{- end -}} -{{- end }} \ No newline at end of file +{{- end }} From 7ab9bc3e313719693a7893275de9a79dca194c71 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 29 Oct 2026 19:27:05 +0800 Subject: [PATCH 4/8] ci(helm): run stable bootstrap topology regression in Helm checks --- .github/actions/helm-checks/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/helm-checks/action.yml b/.github/actions/helm-checks/action.yml index f669ec060b..089b59f518 100644 --- a/.github/actions/helm-checks/action.yml +++ b/.github/actions/helm-checks/action.yml @@ -59,6 +59,10 @@ runs: shell: 'bash -e {0}' run: python3 -m unittest discover -s helm -p 'test_helm_metadata_cluster.py' -v + - name: Stable bootstrap topology regression + shell: 'bash -e {0}' + run: python3 -m unittest discover -s helm -p 'test_helm_bootstrap.py' -v + - name: helm unittest shell: 'bash -e {0}' run: | From 0519f2bffde0f3361a53c659e9ec7a53c3515d39 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 29 Oct 2026 22:26:02 +0800 Subject: [PATCH 5/8] fix(helm): harden retained bootstrap adoption guards --- helm/hugegraph/templates/_bootstrap.tpl | 58 ++++++++---- helm/hugegraph/templates/pd-statefulset.yaml | 4 + .../bootstrap_consumer_missing_test.yaml | 91 +++++++++++++++++++ .../tests/bootstrap_disable_guard_test.yaml | 64 +++++++++++++ .../bootstrap_disabled_upgrade_test.yaml | 65 +++++++++++++ .../hugegraph/tests/bootstrap_fresh_test.yaml | 41 ++++++++- .../tests/bootstrap_legacy_owner_test.yaml | 74 +++++++++++++++ .../tests/bootstrap_pd_growth_test.yaml | 2 + .../tests/bootstrap_pd_shrink_test.yaml | 2 + .../tests/bootstrap_record_limit_test.yaml | 59 ++++++++++++ .../tests/bootstrap_record_test.yaml | 2 + 11 files changed, 439 insertions(+), 23 deletions(-) create mode 100644 helm/hugegraph/tests/bootstrap_consumer_missing_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_disable_guard_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_disabled_upgrade_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_legacy_owner_test.yaml create mode 100644 helm/hugegraph/tests/bootstrap_record_limit_test.yaml diff --git a/helm/hugegraph/templates/_bootstrap.tpl b/helm/hugegraph/templates/_bootstrap.tpl index e33ceebf34..b6c52b1936 100644 --- a/helm/hugegraph/templates/_bootstrap.tpl +++ b/helm/hugegraph/templates/_bootstrap.tpl @@ -32,15 +32,19 @@ never from StatefulSet replicas that a controlled operation may already change. {{- define "hugegraph.bootstrap.resolve" -}} {{- $config := get .Values "bootstrap" | default dict -}} {{- $enabled := get $config "enabled" | default false -}} -{{- $record := lookup "v1" "ConfigMap" .Release.Namespace (include "hugegraph.bootstrap.name" .) -}} -{{- $history := dict -}} -{{- $legacyEnv := dict -}} {{- if not $enabled -}} -{{- if $record -}} -{{- fail "bootstrap.enabled cannot be disabled while the retained bootstrap ConfigMap exists; keep initialization topology pinned" -}} +{{- $resolved := dict "pdReplicas" (int .Values.pd.replicas) "storeReplicas" (int .Values.store.replicas) "initialStoreCount" (int .Values.store.replicas) -}} +{{- $_ := set .Values "__hugegraphBootstrap" $resolved -}} +{{- if .Release.IsUpgrade -}} +{{- $live := lookup "apps/v1" "StatefulSet" .Release.Namespace (include "hugegraph.pd.name" .) -}} +{{- if and $live (eq (dig "metadata" "annotations" "hugegraph.apache.org/bootstrap-retained" "" $live) "true") -}} +{{- fail "bootstrap.enabled cannot be disabled while the live PD StatefulSet records retained bootstrap topology" -}} +{{- end -}} {{- end -}} -{{- $_ := set .Values "__hugegraphBootstrap" (dict "pdReplicas" (int .Values.pd.replicas) "storeReplicas" (int .Values.store.replicas) "initialStoreCount" (int .Values.store.replicas)) -}} {{- else -}} +{{- $record := lookup "v1" "ConfigMap" .Release.Namespace (include "hugegraph.bootstrap.name" .) -}} +{{- $history := dict -}} +{{- $legacyEnv := dict -}} {{- if $record -}} {{- if not .Release.IsUpgrade -}} {{- fail "bootstrap history already exists on an install; use a reviewed recovery procedure rather than treating kept data as a fresh cluster" -}} @@ -60,11 +64,18 @@ never from StatefulSet replicas that a controlled operation may already change. {{- $_ := set $history $key (get $data $key | default "") -}} {{- end -}} {{- else -}} -{{- $live := lookup "apps/v1" "StatefulSet" .Release.Namespace (include "hugegraph.pd.name" .) -}} +{{- $live := lookup "apps/v1" "StatefulSet" .Release.Namespace (includ "hugegraph.pd.name" .) -}} {{- if $live -}} {{- if not .Release.IsUpgrade -}} {{- fail "bootstrap import requires an upgrade of the existing release; inspect live PD and retained data before recovery" -}} {{- end -}} +{{- $owner := dig "metadata" "labels" "app.kubernetes.io/instance" "" $live -}} +{{- if not $owner -}} +{{- fail "bootstrap import requires the existing PD StatefulSet to carry the app.kubernetes.io/instance owner label" -}} +{{- end -} +{{- if ne $owner .Release.Name -}} +{{- fail "bootstrap import found a PD StatefulSet owned by another release" -}} +{{- end -} {{- $env := dict -}} {{- range $container := dig "spec" "template" "spec" "containers" list $live -}} {{- if eq (get $container "name" | default "") "pd" -}} @@ -85,7 +96,7 @@ never from StatefulSet replicas that a controlled operation may already change. {{- $legacyEnv = $env -}} {{- else if .Release.IsUpgrade -}} {{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}} -{{- if eq (include "hugegraph.optionalScalar" (get $config $key)) "" -}} +{{- if eq (includ "hugegraph.optionalScalar" (get $config $key)) "" -}} {{- fail "bootstrap.pdReplicas, bootstrap.storeReplicas and bootstrap.initialStoreCount must all be pinned for an upgrade without a live bootstrap record" -}} {{- end -}} {{- end -}} @@ -93,8 +104,8 @@ never from StatefulSet replicas that a controlled operation may already change. {{- end -}} {{- if not (empty $history) -}} {{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}} -{{- if not (regexMatch "^[1-9][0-9]*$" (get $history $key | toString)) -}} -{{- fail (printf "bootstrap record is missing a valid %s; inspect it before continuing" $key) -}} +{{- if not (regexMatch "^[1-9][0-9]?$" (get $history $key | toString)) -}} +{{- fail (printf "bootstrap record is missing a valid %s between 1 and 99; inspect it before continuing" $key) -}} {{- end -}} {{- end -}} {{- end -}} @@ -108,20 +119,20 @@ never from StatefulSet replicas that a controlled operation may already change. {{- end -}} {{- $value := ternary $pin $old (ne $pin "") -}} {{- if eq $value "" -}}{{- $value = get $defaults $key | toString -}}{{- end -}} -{{- if or (not (regexMatch "^[1-9][0-9]*$" $value)) (lt (int $value) 1) -}} -{{- fail (printf "bootstrap.%s must be a positive integer in the retained bootstrap record" $key) -}} +{{- if not (regexMatch "^[1-9][0-9]?$" $value) -}} +{{- fail (printf "bootstrap.%s must be an integer between 1 and 99" $key) -}} {{- end -}} {{- $_ := set $resolved $key (int $value) -}} -{{- end -}} +{{- end -} +{{- if gt (int $resolved.initialStoreCount) (int $resolved.storeReplicas) -}} +{{- fail "bootstrap.initialStoreCount cannot exceed bootstrap.storeReplicas: the initial allowlist could not satisfy startup" -}} +{{- end -} {{- if and (not .Release.IsUpgrade) (empty $history) -}} {{- if ne (int $resolved.pdReplicas) (int .Values.pd.replicas) -}} {{- fail "bootstrap.pdReplicas must equal pd.replicas on a fresh install; use --is-upgrade with pinned history for offline upgrades" -}} {{- end -}} -{{- if gt (int $resolved.initialStoreCount) (int .Values.store.replicas) -}} -{{- fail "bootstrap.initialStoreCount exceeds the Stores started by a fresh install" -}} -{{- end -}} -{{- if gt (int $resolved.initialStoreCount) (int $resolved.storeReplicas) -}} -{{- fail "bootstrap.initialStoreCount cannot exceed bootstrap.storeReplicas on a fresh install: the initial allowlist could not satisfy startup" -}} +{{- if ne (int $resolved.storeReplicas) (int .Values.store.replicas) -}} +{{- fail "bootstrap.storeReplicas must equal store.replicas on a fresh install; use --is-upgrade with pinned history for offline upgrades" -}} {{- end -}} {{- end -}} {{- $_ := set .Values "__hugegraphBootstrap" $resolved -}} @@ -136,15 +147,22 @@ never from StatefulSet replicas that a controlled operation may already change. {{- $consumer := lookup "apps/v1" $kind $.Release.Namespace $name -}} {{- $key := ternary "HG_STORE_PD_ADDRESS" "HG_SERVER_PD_PEERS" (eq $component "store") -}} {{- if $consumer -}} +{{- $found := false -}} {{- range $container := dig "spec" "template" "spec" "containers" list $consumer -}} {{- if eq (get $container "name" | default "") $component -}} {{- range $item := get $container "env" | default list -}} -{{- if and (eq (get $item "name" | default "") $key) (ne (get $item "value" | default "") $seed) -}} -{{- fail (printf "bootstrap import found %s seeds that differ from the PD initial peers; reconcile the existing templates before migration" $component) -}} +{{- if eq (get $item "name" | default "") $key -}} +{{- $found = true -}} +{{- if ne (get $item "value" | default "") $seed -}} +{{- fail (printf "bootstrap import found %s seeds that differ from the PD initial peers; reconcile the existing template before migration" $component) -}} +{{- end -} {{- end -}} {{- end -}} {{- end -}} {{- end -}} +{{- if not $found -}} +{{- fail (printf "bootstrap import requires the existing %s template to expose %s as a literal value" $component $key) -}} +{{- end -} {{- end -}} {{- end -}} {{- end -}} diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index 00d57efd64..cf52177bfa 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -23,6 +23,10 @@ metadata: labels: {{- include "hugegraph.labels" . | nindent 4 }} app.kubernetes.io/component: pd + {{- if get (.Values.bootstrap | default dict) "enabled" }} + annotations: + hugegraph.apache.org/bootstrap-retained: "true" + {{- end }} spec: serviceName: {{ include "hugegraph.pd.name" . }} replicas: {{ .Values.pd.replicas }} diff --git a/helm/hugegraph/tests/bootstrap_consumer_missing_test.yaml b/helm/hugegraph/tests/bootstrap_consumer_missing_test.yaml new file mode 100644 index 0000000000..e73e4308d2 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_consumer_missing_test.yaml @@ -0,0 +1,91 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Bootstrap import rejects an incomplete existing Store seed template +templates: +- store-statefulset.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + template: + spec: + containers: + - name: pd + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + template: + spec: + containers: + - name: store + env: + - name: ANOTHER_ENV + value: present +tests: +- it: fails instead of silently adding a missing literal Store seed + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap import requires the existing store template to expose HG_STORE_PD_ADDRESS as a literal value diff --git a/helm/hugegraph/tests/bootstrap_disable_guard_test.yaml b/helm/hugegraph/tests/bootstrap_disable_guard_test.yaml new file mode 100644 index 0000000000..cc539fc8f6 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_disable_guard_test.yaml @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Disabled bootstrap refuses to discard retained live topology +templates: +- pd-statefulset.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: false + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + annotations: + hugegraph.apache.org/bootstrap-retained: 'true' + labels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + template: + spec: + containers: + - name: pd + env: [] +tests: +- it: requires bootstrap to remain enabled after the retained marker is recorded + template: pd-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap.enabled cannot be disabled while the live PD StatefulSet records retained bootstrap topology diff --git a/helm/hugegraph/tests/bootstrap_disabled_upgrade_test.yaml b/helm/hugegraph/tests/bootstrap_disabled_upgrade_test.yaml new file mode 100644 index 0000000000..8f4cdcd1d6 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_disabled_upgrade_test.yaml @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Disabled bootstrap keeps the default upgrade path free of ConfigMap reads +templates: +- pd-statefulset.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: false + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + template: + spec: + containers: + - name: pd + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 +tests: +- it: renders without a ConfigMap lookup when bootstrap stays disabled + template: pd-statefulset.yaml + asserts: + - equal: + path: spec.replicas + value: 3 diff --git a/helm/hugegraph/tests/bootstrap_fresh_test.yaml b/helm/hugegraph/tests/bootstrap_fresh_test.yaml index b6b4d9e538..3026a8755e 100644 --- a/helm/hugegraph/tests/bootstrap_fresh_test.yaml +++ b/helm/hugegraph/tests/bootstrap_fresh_test.yaml @@ -91,7 +91,7 @@ tests: path: data.storeReplicas value: '3' - equal: - path: data.initialStoreCount + path: initialStoreCount value: '3' - equal: path: immutable @@ -181,14 +181,38 @@ tests: asserts: - failedTemplate: errorPattern: bootstrap.pdReplicas must equal pd.replicas on a fresh install +- it: rejects fresh Store1 with a three-Store bootstrap + template: store-statefulset.yaml + set: + store.replicas: 1 + store.pdb.minAvailable: 1 + bootstrap.storeReplicas: 3 + asserts: + - failedTemplate: + errorPattern: bootstrap.storeReplicas must equal store.replicas on a fresh install +- it: rejects fresh Store3 with a one-Store bootstrap + template: store-statefulset.yaml + set: + store.replicas: 3 + store.pdb.minAvailable: 2 + bootstrap.storeReplicas: 1 + asserts: + - failedTemplate: + errorPattern: bootstrap.storeReplicas must equal store.replicas on a fresh install - it: rejects unreachable startup Store threshold template: store-statefulset.yaml set: bootstrap.initialStoreCount: 4 asserts: - failedTemplate: - errorPattern: bootstrap.initialStoreCount exceeds the Stores started by a fresh - install + errorPattern: bootstrap.initialStoreCount cannot exceed bootstrap.storeReplicas +- it: rejects bootstrap counts above the chart replica limit + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 100 + asserts: + - failedTemplate: + errorPattern: bootstrap.pdReplicas must be an integer between 1 and 99 - it: requires offline upgrades to pin all history release: upgrade: true @@ -196,6 +220,17 @@ tests: asserts: - failedTemplate: errorPattern: must all be pinned for an upgrade without a live bootstrap record +- it: rejects internally inconsistent offline history + release: + upgrade: true + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 3 + bootstrap.storeReplicas: 3 + bootstrap.initialStoreCount: 4 + asserts: + - failedTemplate: + errorPattern: bootstrap.initialStoreCount cannot exceed bootstrap.storeReplicas - it: allows an offline upgrade with explicitly pinned history release: upgrade: true diff --git a/helm/hugegraph/tests/bootstrap_legacy_owner_test.yaml b/helm/hugegraph/tests/bootstrap_legacy_owner_test.yaml new file mode 100644 index 0000000000..55c171a25e --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_legacy_owner_test.yaml @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Bootstrap import rejects a PD StatefulSet owned by another release +templates: +- pd-statefulset.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: OTHER-RELEASE + app.kubernetes.io/component: pd + spec: + template: + spec: + containers: + - name: pd + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' +tests: +- it: fails before importing another release's initialization history + template: pd-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap import found a PD StatefulSet owned by another release diff --git a/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml b/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml index 9f857423f3..c471b36a08 100644 --- a/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml +++ b/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml @@ -64,6 +64,8 @@ kubernetesProvider: metadata: name: cluster-pd namespace: NAMESPACE + annotations: + hugegraph.apache.org/bootstrap-retained: 'true' labels: &id001 app.kubernetes.io/name: hugegraph app.kubernetes.io/instance: RELEASE-NAME diff --git a/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml b/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml index c3c13829a4..b2c501b64f 100644 --- a/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml +++ b/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml @@ -64,6 +64,8 @@ kubernetesProvider: metadata: name: cluster-pd namespace: NAMESPACE + annotations: + hugegraph.apache.org/bootstrap-retained: 'true' labels: &id001 app.kubernetes.io/name: hugegraph app.kubernetes.io/instance: RELEASE-NAME diff --git a/helm/hugegraph/tests/bootstrap_record_limit_test.yaml b/helm/hugegraph/tests/bootstrap_record_limit_test.yaml new file mode 100644 index 0000000000..463085adc3 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_record_limit_test.yaml @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Bootstrap rejects retained counts above the chart replica limit +templates: +- pd-statefulset.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME + data: + schemaVersion: '1' + pdReplicas: '100' + storeReplicas: '3' + initialStoreCount: '3' +tests: +- it: rejects an oversized retained PD count before rendering peer lists + template: pd-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap record is missing a valid pdReplicas between 1 and 99 diff --git a/helm/hugegraph/tests/bootstrap_record_test.yaml b/helm/hugegraph/tests/bootstrap_record_test.yaml index e2181362bb..d54646d7ea 100644 --- a/helm/hugegraph/tests/bootstrap_record_test.yaml +++ b/helm/hugegraph/tests/bootstrap_record_test.yaml @@ -64,6 +64,8 @@ kubernetesProvider: metadata: name: cluster-pd namespace: NAMESPACE + annotations: + hugegraph.apache.org/bootstrap-retained: 'true' labels: &id001 app.kubernetes.io/name: hugegraph app.kubernetes.io/instance: RELEASE-NAME From 96ef3e209f389a92de6257022ca56cd920412b47 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 29 Oct 2026 22:28:01 +0800 Subject: [PATCH 6/8] fix(helm): restore bootstrap template syntax --- helm/hugegraph/templates/_bootstrap.tpl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/helm/hugegraph/templates/_bootstrap.tpl b/helm/hugegraph/templates/_bootstrap.tpl index b6c52b1936..57c3538a45 100644 --- a/helm/hugegraph/templates/_bootstrap.tpl +++ b/helm/hugegraph/templates/_bootstrap.tpl @@ -64,7 +64,7 @@ never from StatefulSet replicas that a controlled operation may already change. {{- $_ := set $history $key (get $data $key | default "") -}} {{- end -}} {{- else -}} -{{- $live := lookup "apps/v1" "StatefulSet" .Release.Namespace (includ "hugegraph.pd.name" .) -}} +{{- $live := lookup "apps/v1" "StatefulSet" .Release.Namespace (include "hugegraph.pd.name" .) -}} {{- if $live -}} {{- if not .Release.IsUpgrade -}} {{- fail "bootstrap import requires an upgrade of the existing release; inspect live PD and retained data before recovery" -}} @@ -72,10 +72,10 @@ never from StatefulSet replicas that a controlled operation may already change. {{- $owner := dig "metadata" "labels" "app.kubernetes.io/instance" "" $live -}} {{- if not $owner -}} {{- fail "bootstrap import requires the existing PD StatefulSet to carry the app.kubernetes.io/instance owner label" -}} -{{- end -} +{{- end -}} {{- if ne $owner .Release.Name -}} {{- fail "bootstrap import found a PD StatefulSet owned by another release" -}} -{{- end -} +{{- end -}} {{- $env := dict -}} {{- range $container := dig "spec" "template" "spec" "containers" list $live -}} {{- if eq (get $container "name" | default "") "pd" -}} @@ -96,7 +96,7 @@ never from StatefulSet replicas that a controlled operation may already change. {{- $legacyEnv = $env -}} {{- else if .Release.IsUpgrade -}} {{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}} -{{- if eq (includ "hugegraph.optionalScalar" (get $config $key)) "" -}} +{{- if eq (include "hugegraph.optionalScalar" (get $config $key)) "" -}} {{- fail "bootstrap.pdReplicas, bootstrap.storeReplicas and bootstrap.initialStoreCount must all be pinned for an upgrade without a live bootstrap record" -}} {{- end -}} {{- end -}} @@ -123,10 +123,10 @@ never from StatefulSet replicas that a controlled operation may already change. {{- fail (printf "bootstrap.%s must be an integer between 1 and 99" $key) -}} {{- end -}} {{- $_ := set $resolved $key (int $value) -}} -{{- end -} +{{- end -}} {{- if gt (int $resolved.initialStoreCount) (int $resolved.storeReplicas) -}} {{- fail "bootstrap.initialStoreCount cannot exceed bootstrap.storeReplicas: the initial allowlist could not satisfy startup" -}} -{{- end -} +{{- end -}} {{- if and (not .Release.IsUpgrade) (empty $history) -}} {{- if ne (int $resolved.pdReplicas) (int .Values.pd.replicas) -}} {{- fail "bootstrap.pdReplicas must equal pd.replicas on a fresh install; use --is-upgrade with pinned history for offline upgrades" -}} @@ -138,7 +138,7 @@ never from StatefulSet replicas that a controlled operation may already change. {{- $_ := set .Values "__hugegraphBootstrap" $resolved -}} {{- if not (empty $legacyEnv) -}} {{- if or (ne (get $legacyEnv "HG_PD_RAFT_PEERS_LIST") (include "hugegraph.pd.raftPeersList" .)) (ne (get $legacyEnv "HG_PD_INITIAL_STORE_LIST") (include "hugegraph.store.initialStoreList" .)) -}} -{{- fail "bootstrap import only supports the chart's contiguous ordinal endpoint lists; inspect customized live endpoints before migration" -}} +{{- fail "bootstrap only supports the chart's contiguous ordinal endpoint lists; inspect customized live endpoints before migration" -}} {{- end -}} {{- $seed := include "hugegraph.pd.grpcPeersList" . -}} {{- range $component := list "store" "server" -}} @@ -154,15 +154,15 @@ never from StatefulSet replicas that a controlled operation may already change. {{- if eq (get $item "name" | default "") $key -}} {{- $found = true -}} {{- if ne (get $item "value" | default "") $seed -}} -{{- fail (printf "bootstrap import found %s seeds that differ from the PD initial peers; reconcile the existing template before migration" $component) -}} -{{- end -} +{{- fail (printf "bootstrap import found %s seeds that differ from the PD initial peers; reconcile the existing templates before migration" $component) -}} +{{- end -}} {{- end -}} {{- end -}} {{- end -}} {{- end -}} {{- if not $found -}} {{- fail (printf "bootstrap import requires the existing %s template to expose %s as a literal value" $component $key) -}} -{{- end -} +{{- end -}} {{- end -}} {{- end -}} {{- end -}} From 9b4758d05f35a57223f1c820cc872081dad7b479 Mon Sep 17 00:00:00 2001 From: orion Date: Thu, 29 Oct 2026 16:53:17 +0000 Subject: [PATCH 7/8] fix(helm): align bootstrap tests and recovery docs Correct legacy fixtures and isolate fresh topology guards. Cover retained-record checks in enabled and disabled installs. Clarify recovery, consumer drift and static seed limits. --- helm/hugegraph/docs/operations.md | 33 ++++++---- .../tests/bootstrap_custom_legacy_test.yaml | 4 +- .../bootstrap_disabled_reinstall_test.yaml | 65 +++++++++++++++++++ .../hugegraph/tests/bootstrap_fresh_test.yaml | 3 +- .../tests/bootstrap_legacy_test.yaml | 3 + 5 files changed, 93 insertions(+), 15 deletions(-) create mode 100644 helm/hugegraph/tests/bootstrap_disabled_reinstall_test.yaml diff --git a/helm/hugegraph/docs/operations.md b/helm/hugegraph/docs/operations.md index f8366ea821..297e918b23 100644 --- a/helm/hugegraph/docs/operations.md +++ b/helm/hugegraph/docs/operations.md @@ -260,12 +260,14 @@ which a controlled operation may already have changed. Import supports the chart's contiguous ordinal endpoints; customized endpoint lists require a separate reviewed migration. Import preserves a Store allowlist of four and a startup threshold of three as distinct inputs. No bootstrap checksum is added -to Pod templates, so enabling the feature does not itself request a rollout. +to Pod templates; adopting an unchanged chart-managed topology preserves these inputs. An incomplete live environment is rejected even when all value pins are given; pins cannot reconstruct unknown online history. -Existing Store and Server seed inputs must agree with that original PD peer -count; a known inconsistent consumer is rejected instead of silently changing -its Pod template during import. +Existing Store and Server gRPC seed inputs must be literal values matching the +original PD peer list. Import does not validate Hubble, Vermeer, Server's REST +endpoint or Store's init script against their live templates. Review the full +Pod-template diff before adoption: pre-existing drift in these consumers can +cause a rollout, including a Hubble restart under its `Recreate` strategy. Recorded bootstrap counts are initialization inputs, not the current voting configuration or a list of all admitted Stores. A new Store outside the @@ -276,13 +278,17 @@ growth/shrink and Store shrink until the documented database operation and resource steps are complete. Matching live replicas alone remains insufficient proof of completion. -Keep the feature enabled once a record exists. An online render rejects pins -that conflict with the record, or disabling it while the record remains. -Bootstrap records are kept on uninstall like data PVCs. An install that finds -an existing record or live PD is rejected rather than misrepresenting recovery -as a fresh cluster. Recovery with kept data requires a separate review of the -actual membership, original bootstrap inputs and volume identities; this -feature does not add an automatic restore mode. +Keep the feature enabled once a record exists. With bootstrap enabled, online +renders reject pins that conflict with the record, and installs reject an +existing record or live PD. Online upgrades also reject disabling bootstrap +while the live PD StatefulSet carries the retained-bootstrap marker. +Bootstrap records are kept on uninstall like data PVCs. With bootstrap disabled, +the chart does not read ConfigMaps, preserving the default deployment's RBAC +requirements. After uninstall removes the PD StatefulSet, a disabled reinstall +cannot detect an orphaned bootstrap record and derives startup inputs from the +requested replicas. Do not use that path with retained data PVCs. Recovery +requires reviewing actual membership, original bootstrap inputs and volume +identities; this feature does not add an automatic restore mode. A controlled rollback must retain the same bootstrap record and use a chart version that understands it. Rolling back to a chart or release configuration that predates stable bootstrap can rederive Pod startup inputs from replica @@ -303,7 +309,10 @@ install semantics cannot safely infer an existing topology during replica changes; unattended GitOps topology reconciliation is not supported by this feature. Suspend competing reconciliation during controlled resource steps. Static bootstrap seed majority checks are conservative and are not a replacement for -an authenticated read of current PD membership and quorum health. For example, +an authenticated read of current PD membership and quorum health. Shrinking +below the recorded seed majority is unsupported: with three recorded PDs and +one remaining PD, every restarted Store still waits for two seeds and times out. +Even when the remaining PD count meets that threshold, for example, after an original five-PD cluster shrinks to three, Store startup still requires three of the five recorded seeds. If one of the remaining PDs then fails, current PD quorum may be healthy while a newly created Store remains blocked diff --git a/helm/hugegraph/tests/bootstrap_custom_legacy_test.yaml b/helm/hugegraph/tests/bootstrap_custom_legacy_test.yaml index b352f59e90..06394df39f 100644 --- a/helm/hugegraph/tests/bootstrap_custom_legacy_test.yaml +++ b/helm/hugegraph/tests/bootstrap_custom_legacy_test.yaml @@ -78,7 +78,7 @@ kubernetesProvider: - name: HG_PD_RAFT_PEERS_LIST value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 - name: HG_PD_INITIAL_STORE_LIST - value: custom-store:8500 + value: custom-store-0:8500,custom-store-1:8500,custom-store-2:8500 - name: HG_PD_INITIAL_STORE_COUNT value: '3' - apiVersion: apps/v1 @@ -113,4 +113,4 @@ tests: template: store-statefulset.yaml asserts: - failedTemplate: - errorPattern: bootstrap import only supports the chart + errorPattern: bootstrap only supports the chart diff --git a/helm/hugegraph/tests/bootstrap_disabled_reinstall_test.yaml b/helm/hugegraph/tests/bootstrap_disabled_reinstall_test.yaml new file mode 100644 index 0000000000..3de92472b0 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_disabled_reinstall_test.yaml @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Bootstrap reinstall checks depend on enabled mode +templates: +- pd-statefulset.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: false + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 5 + pd.pdb.minAvailable: 3 +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' +tests: +- it: leaves orphaned history unchecked when bootstrap is disabled + asserts: + - equal: + path: spec.replicas + value: 5 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610,cluster-pd-3.cluster-pd.NAMESPACE.svc:8610,cluster-pd-4.cluster-pd.NAMESPACE.svc:8610 +- it: refuses an orphaned record when bootstrap is enabled + set: + bootstrap.enabled: true + asserts: + - failedTemplate: + errorPattern: bootstrap history already exists on an install diff --git a/helm/hugegraph/tests/bootstrap_fresh_test.yaml b/helm/hugegraph/tests/bootstrap_fresh_test.yaml index 3026a8755e..8a5a0d2e6e 100644 --- a/helm/hugegraph/tests/bootstrap_fresh_test.yaml +++ b/helm/hugegraph/tests/bootstrap_fresh_test.yaml @@ -91,7 +91,7 @@ tests: path: data.storeReplicas value: '3' - equal: - path: initialStoreCount + path: data.initialStoreCount value: '3' - equal: path: immutable @@ -196,6 +196,7 @@ tests: store.replicas: 3 store.pdb.minAvailable: 2 bootstrap.storeReplicas: 1 + bootstrap.initialStoreCount: 1 asserts: - failedTemplate: errorPattern: bootstrap.storeReplicas must equal store.replicas on a fresh install diff --git a/helm/hugegraph/tests/bootstrap_legacy_test.yaml b/helm/hugegraph/tests/bootstrap_legacy_test.yaml index 64c040e4d7..6424849fa2 100644 --- a/helm/hugegraph/tests/bootstrap_legacy_test.yaml +++ b/helm/hugegraph/tests/bootstrap_legacy_test.yaml @@ -101,6 +101,9 @@ kubernetesProvider: spec: containers: - name: store + env: + - name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 ports: - name: grpc containerPort: '8500' From 53a469ef2cb02a57b261a93a24573f2eedde7357 Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 30 Oct 2026 01:09:30 +0800 Subject: [PATCH 8/8] fix(helm): improve bootstrap review diagnostics --- helm/hugegraph/docs/operations.md | 6 ++++-- helm/test_helm_bootstrap.py | 13 ++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/helm/hugegraph/docs/operations.md b/helm/hugegraph/docs/operations.md index 297e918b23..bd27360e8a 100644 --- a/helm/hugegraph/docs/operations.md +++ b/helm/hugegraph/docs/operations.md @@ -258,8 +258,10 @@ the live PD's `HG_PD_RAFT_PEERS_LIST`, `HG_PD_INITIAL_STORE_LIST` and `HG_PD_INITIAL_STORE_COUNT`. It does not infer history from live replica counts, which a controlled operation may already have changed. Import supports the chart's contiguous ordinal endpoints; customized endpoint lists require a -separate reviewed migration. Import preserves a Store allowlist of four and a -startup threshold of three as distinct inputs. No bootstrap checksum is added +separate reviewed migration. Import preserves the actual Store allowlist length from +`HG_PD_INITIAL_STORE_LIST` and the startup threshold from +`HG_PD_INITIAL_STORE_COUNT` as separate inputs; neither is inferred from +current replica counts. No bootstrap checksum is added to Pod templates; adopting an unchanged chart-managed topology preserves these inputs. An incomplete live environment is rejected even when all value pins are given; pins cannot reconstruct unknown online history. diff --git a/helm/test_helm_bootstrap.py b/helm/test_helm_bootstrap.py index 668fdcc26d..19fc2e95b7 100644 --- a/helm/test_helm_bootstrap.py +++ b/helm/test_helm_bootstrap.py @@ -51,7 +51,18 @@ def render(self, pd=3, stores=3, initial_pd=3, initial_stores=3, initial_count=3 command.append("--is-upgrade") for key, value in values.items(): command.extend(["--set", f"{key}={value}"]) - result = subprocess.run(command, check=True, capture_output=True, text=True) + try: + result = subprocess.run( + command, + check=True, + capture_output=True, + text=True, + timeout=300, + ) + except subprocess.CalledProcessError as error: + self.fail(f"helm template failed (exit {error.returncode}):\n{error.stderr}") + except subprocess.TimeoutExpired as error: + self.fail(f"helm template timed out after {error.timeout}s") documents = {} for document in result.stdout.split("---\n"): source = re.search(r"^# Source: [^\n]+/templates/([^\n]+)$", document, re.M)