feat(helm): retain cluster bootstrap topology

.github/actions/helm-checks/action.yml

@@ -59,6 +59,10 @@ runs:
     shell: 'bash -e {0}'
     run: python3 -m unittest discover -s helm -p 'test_helm_metadata_cluster.py' -v
 
+  - name: Stable bootstrap topology regression
+    shell: 'bash -e {0}'
+    run: python3 -m unittest discover -s helm -p 'test_helm_bootstrap.py' -v
+
   - name: helm unittest
     shell: 'bash -e {0}'
     run: |

helm/hugegraph/README.md

@@ -34,6 +34,12 @@ use `latest` images; pin component `image.digest` values when needed.
 The source installation and configuration examples below assume a repository
 checkout and commands run from its root.
 
+The [stable bootstrap configuration](docs/operations.md#stable-bootstrap-topology)
+separates initialization inputs from node replicas. It is disabled by default
+and requires PD images with separate bootstrap and runtime quorum checks before
+using it for node operations. It does not implement safe node admission or
+retirement by itself.
+
 ## Install from source and open Hubble
 
 ```bash

helm/hugegraph/docs/operations.md

@@ -226,6 +226,101 @@ Server replicas can be changed routinely; CPU-utilization HPA requires positive
 CPU requests. Store growth still needs volume and shard planning. Use chart
 updates for Store scale changes so Hubble's endpoint list is updated too.
 
+## Stable bootstrap topology
+
+This feature is disabled by default. Enable it with `bootstrap.enabled=true`. It separates
+initial PD peers, initial Store allowlist and startup Store threshold from
+current `pd.replicas` and `store.replicas`. It does not perform database member
+changes, approve Pending Stores or prove that a Store is safe to delete.
+
+Before using it for node operations, deploy a PD image that treats
+`pd.initial-store-count` as an admission threshold while no shard groups exist
+and checks existing groups against their known Store voting majorities. Older PD images can still apply the
+initial count during normal operation even when a runtime quorum is healthy.
+Pin the compatible PD image version or digest; this chart cannot infer that
+capability from an image tag. See the [PD configuration guide](../../../hugegraph-pd/docs/configuration.md).
+
+On a fresh install, empty `bootstrap.pdReplicas`, `bootstrap.storeReplicas` and
+`bootstrap.initialStoreCount` inherit the requested PD count, Store count and
+Store count, respectively. A fresh five-PD installation therefore starts with
+five initial voters, not three. The chart records these counts in a kept,
+immutable `<fullname>-bootstrap` ConfigMap (with a hash for long names). PD startup inputs, client PD
+seeds, Store startup seed wait, and an unspecified initial shard count use the
+recorded topology. PDB validation still uses the current desired PD majority.
+Hubble's Store REST allowlist can change with Store count as before.
+PD changes between three and five replicas preserve these Pod startup inputs.
+Crossing the one-PD boundary can still change the inherited liveness path;
+use a compatible PD image and explicitly pin `pd.livenessPath` before such a
+controlled operation when the existing Pod template must remain unchanged.
+
+For an existing release without this record, the first online upgrade imports
+the live PD's `HG_PD_RAFT_PEERS_LIST`, `HG_PD_INITIAL_STORE_LIST` and
+`HG_PD_INITIAL_STORE_COUNT`. It does not infer history from live replica counts,
+which a controlled operation may already have changed. Import supports the
+chart's contiguous ordinal endpoints; customized endpoint lists require a
+separate reviewed migration. Import preserves the actual Store allowlist length from
+`HG_PD_INITIAL_STORE_LIST` and the startup threshold from
+`HG_PD_INITIAL_STORE_COUNT` as separate inputs; neither is inferred from
+current replica counts. No bootstrap checksum is added
+to Pod templates; adopting an unchanged chart-managed topology preserves these inputs.
+An incomplete live environment is rejected even when all value pins are given;
+pins cannot reconstruct unknown online history.
+Existing Store and Server gRPC seed inputs must be literal values matching the
+original PD peer list. Import does not validate Hubble, Vermeer, Server's REST
+endpoint or Store's init script against their live templates. Review the full
+Pod-template diff before adoption: pre-existing drift in these consumers can
+cause a rollout, including a Hubble restart under its `Recreate` strategy.
+
+Recorded bootstrap counts are initialization inputs, not the current voting
+configuration or a list of all admitted Stores. A new Store outside the
+recorded allowlist remains Pending until a controlled management action admits
+it after identity, disk and capacity checks. Plan migration and verify actual
+Raft members before changing Pod counts. Existing live guards still reject PD
+growth/shrink and Store shrink until the documented database operation and
+resource steps are complete. Matching live replicas alone remains insufficient
+proof of completion.
+
+Keep the feature enabled once a record exists. With bootstrap enabled, online
+renders reject pins that conflict with the record, and installs reject an
+existing record or live PD. Online upgrades also reject disabling bootstrap
+while the live PD StatefulSet carries the retained-bootstrap marker.
+Bootstrap records are kept on uninstall like data PVCs. With bootstrap disabled,
+the chart does not read ConfigMaps, preserving the default deployment's RBAC
+requirements. After uninstall removes the PD StatefulSet, a disabled reinstall
+cannot detect an orphaned bootstrap record and derives startup inputs from the
+requested replicas. Do not use that path with retained data PVCs. Recovery
+requires reviewing actual membership, original bootstrap inputs and volume
+identities; this feature does not add an automatic restore mode.
+A controlled rollback must retain the same bootstrap record and use a chart
+version that understands it. Rolling back to a chart or release configuration
+that predates stable bootstrap can rederive Pod startup inputs from replica
+counts; it is not a supported automatic rollback procedure.
+
+Offline upgrades cannot discover this history. Render them with `--is-upgrade`
+and pin all three counts from the retained record, for example:
+
+```bash
+helm template hugegraph ./helm/hugegraph -n hugegraph --is-upgrade \
+  --set bootstrap.enabled=true \
+  --set bootstrap.pdReplicas=3 --set bootstrap.storeReplicas=3 \
+  --set bootstrap.initialStoreCount=3
+```
+
+An offline upgrade without all three pins fails. A renderer that always uses
+install semantics cannot safely infer an existing topology during replica
+changes; unattended GitOps topology reconciliation is not supported by this feature.
+Suspend competing reconciliation during controlled resource steps. Static
+bootstrap seed majority checks are conservative and are not a replacement for
+an authenticated read of current PD membership and quorum health. Shrinking
+below the recorded seed majority is unsupported: with three recorded PDs and
+one remaining PD, every restarted Store still waits for two seeds and times out.
+Even when the remaining PD count meets that threshold, for example,
+after an original five-PD cluster shrinks to three, Store startup still requires
+three of the five recorded seeds. If one of the remaining PDs then fails,
+current PD quorum may be healthy while a newly created Store remains blocked
+by this static startup check. Dynamic membership-aware startup is a separate
+follow-up; unchanged Pod templates do not prove every combined failure case.
+
 ## Rolling Store updates and recovery
 
 Use `store.updateStrategy.type=OnDelete` for a controlled rollout. Replace one

helm/hugegraph/templates/_bootstrap.tpl

@@ -0,0 +1,170 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+{{/*
+Retain only initialization counts, not a second membership view. Explicit pins may reproduce history offline but cannot change
+an online record. A legacy release is imported from its actual PD environment,
+never from StatefulSet replicas that a controlled operation may already change.
+*/}}
+{{- define "hugegraph.bootstrap.name" -}}
+{{- $name := include "hugegraph.fullname" . -}}
+{{- if gt (len $name) 53 -}}
+{{- printf "%s-%s-bootstrap" ($name | trunc 44 | trimSuffix "-") ($name | sha256sum | trunc 8) -}}
+{{- else -}}
+{{- printf "%s-bootstrap" $name -}}
+{{- end -}}
+{{- end }}
+
+{{- define "hugegraph.bootstrap.resolve" -}}
+{{- $config := get .Values "bootstrap" | default dict -}}
+{{- $enabled := get $config "enabled" | default false -}}
+{{- if not $enabled -}}
+{{- $resolved := dict "pdReplicas" (int .Values.pd.replicas) "storeReplicas" (int .Values.store.replicas) "initialStoreCount" (int .Values.store.replicas) -}}
+{{- $_ := set .Values "__hugegraphBootstrap" $resolved -}}
+{{- if .Release.IsUpgrade -}}
+{{- $live := lookup "apps/v1" "StatefulSet" .Release.Namespace (include "hugegraph.pd.name" .) -}}
+{{- if and $live (eq (dig "metadata" "annotations" "hugegraph.apache.org/bootstrap-retained" "" $live) "true") -}}
+{{- fail "bootstrap.enabled cannot be disabled while the live PD StatefulSet records retained bootstrap topology" -}}
+{{- end -}}
+{{- end -}}
+{{- else -}}
+{{- $record := lookup "v1" "ConfigMap" .Release.Namespace (include "hugegraph.bootstrap.name" .) -}}
+{{- $history := dict -}}
+{{- $legacyEnv := dict -}}
+{{- if $record -}}
+{{- if not .Release.IsUpgrade -}}
+{{- fail "bootstrap history already exists on an install; use a reviewed recovery procedure rather than treating kept data as a fresh cluster" -}}
+{{- end -}}
+{{- $owner := dig "metadata" "labels" "app.kubernetes.io/instance" "" $record -}}
+{{- if not $owner -}}
+{{- fail "bootstrap ConfigMap is missing the app.kubernetes.io/instance owner label; inspect the retained initialization topology before continuing" -}}
+{{- end -}}
+{{- if ne $owner .Release.Name -}}
+{{- fail "bootstrap ConfigMap belongs to another release; inspect the retained initialization topology" -}}
+{{- end -}}
+{{- $data := get $record "data" | default dict -}}
+{{- if ne (get $data "schemaVersion" | default "") "1" -}}
+{{- fail "bootstrap ConfigMap has an unsupported schemaVersion; inspect it before continuing" -}}
+{{- end -}}
+{{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}}
+{{- $_ := set $history $key (get $data $key | default "") -}}
+{{- end -}}
+{{- else -}}
+{{- $live := lookup "apps/v1" "StatefulSet" .Release.Namespace (include "hugegraph.pd.name" .) -}}
+{{- if $live -}}
+{{- if not .Release.IsUpgrade -}}
+{{- fail "bootstrap import requires an upgrade of the existing release; inspect live PD and retained data before recovery" -}}
+{{- end -}}
+{{- $owner := dig "metadata" "labels" "app.kubernetes.io/instance" "" $live -}}
+{{- if not $owner -}}
+{{- fail "bootstrap import requires the existing PD StatefulSet to carry the app.kubernetes.io/instance owner label" -}}
+{{- end -}}
+{{- if ne $owner .Release.Name -}}
+{{- fail "bootstrap import found a PD StatefulSet owned by another release" -}}
+{{- end -}}
+{{- $env := dict -}}
+{{- range $container := dig "spec" "template" "spec" "containers" list $live -}}
+{{- if eq (get $container "name" | default "") "pd" -}}
+{{- range $item := get $container "env" | default list -}}
+{{- $_ := set $env (get $item "name") (get $item "value" | default "") -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- $pdPeers := get $env "HG_PD_RAFT_PEERS_LIST" | default "" -}}
+{{- $stores := get $env "HG_PD_INITIAL_STORE_LIST" | default "" -}}
+{{- $count := get $env "HG_PD_INITIAL_STORE_COUNT" | default "" -}}
+{{- if or (not $pdPeers) (not $stores) (not $count) -}}
+{{- fail "bootstrap import requires the existing PD peer list, initial Store list and initial Store count; inspect the live environment instead of guessing from replicas" -}}
+{{- end -}}
+{{- $_ := set $history "pdReplicas" (len (splitList "," $pdPeers) | toString) -}}
+{{- $_ := set $history "storeReplicas" (len (splitList "," $stores) | toString) -}}
+{{- $_ := set $history "initialStoreCount" $count -}}
+{{- $legacyEnv = $env -}}
+{{- else if .Release.IsUpgrade -}}
+{{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}}
+{{- if eq (include "hugegraph.optionalScalar" (get $config $key)) "" -}}
+{{- fail "bootstrap.pdReplicas, bootstrap.storeReplicas and bootstrap.initialStoreCount must all be pinned for an upgrade without a live bootstrap record" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- if not (empty $history) -}}
+{{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}}
+{{- if not (regexMatch "^[1-9][0-9]?$" (get $history $key | toString)) -}}
+{{- fail (printf "bootstrap record is missing a valid %s between 1 and 99; inspect it before continuing" $key) -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- $resolved := dict -}}
+{{- $defaults := dict "pdReplicas" .Values.pd.replicas "storeReplicas" .Values.store.replicas "initialStoreCount" .Values.store.replicas -}}
+{{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}}
+{{- $pin := include "hugegraph.optionalScalar" (get $config $key) -}}
+{{- $old := get $history $key | default "" | toString -}}
+{{- if and (ne $old "") (ne $pin "") (ne $pin $old) -}}
+{{- fail (printf "bootstrap.%s cannot change from %s to %s on an initialized release" $key $old $pin) -}}
+{{- end -}}
+{{- $value := ternary $pin $old (ne $pin "") -}}
+{{- if eq $value "" -}}{{- $value = get $defaults $key | toString -}}{{- end -}}
+{{- if not (regexMatch "^[1-9][0-9]?$" $value) -}}
+{{- fail (printf "bootstrap.%s must be an integer between 1 and 99" $key) -}}
+{{- end -}}
+{{- $_ := set $resolved $key (int $value) -}}
+{{- end -}}
+{{- if gt (int $resolved.initialStoreCount) (int $resolved.storeReplicas) -}}
+{{- fail "bootstrap.initialStoreCount cannot exceed bootstrap.storeReplicas: the initial allowlist could not satisfy startup" -}}
+{{- end -}}
+{{- if and (not .Release.IsUpgrade) (empty $history) -}}
+{{- if ne (int $resolved.pdReplicas) (int .Values.pd.replicas) -}}
+{{- fail "bootstrap.pdReplicas must equal pd.replicas on a fresh install; use --is-upgrade with pinned history for offline upgrades" -}}
+{{- end -}}
+{{- if ne (int $resolved.storeReplicas) (int .Values.store.replicas) -}}
+{{- fail "bootstrap.storeReplicas must equal store.replicas on a fresh install; use --is-upgrade with pinned history for offline upgrades" -}}
+{{- end -}}
+{{- end -}}
+{{- $_ := set .Values "__hugegraphBootstrap" $resolved -}}
+{{- if not (empty $legacyEnv) -}}
+{{- if or (ne (get $legacyEnv "HG_PD_RAFT_PEERS_LIST") (include "hugegraph.pd.raftPeersList" .)) (ne (get $legacyEnv "HG_PD_INITIAL_STORE_LIST") (include "hugegraph.store.initialStoreList" .)) -}}
+{{- fail "bootstrap only supports the chart's contiguous ordinal endpoint lists; inspect customized live endpoints before migration" -}}
+{{- end -}}
+{{- $seed := include "hugegraph.pd.grpcPeersList" . -}}
+{{- range $component := list "store" "server" -}}
+{{- $kind := ternary "StatefulSet" "Deployment" (eq $component "store") -}}
+{{- $name := ternary (include "hugegraph.store.name" $) (include "hugegraph.server.name" $) (eq $component "store") -}}
+{{- $consumer := lookup "apps/v1" $kind $.Release.Namespace $name -}}
+{{- $key := ternary "HG_STORE_PD_ADDRESS" "HG_SERVER_PD_PEERS" (eq $component "store") -}}
+{{- if $consumer -}}
+{{- $found := false -}}
+{{- range $container := dig "spec" "template" "spec" "containers" list $consumer -}}
+{{- if eq (get $container "name" | default "") $component -}}
+{{- range $item := get $container "env" | default list -}}
+{{- if eq (get $item "name" | default "") $key -}}
+{{- $found = true -}}
+{{- if ne (get $item "value" | default "") $seed -}}
+{{- fail (printf "bootstrap import found %s seeds that differ from the PD initial peers; reconcile the existing templates before migration" $component) -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- if not $found -}}
+{{- fail (printf "bootstrap import requires the existing %s template to expose %s as a literal value" $component $key) -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end }}