diff --git a/.github/actions/helm-checks/action.yml b/.github/actions/helm-checks/action.yml index f669ec060b..089b59f518 100644 --- a/.github/actions/helm-checks/action.yml +++ b/.github/actions/helm-checks/action.yml @@ -59,6 +59,10 @@ runs: shell: 'bash -e {0}' run: python3 -m unittest discover -s helm -p 'test_helm_metadata_cluster.py' -v + - name: Stable bootstrap topology regression + shell: 'bash -e {0}' + run: python3 -m unittest discover -s helm -p 'test_helm_bootstrap.py' -v + - name: helm unittest shell: 'bash -e {0}' run: | diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 7942cac8aa..db5242770f 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -34,6 +34,12 @@ use `latest` images; pin component `image.digest` values when needed. The source installation and configuration examples below assume a repository checkout and commands run from its root. +The [stable bootstrap configuration](docs/operations.md#stable-bootstrap-topology) +separates initialization inputs from node replicas. It is disabled by default +and requires PD images with separate bootstrap and runtime quorum checks before +using it for node operations. It does not implement safe node admission or +retirement by itself. + ## Install from source and open Hubble ```bash diff --git a/helm/hugegraph/docs/operations.md b/helm/hugegraph/docs/operations.md index e9cb2f516e..bd27360e8a 100644 --- a/helm/hugegraph/docs/operations.md +++ b/helm/hugegraph/docs/operations.md @@ -226,6 +226,101 @@ Server replicas can be changed routinely; CPU-utilization HPA requires positive CPU requests. Store growth still needs volume and shard planning. Use chart updates for Store scale changes so Hubble's endpoint list is updated too. +## Stable bootstrap topology + +This feature is disabled by default. Enable it with `bootstrap.enabled=true`. It separates +initial PD peers, initial Store allowlist and startup Store threshold from +current `pd.replicas` and `store.replicas`. It does not perform database member +changes, approve Pending Stores or prove that a Store is safe to delete. + +Before using it for node operations, deploy a PD image that treats +`pd.initial-store-count` as an admission threshold while no shard groups exist +and checks existing groups against their known Store voting majorities. Older PD images can still apply the +initial count during normal operation even when a runtime quorum is healthy. +Pin the compatible PD image version or digest; this chart cannot infer that +capability from an image tag. See the [PD configuration guide](../../../hugegraph-pd/docs/configuration.md). + +On a fresh install, empty `bootstrap.pdReplicas`, `bootstrap.storeReplicas` and +`bootstrap.initialStoreCount` inherit the requested PD count, Store count and +Store count, respectively. A fresh five-PD installation therefore starts with +five initial voters, not three. The chart records these counts in a kept, +immutable `-bootstrap` ConfigMap (with a hash for long names). PD startup inputs, client PD +seeds, Store startup seed wait, and an unspecified initial shard count use the +recorded topology. PDB validation still uses the current desired PD majority. +Hubble's Store REST allowlist can change with Store count as before. +PD changes between three and five replicas preserve these Pod startup inputs. +Crossing the one-PD boundary can still change the inherited liveness path; +use a compatible PD image and explicitly pin `pd.livenessPath` before such a +controlled operation when the existing Pod template must remain unchanged. + +For an existing release without this record, the first online upgrade imports +the live PD's `HG_PD_RAFT_PEERS_LIST`, `HG_PD_INITIAL_STORE_LIST` and +`HG_PD_INITIAL_STORE_COUNT`. It does not infer history from live replica counts, +which a controlled operation may already have changed. Import supports the +chart's contiguous ordinal endpoints; customized endpoint lists require a +separate reviewed migration. Import preserves the actual Store allowlist length from +`HG_PD_INITIAL_STORE_LIST` and the startup threshold from +`HG_PD_INITIAL_STORE_COUNT` as separate inputs; neither is inferred from +current replica counts. No bootstrap checksum is added +to Pod templates; adopting an unchanged chart-managed topology preserves these inputs. +An incomplete live environment is rejected even when all value pins are given; +pins cannot reconstruct unknown online history. +Existing Store and Server gRPC seed inputs must be literal values matching the +original PD peer list. Import does not validate Hubble, Vermeer, Server's REST +endpoint or Store's init script against their live templates. Review the full +Pod-template diff before adoption: pre-existing drift in these consumers can +cause a rollout, including a Hubble restart under its `Recreate` strategy. + +Recorded bootstrap counts are initialization inputs, not the current voting +configuration or a list of all admitted Stores. A new Store outside the +recorded allowlist remains Pending until a controlled management action admits +it after identity, disk and capacity checks. Plan migration and verify actual +Raft members before changing Pod counts. Existing live guards still reject PD +growth/shrink and Store shrink until the documented database operation and +resource steps are complete. Matching live replicas alone remains insufficient +proof of completion. + +Keep the feature enabled once a record exists. With bootstrap enabled, online +renders reject pins that conflict with the record, and installs reject an +existing record or live PD. Online upgrades also reject disabling bootstrap +while the live PD StatefulSet carries the retained-bootstrap marker. +Bootstrap records are kept on uninstall like data PVCs. With bootstrap disabled, +the chart does not read ConfigMaps, preserving the default deployment's RBAC +requirements. After uninstall removes the PD StatefulSet, a disabled reinstall +cannot detect an orphaned bootstrap record and derives startup inputs from the +requested replicas. Do not use that path with retained data PVCs. Recovery +requires reviewing actual membership, original bootstrap inputs and volume +identities; this feature does not add an automatic restore mode. +A controlled rollback must retain the same bootstrap record and use a chart +version that understands it. Rolling back to a chart or release configuration +that predates stable bootstrap can rederive Pod startup inputs from replica +counts; it is not a supported automatic rollback procedure. + +Offline upgrades cannot discover this history. Render them with `--is-upgrade` +and pin all three counts from the retained record, for example: + +```bash +helm template hugegraph ./helm/hugegraph -n hugegraph --is-upgrade \ + --set bootstrap.enabled=true \ + --set bootstrap.pdReplicas=3 --set bootstrap.storeReplicas=3 \ + --set bootstrap.initialStoreCount=3 +``` + +An offline upgrade without all three pins fails. A renderer that always uses +install semantics cannot safely infer an existing topology during replica +changes; unattended GitOps topology reconciliation is not supported by this feature. +Suspend competing reconciliation during controlled resource steps. Static +bootstrap seed majority checks are conservative and are not a replacement for +an authenticated read of current PD membership and quorum health. Shrinking +below the recorded seed majority is unsupported: with three recorded PDs and +one remaining PD, every restarted Store still waits for two seeds and times out. +Even when the remaining PD count meets that threshold, for example, +after an original five-PD cluster shrinks to three, Store startup still requires +three of the five recorded seeds. If one of the remaining PDs then fails, +current PD quorum may be healthy while a newly created Store remains blocked +by this static startup check. Dynamic membership-aware startup is a separate +follow-up; unchanged Pod templates do not prove every combined failure case. + ## Rolling Store updates and recovery Use `store.updateStrategy.type=OnDelete` for a controlled rollout. Replace one diff --git a/helm/hugegraph/templates/_bootstrap.tpl b/helm/hugegraph/templates/_bootstrap.tpl new file mode 100644 index 0000000000..57c3538a45 --- /dev/null +++ b/helm/hugegraph/templates/_bootstrap.tpl @@ -0,0 +1,170 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{/* +Retain only initialization counts, not a second membership view. Explicit pins may reproduce history offline but cannot change +an online record. A legacy release is imported from its actual PD environment, +never from StatefulSet replicas that a controlled operation may already change. +*/}} +{{- define "hugegraph.bootstrap.name" -}} +{{- $name := include "hugegraph.fullname" . -}} +{{- if gt (len $name) 53 -}} +{{- printf "%s-%s-bootstrap" ($name | trunc 44 | trimSuffix "-") ($name | sha256sum | trunc 8) -}} +{{- else -}} +{{- printf "%s-bootstrap" $name -}} +{{- end -}} +{{- end }} + +{{- define "hugegraph.bootstrap.resolve" -}} +{{- $config := get .Values "bootstrap" | default dict -}} +{{- $enabled := get $config "enabled" | default false -}} +{{- if not $enabled -}} +{{- $resolved := dict "pdReplicas" (int .Values.pd.replicas) "storeReplicas" (int .Values.store.replicas) "initialStoreCount" (int .Values.store.replicas) -}} +{{- $_ := set .Values "__hugegraphBootstrap" $resolved -}} +{{- if .Release.IsUpgrade -}} +{{- $live := lookup "apps/v1" "StatefulSet" .Release.Namespace (include "hugegraph.pd.name" .) -}} +{{- if and $live (eq (dig "metadata" "annotations" "hugegraph.apache.org/bootstrap-retained" "" $live) "true") -}} +{{- fail "bootstrap.enabled cannot be disabled while the live PD StatefulSet records retained bootstrap topology" -}} +{{- end -}} +{{- end -}} +{{- else -}} +{{- $record := lookup "v1" "ConfigMap" .Release.Namespace (include "hugegraph.bootstrap.name" .) -}} +{{- $history := dict -}} +{{- $legacyEnv := dict -}} +{{- if $record -}} +{{- if not .Release.IsUpgrade -}} +{{- fail "bootstrap history already exists on an install; use a reviewed recovery procedure rather than treating kept data as a fresh cluster" -}} +{{- end -}} +{{- $owner := dig "metadata" "labels" "app.kubernetes.io/instance" "" $record -}} +{{- if not $owner -}} +{{- fail "bootstrap ConfigMap is missing the app.kubernetes.io/instance owner label; inspect the retained initialization topology before continuing" -}} +{{- end -}} +{{- if ne $owner .Release.Name -}} +{{- fail "bootstrap ConfigMap belongs to another release; inspect the retained initialization topology" -}} +{{- end -}} +{{- $data := get $record "data" | default dict -}} +{{- if ne (get $data "schemaVersion" | default "") "1" -}} +{{- fail "bootstrap ConfigMap has an unsupported schemaVersion; inspect it before continuing" -}} +{{- end -}} +{{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}} +{{- $_ := set $history $key (get $data $key | default "") -}} +{{- end -}} +{{- else -}} +{{- $live := lookup "apps/v1" "StatefulSet" .Release.Namespace (include "hugegraph.pd.name" .) -}} +{{- if $live -}} +{{- if not .Release.IsUpgrade -}} +{{- fail "bootstrap import requires an upgrade of the existing release; inspect live PD and retained data before recovery" -}} +{{- end -}} +{{- $owner := dig "metadata" "labels" "app.kubernetes.io/instance" "" $live -}} +{{- if not $owner -}} +{{- fail "bootstrap import requires the existing PD StatefulSet to carry the app.kubernetes.io/instance owner label" -}} +{{- end -}} +{{- if ne $owner .Release.Name -}} +{{- fail "bootstrap import found a PD StatefulSet owned by another release" -}} +{{- end -}} +{{- $env := dict -}} +{{- range $container := dig "spec" "template" "spec" "containers" list $live -}} +{{- if eq (get $container "name" | default "") "pd" -}} +{{- range $item := get $container "env" | default list -}} +{{- $_ := set $env (get $item "name") (get $item "value" | default "") -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- $pdPeers := get $env "HG_PD_RAFT_PEERS_LIST" | default "" -}} +{{- $stores := get $env "HG_PD_INITIAL_STORE_LIST" | default "" -}} +{{- $count := get $env "HG_PD_INITIAL_STORE_COUNT" | default "" -}} +{{- if or (not $pdPeers) (not $stores) (not $count) -}} +{{- fail "bootstrap import requires the existing PD peer list, initial Store list and initial Store count; inspect the live environment instead of guessing from replicas" -}} +{{- end -}} +{{- $_ := set $history "pdReplicas" (len (splitList "," $pdPeers) | toString) -}} +{{- $_ := set $history "storeReplicas" (len (splitList "," $stores) | toString) -}} +{{- $_ := set $history "initialStoreCount" $count -}} +{{- $legacyEnv = $env -}} +{{- else if .Release.IsUpgrade -}} +{{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}} +{{- if eq (include "hugegraph.optionalScalar" (get $config $key)) "" -}} +{{- fail "bootstrap.pdReplicas, bootstrap.storeReplicas and bootstrap.initialStoreCount must all be pinned for an upgrade without a live bootstrap record" -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- if not (empty $history) -}} +{{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}} +{{- if not (regexMatch "^[1-9][0-9]?$" (get $history $key | toString)) -}} +{{- fail (printf "bootstrap record is missing a valid %s between 1 and 99; inspect it before continuing" $key) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- $resolved := dict -}} +{{- $defaults := dict "pdReplicas" .Values.pd.replicas "storeReplicas" .Values.store.replicas "initialStoreCount" .Values.store.replicas -}} +{{- range $key := list "pdReplicas" "storeReplicas" "initialStoreCount" -}} +{{- $pin := include "hugegraph.optionalScalar" (get $config $key) -}} +{{- $old := get $history $key | default "" | toString -}} +{{- if and (ne $old "") (ne $pin "") (ne $pin $old) -}} +{{- fail (printf "bootstrap.%s cannot change from %s to %s on an initialized release" $key $old $pin) -}} +{{- end -}} +{{- $value := ternary $pin $old (ne $pin "") -}} +{{- if eq $value "" -}}{{- $value = get $defaults $key | toString -}}{{- end -}} +{{- if not (regexMatch "^[1-9][0-9]?$" $value) -}} +{{- fail (printf "bootstrap.%s must be an integer between 1 and 99" $key) -}} +{{- end -}} +{{- $_ := set $resolved $key (int $value) -}} +{{- end -}} +{{- if gt (int $resolved.initialStoreCount) (int $resolved.storeReplicas) -}} +{{- fail "bootstrap.initialStoreCount cannot exceed bootstrap.storeReplicas: the initial allowlist could not satisfy startup" -}} +{{- end -}} +{{- if and (not .Release.IsUpgrade) (empty $history) -}} +{{- if ne (int $resolved.pdReplicas) (int .Values.pd.replicas) -}} +{{- fail "bootstrap.pdReplicas must equal pd.replicas on a fresh install; use --is-upgrade with pinned history for offline upgrades" -}} +{{- end -}} +{{- if ne (int $resolved.storeReplicas) (int .Values.store.replicas) -}} +{{- fail "bootstrap.storeReplicas must equal store.replicas on a fresh install; use --is-upgrade with pinned history for offline upgrades" -}} +{{- end -}} +{{- end -}} +{{- $_ := set .Values "__hugegraphBootstrap" $resolved -}} +{{- if not (empty $legacyEnv) -}} +{{- if or (ne (get $legacyEnv "HG_PD_RAFT_PEERS_LIST") (include "hugegraph.pd.raftPeersList" .)) (ne (get $legacyEnv "HG_PD_INITIAL_STORE_LIST") (include "hugegraph.store.initialStoreList" .)) -}} +{{- fail "bootstrap only supports the chart's contiguous ordinal endpoint lists; inspect customized live endpoints before migration" -}} +{{- end -}} +{{- $seed := include "hugegraph.pd.grpcPeersList" . -}} +{{- range $component := list "store" "server" -}} +{{- $kind := ternary "StatefulSet" "Deployment" (eq $component "store") -}} +{{- $name := ternary (include "hugegraph.store.name" $) (include "hugegraph.server.name" $) (eq $component "store") -}} +{{- $consumer := lookup "apps/v1" $kind $.Release.Namespace $name -}} +{{- $key := ternary "HG_STORE_PD_ADDRESS" "HG_SERVER_PD_PEERS" (eq $component "store") -}} +{{- if $consumer -}} +{{- $found := false -}} +{{- range $container := dig "spec" "template" "spec" "containers" list $consumer -}} +{{- if eq (get $container "name" | default "") $component -}} +{{- range $item := get $container "env" | default list -}} +{{- if eq (get $item "name" | default "") $key -}} +{{- $found = true -}} +{{- if ne (get $item "value" | default "") $seed -}} +{{- fail (printf "bootstrap import found %s seeds that differ from the PD initial peers; reconcile the existing templates before migration" $component) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- if not $found -}} +{{- fail (printf "bootstrap import requires the existing %s template to expose %s as a literal value" $component $key) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end }} diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index f41ccab6e0..2615cbe5e0 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -109,6 +109,7 @@ Fail before dereferencing missing blocks; keep values.yaml the sole defaults sou {{- $budget := add (int $server.storageWaitTimeoutSeconds) 150 -}} {{- $_ := set $startup "failureThreshold" (add (div (add (sub $budget 1) $period) $period) 1) -}} {{- end -}} +{{- include "hugegraph.bootstrap.resolve" . -}} {{- end -}} {{- end }} @@ -439,7 +440,7 @@ Uses short headless DNS (cluster.local optional) resolvable inside the namespace */}} {{- define "hugegraph.pd.raftPeersList" -}} {{- $peers := list -}} -{{- $replicas := int .Values.pd.replicas -}} +{{- $replicas := int .Values.__hugegraphBootstrap.pdReplicas -}} {{- $name := include "hugegraph.pd.name" . -}} {{- $ns := .Release.Namespace -}} {{- $port := int .Values.pd.ports.raft -}} @@ -454,7 +455,7 @@ PD gRPC peers for Store/Server. */}} {{- define "hugegraph.pd.grpcPeersList" -}} {{- $peers := list -}} -{{- $replicas := int .Values.pd.replicas -}} +{{- $replicas := int .Values.__hugegraphBootstrap.pdReplicas -}} {{- $name := include "hugegraph.pd.name" . -}} {{- $ns := .Release.Namespace -}} {{- $port := int .Values.pd.ports.grpc -}} @@ -469,7 +470,7 @@ PD REST endpoints for Server storage-readiness checks. */}} {{- define "hugegraph.pd.restPeersList" -}} {{- $peers := list -}} -{{- $replicas := int .Values.pd.replicas -}} +{{- $replicas := int .Values.__hugegraphBootstrap.pdReplicas -}} {{- $name := include "hugegraph.pd.name" . -}} {{- $ns := .Release.Namespace -}} {{- $port := int .Values.pd.ports.rest -}} @@ -523,7 +524,7 @@ Initial store list for PD bootstrap: store-0.svc.ns.svc:8500,... */}} {{- define "hugegraph.store.initialStoreList" -}} {{- $peers := list -}} -{{- $replicas := int .Values.store.replicas -}} +{{- $replicas := int .Values.__hugegraphBootstrap.storeReplicas -}} {{- $name := include "hugegraph.store.name" . -}} {{- $ns := .Release.Namespace -}} {{- $port := int .Values.store.ports.grpc -}} @@ -592,6 +593,11 @@ Quorum size: floor(replicas/2)+1 {{- add (div (int .Values.pd.replicas) 2) 1 -}} {{- end }} +{{/* The static seed wait is independent of the current voting/PDB majority. */}} +{{- define "hugegraph.pd.bootstrapQuorum" -}} +{{- add (div (int .Values.__hugegraphBootstrap.pdReplicas) 2) 1 -}} +{{- end }} + {{/* Render JAVA_OPTS only when explicitly configured. An empty value preserves the image entrypoint's existing automatic JVM sizing behavior. @@ -679,7 +685,7 @@ well, which is what a leader election waited on. {{- $flags := list -}} {{- $shardCount := include "hugegraph.optionalScalar" (get $partition "defaultShardCount") -}} {{- if eq $shardCount "" -}} -{{- $shardCount = ternary "3" "1" (ge (int .Values.store.replicas) 3) -}} +{{- $shardCount = ternary "3" "1" (ge (int .Values.__hugegraphBootstrap.storeReplicas) 3) -}} {{- end -}} {{- $flags = append $flags (printf "-Dpartition.default-shard-count=%s" $shardCount) -}} {{- $maxShard := include "hugegraph.optionalScalar" (get $partition "storeMaxShardCount") -}} @@ -872,8 +878,8 @@ the chart renders reaches it only as NodeOptions.setInitialConf, which jraft applies when bootstrapping a node that has no configuration of its own. On an initialized group, adding Pods adds non-voting strangers: the extra PD starts, the peer list changes, and the voting configuration does not. PD exposes the -change through RaftEngine.changePeerList, reachable from the PD client API but -from no REST route, so the chart cannot perform it and does not pretend to. +change through RaftEngine.changePeerList and its REST management route, but +the chart does not coordinate node preparation, catch-up and member completion. Growing Store is ordinary scale-out and stays allowed. */}} {{- if and (eq $comp "pd") (gt $liveReplicas 0) (gt $desired $liveReplicas) -}} diff --git a/helm/hugegraph/templates/bootstrap-configmap.yaml b/helm/hugegraph/templates/bootstrap-configmap.yaml new file mode 100644 index 0000000000..3d20a8cd88 --- /dev/null +++ b/helm/hugegraph/templates/bootstrap-configmap.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{ include "hugegraph.resolveValues" . }} +{{- if get (.Values.bootstrap | default dict) "enabled" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hugegraph.bootstrap.name" . }} + annotations: + helm.sh/resource-policy: keep + labels: + {{- include "hugegraph.labels" . | nindent 4 }} +immutable: true +data: + schemaVersion: "1" + pdReplicas: {{ .Values.__hugegraphBootstrap.pdReplicas | quote }} + storeReplicas: {{ .Values.__hugegraphBootstrap.storeReplicas | quote }} + initialStoreCount: {{ .Values.__hugegraphBootstrap.initialStoreCount | quote }} +{{- end }} diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index d1603e319d..cf52177bfa 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -23,6 +23,10 @@ metadata: labels: {{- include "hugegraph.labels" . | nindent 4 }} app.kubernetes.io/component: pd + {{- if get (.Values.bootstrap | default dict) "enabled" }} + annotations: + hugegraph.apache.org/bootstrap-retained: "true" + {{- end }} spec: serviceName: {{ include "hugegraph.pd.name" . }} replicas: {{ .Values.pd.replicas }} @@ -122,7 +126,7 @@ spec: - name: HG_PD_INITIAL_STORE_LIST value: {{ include "hugegraph.store.initialStoreList" . | quote }} - name: HG_PD_INITIAL_STORE_COUNT - value: {{ .Values.store.replicas | quote }} + value: {{ .Values.__hugegraphBootstrap.initialStoreCount | quote }} - name: HG_PD_DATA_PATH value: {{ .Values.pd.dataPath | quote }} - name: HG_PD_AUTH_SECRET_KEY diff --git a/helm/hugegraph/templates/store-statefulset.yaml b/helm/hugegraph/templates/store-statefulset.yaml index e76832df75..0907cf2f2f 100644 --- a/helm/hugegraph/templates/store-statefulset.yaml +++ b/helm/hugegraph/templates/store-statefulset.yaml @@ -101,7 +101,7 @@ spec: - -c - | set -eu - REQUIRED={{ include "hugegraph.pd.quorum" . }} + REQUIRED={{ include "hugegraph.pd.bootstrapQuorum" . }} HEALTH_PEERS=$(echo "{{ include "hugegraph.pd.restPeersList" . }}" | tr ',' ' ') TIMEOUT={{ .Values.store.waitTimeoutSeconds | default 900 }} DEADLINE=$(( $(date +%s) + TIMEOUT )) diff --git a/helm/hugegraph/tests/bootstrap_consumer_mismatch_test.yaml b/helm/hugegraph/tests/bootstrap_consumer_mismatch_test.yaml new file mode 100644 index 0000000000..31cd00405a --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_consumer_mismatch_test.yaml @@ -0,0 +1,119 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap refuses inconsistent existing consumers +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500,cluster-store-3.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' + env: + - name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686 +tests: +- it: does not silently rewrite an existing Store seed template + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap import found store seeds that differ diff --git a/helm/hugegraph/tests/bootstrap_consumer_missing_test.yaml b/helm/hugegraph/tests/bootstrap_consumer_missing_test.yaml new file mode 100644 index 0000000000..e73e4308d2 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_consumer_missing_test.yaml @@ -0,0 +1,91 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Bootstrap import rejects an incomplete existing Store seed template +templates: +- store-statefulset.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + template: + spec: + containers: + - name: pd + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + template: + spec: + containers: + - name: store + env: + - name: ANOTHER_ENV + value: present +tests: +- it: fails instead of silently adding a missing literal Store seed + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap import requires the existing store template to expose HG_STORE_PD_ADDRESS as a literal value diff --git a/helm/hugegraph/tests/bootstrap_corrupt_record_test.yaml b/helm/hugegraph/tests/bootstrap_corrupt_record_test.yaml new file mode 100644 index 0000000000..04da1daa0e --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_corrupt_record_test.yaml @@ -0,0 +1,134 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap rejects corrupt records +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: refuses a missing count rather than deriving from desired + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap record is missing a valid initialStoreCount +- it: refuses to repair incomplete history through a value pin + template: store-statefulset.yaml + set: + bootstrap.initialStoreCount: 3 + asserts: + - failedTemplate: + errorPattern: bootstrap record is missing a valid initialStoreCount diff --git a/helm/hugegraph/tests/bootstrap_custom_legacy_test.yaml b/helm/hugegraph/tests/bootstrap_custom_legacy_test.yaml new file mode 100644 index 0000000000..06394df39f --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_custom_legacy_test.yaml @@ -0,0 +1,116 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap rejects customized legacy endpoints +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: custom-store-0:8500,custom-store-1:8500,custom-store-2:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: requires review instead of rewriting custom Store endpoints + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap only supports the chart diff --git a/helm/hugegraph/tests/bootstrap_disable_guard_test.yaml b/helm/hugegraph/tests/bootstrap_disable_guard_test.yaml new file mode 100644 index 0000000000..cc539fc8f6 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_disable_guard_test.yaml @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Disabled bootstrap refuses to discard retained live topology +templates: +- pd-statefulset.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: false + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + annotations: + hugegraph.apache.org/bootstrap-retained: 'true' + labels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + template: + spec: + containers: + - name: pd + env: [] +tests: +- it: requires bootstrap to remain enabled after the retained marker is recorded + template: pd-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap.enabled cannot be disabled while the live PD StatefulSet records retained bootstrap topology diff --git a/helm/hugegraph/tests/bootstrap_disabled_reinstall_test.yaml b/helm/hugegraph/tests/bootstrap_disabled_reinstall_test.yaml new file mode 100644 index 0000000000..3de92472b0 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_disabled_reinstall_test.yaml @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Bootstrap reinstall checks depend on enabled mode +templates: +- pd-statefulset.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: false + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 5 + pd.pdb.minAvailable: 3 +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' +tests: +- it: leaves orphaned history unchecked when bootstrap is disabled + asserts: + - equal: + path: spec.replicas + value: 5 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610,cluster-pd-3.cluster-pd.NAMESPACE.svc:8610,cluster-pd-4.cluster-pd.NAMESPACE.svc:8610 +- it: refuses an orphaned record when bootstrap is enabled + set: + bootstrap.enabled: true + asserts: + - failedTemplate: + errorPattern: bootstrap history already exists on an install diff --git a/helm/hugegraph/tests/bootstrap_disabled_upgrade_test.yaml b/helm/hugegraph/tests/bootstrap_disabled_upgrade_test.yaml new file mode 100644 index 0000000000..8f4cdcd1d6 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_disabled_upgrade_test.yaml @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Disabled bootstrap keeps the default upgrade path free of ConfigMap reads +templates: +- pd-statefulset.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: false + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + template: + spec: + containers: + - name: pd + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 +tests: +- it: renders without a ConfigMap lookup when bootstrap stays disabled + template: pd-statefulset.yaml + asserts: + - equal: + path: spec.replicas + value: 3 diff --git a/helm/hugegraph/tests/bootstrap_fresh_test.yaml b/helm/hugegraph/tests/bootstrap_fresh_test.yaml new file mode 100644 index 0000000000..8a5a0d2e6e --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_fresh_test.yaml @@ -0,0 +1,255 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap fresh installs +templates: +- bootstrap-configmap.yaml +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token +tests: +- it: records fresh 1 PD and Store topology + template: bootstrap-configmap.yaml + set: &id001 + pd.replicas: 1 + store.replicas: 1 + pd.pdb.minAvailable: 1 + store.pdb.minAvailable: 1 + asserts: + - equal: + path: data.pdReplicas + value: '1' + - equal: + path: data.storeReplicas + value: '1' + - equal: + path: data.initialStoreCount + value: '1' + - equal: + path: immutable + value: true +- it: initializes all 1 fresh PD voters + template: pd-statefulset.yaml + set: *id001 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '1' +- it: gives fresh 1 topology matching client seeds + template: store-statefulset.yaml + set: *id001 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686 +- it: records fresh 3 PD and Store topology + template: bootstrap-configmap.yaml + set: &id002 + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 + asserts: + - equal: + path: data.pdReplicas + value: '3' + - equal: + path: data.storeReplicas + value: '3' + - equal: + path: data.initialStoreCount + value: '3' + - equal: + path: immutable + value: true +- it: initializes all 3 fresh PD voters + template: pd-statefulset.yaml + set: *id002 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '3' +- it: gives fresh 3 topology matching client seeds + template: store-statefulset.yaml + set: *id002 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 +- it: records fresh 5 PD and Store topology + template: bootstrap-configmap.yaml + set: &id003 + pd.replicas: 5 + store.replicas: 5 + pd.pdb.minAvailable: 3 + store.pdb.minAvailable: 4 + asserts: + - equal: + path: data.pdReplicas + value: '5' + - equal: + path: data.storeReplicas + value: '5' + - equal: + path: data.initialStoreCount + value: '5' + - equal: + path: immutable + value: true +- it: initializes all 5 fresh PD voters + template: pd-statefulset.yaml + set: *id003 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610,cluster-pd-3.cluster-pd.NAMESPACE.svc:8610,cluster-pd-4.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500,cluster-store-3.cluster-store.NAMESPACE.svc:8500,cluster-store-4.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '5' +- it: gives fresh 5 topology matching client seeds + template: store-statefulset.yaml + set: *id003 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686,cluster-pd-3.cluster-pd.NAMESPACE.svc:8686,cluster-pd-4.cluster-pd.NAMESPACE.svc:8686 +- it: rejects fresh PD5 with a three-voter bootstrap + template: store-statefulset.yaml + set: + pd.replicas: 5 + pd.pdb.minAvailable: 3 + bootstrap.pdReplicas: 3 + asserts: + - failedTemplate: + errorPattern: bootstrap.pdReplicas must equal pd.replicas on a fresh install +- it: rejects fresh Store1 with a three-Store bootstrap + template: store-statefulset.yaml + set: + store.replicas: 1 + store.pdb.minAvailable: 1 + bootstrap.storeReplicas: 3 + asserts: + - failedTemplate: + errorPattern: bootstrap.storeReplicas must equal store.replicas on a fresh install +- it: rejects fresh Store3 with a one-Store bootstrap + template: store-statefulset.yaml + set: + store.replicas: 3 + store.pdb.minAvailable: 2 + bootstrap.storeReplicas: 1 + bootstrap.initialStoreCount: 1 + asserts: + - failedTemplate: + errorPattern: bootstrap.storeReplicas must equal store.replicas on a fresh install +- it: rejects unreachable startup Store threshold + template: store-statefulset.yaml + set: + bootstrap.initialStoreCount: 4 + asserts: + - failedTemplate: + errorPattern: bootstrap.initialStoreCount cannot exceed bootstrap.storeReplicas +- it: rejects bootstrap counts above the chart replica limit + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 100 + asserts: + - failedTemplate: + errorPattern: bootstrap.pdReplicas must be an integer between 1 and 99 +- it: requires offline upgrades to pin all history + release: + upgrade: true + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: must all be pinned for an upgrade without a live bootstrap record +- it: rejects internally inconsistent offline history + release: + upgrade: true + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 3 + bootstrap.storeReplicas: 3 + bootstrap.initialStoreCount: 4 + asserts: + - failedTemplate: + errorPattern: bootstrap.initialStoreCount cannot exceed bootstrap.storeReplicas +- it: allows an offline upgrade with explicitly pinned history + release: + upgrade: true + template: pd-statefulset.yaml + set: + store.replicas: 4 + store.pdb.minAvailable: 3 + bootstrap.pdReplicas: 3 + bootstrap.storeReplicas: 3 + bootstrap.initialStoreCount: 3 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 diff --git a/helm/hugegraph/tests/bootstrap_incomplete_legacy_test.yaml b/helm/hugegraph/tests/bootstrap_incomplete_legacy_test.yaml new file mode 100644 index 0000000000..a25be70ed8 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_incomplete_legacy_test.yaml @@ -0,0 +1,123 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap rejects incomplete live initialization +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500,cluster-store-3.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + valueFrom: + configMapKeyRef: + name: custom + key: count + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: does not guess a live valueFrom count even with pins + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 3 + bootstrap.storeReplicas: 4 + bootstrap.initialStoreCount: 3 + asserts: + - failedTemplate: + errorPattern: bootstrap import requires the existing PD peer list diff --git a/helm/hugegraph/tests/bootstrap_legacy_owner_test.yaml b/helm/hugegraph/tests/bootstrap_legacy_owner_test.yaml new file mode 100644 index 0000000000..55c171a25e --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_legacy_owner_test.yaml @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Bootstrap import rejects a PD StatefulSet owned by another release +templates: +- pd-statefulset.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: OTHER-RELEASE + app.kubernetes.io/component: pd + spec: + template: + spec: + containers: + - name: pd + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' +tests: +- it: fails before importing another release's initialization history + template: pd-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap import found a PD StatefulSet owned by another release diff --git a/helm/hugegraph/tests/bootstrap_legacy_test.yaml b/helm/hugegraph/tests/bootstrap_legacy_test.yaml new file mode 100644 index 0000000000..6424849fa2 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_legacy_test.yaml @@ -0,0 +1,173 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap bootstrap_legacy_test.yaml +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500,cluster-store-3.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + env: + - name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: retains PD bootstrap inputs + template: pd-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500,cluster-store-3.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '3' +- it: retains Store seeds and bootstrap wait quorum + template: store-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: REQUIRED=2 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: cluster-pd-0\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-1\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-2\.cluster-pd\.NAMESPACE\.svc:8620 +- it: retains Server seeds + template: server-deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_PD_PEERS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 +- it: records the original topology + template: bootstrap-configmap.yaml + asserts: + - equal: + path: data.pdReplicas + value: '3' + - equal: + path: data.storeReplicas + value: '4' + - equal: + path: data.initialStoreCount + value: '3' +- it: refuses a bootstrap pin that conflicts with history + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 5 + asserts: + - failedTemplate: + errorPattern: bootstrap.pdReplicas cannot change diff --git a/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml b/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml new file mode 100644 index 0000000000..c471b36a08 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_pd_growth_test.yaml @@ -0,0 +1,198 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap bootstrap_pd_growth_test.yaml +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 5 + store.replicas: 3 + pd.pdb.minAvailable: 3 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + annotations: + hugegraph.apache.org/bootstrap-retained: 'true' + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '5' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: retains PD bootstrap inputs + template: pd-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '3' +- it: retains Store seeds and bootstrap wait quorum + template: store-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: REQUIRED=2 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: cluster-pd-0\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-1\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-2\.cluster-pd\.NAMESPACE\.svc:8620 +- it: retains Server seeds + template: server-deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_PD_PEERS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 +- it: records the original topology + template: bootstrap-configmap.yaml + asserts: + - equal: + path: data.pdReplicas + value: '3' + - equal: + path: data.storeReplicas + value: '3' + - equal: + path: data.initialStoreCount + value: '3' +- it: refuses a bootstrap pin that conflicts with history + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 5 + asserts: + - failedTemplate: + errorPattern: bootstrap.pdReplicas cannot change +- it: refuses disabling the retained record + template: store-statefulset.yaml + set: + bootstrap.enabled: false + asserts: + - failedTemplate: + errorPattern: bootstrap.enabled cannot be disabled +- it: validates current PD5 quorum for PDB rather than old PD3 seeds + template: server-deployment.yaml + set: + pd.pdb.minAvailable: 2 + asserts: + - failedTemplate: + errorPattern: pd.pdb.minAvailable must be at least the PD Raft majority diff --git a/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml b/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml new file mode 100644 index 0000000000..b2c501b64f --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_pd_shrink_test.yaml @@ -0,0 +1,191 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap bootstrap_pd_shrink_test.yaml +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME + data: + schemaVersion: '1' + pdReplicas: '5' + storeReplicas: '3' + initialStoreCount: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + annotations: + hugegraph.apache.org/bootstrap-retained: 'true' + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610,cluster-pd-3.cluster-pd.NAMESPACE.svc:8610,cluster-pd-4.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: retains PD bootstrap inputs + template: pd-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610,cluster-pd-3.cluster-pd.NAMESPACE.svc:8610,cluster-pd-4.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '3' +- it: retains Store seeds and bootstrap wait quorum + template: store-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686,cluster-pd-3.cluster-pd.NAMESPACE.svc:8686,cluster-pd-4.cluster-pd.NAMESPACE.svc:8686 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: REQUIRED=3 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: cluster-pd-0\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-1\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-2\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-3\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-4\.cluster-pd\.NAMESPACE\.svc:8620 +- it: retains Server seeds + template: server-deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_PD_PEERS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686,cluster-pd-3.cluster-pd.NAMESPACE.svc:8686,cluster-pd-4.cluster-pd.NAMESPACE.svc:8686 +- it: records the original topology + template: bootstrap-configmap.yaml + asserts: + - equal: + path: data.pdReplicas + value: '5' + - equal: + path: data.storeReplicas + value: '3' + - equal: + path: data.initialStoreCount + value: '3' +- it: refuses a bootstrap pin that conflicts with history + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 7 + asserts: + - failedTemplate: + errorPattern: bootstrap.pdReplicas cannot change +- it: refuses disabling the retained record + template: store-statefulset.yaml + set: + bootstrap.enabled: false + asserts: + - failedTemplate: + errorPattern: bootstrap.enabled cannot be disabled diff --git a/helm/hugegraph/tests/bootstrap_record_limit_test.yaml b/helm/hugegraph/tests/bootstrap_record_limit_test.yaml new file mode 100644 index 0000000000..463085adc3 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_record_limit_test.yaml @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Bootstrap rejects retained counts above the chart replica limit +templates: +- pd-statefulset.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME + data: + schemaVersion: '1' + pdReplicas: '100' + storeReplicas: '3' + initialStoreCount: '3' +tests: +- it: rejects an oversized retained PD count before rendering peer lists + template: pd-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap record is missing a valid pdReplicas between 1 and 99 diff --git a/helm/hugegraph/tests/bootstrap_record_missing_owner_test.yaml b/helm/hugegraph/tests/bootstrap_record_missing_owner_test.yaml new file mode 100644 index 0000000000..ee9f32cd6e --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_record_missing_owner_test.yaml @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Bootstrap rejects an ownerless retained record +templates: +- store-statefulset.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' +tests: +- it: fails closed when retained history has no release owner + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: missing the app.kubernetes.io/instance owner label diff --git a/helm/hugegraph/tests/bootstrap_record_test.yaml b/helm/hugegraph/tests/bootstrap_record_test.yaml new file mode 100644 index 0000000000..d54646d7ea --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_record_test.yaml @@ -0,0 +1,207 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap bootstrap_record_test.yaml +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + annotations: + hugegraph.apache.org/bootstrap-retained: 'true' + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: retains PD bootstrap inputs + template: pd-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_INITIAL_STORE_COUNT + value: '3' +- it: retains Store seeds and bootstrap wait quorum + template: store-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_STORE_PD_ADDRESS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: REQUIRED=2 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: cluster-pd-0\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-1\.cluster-pd\.NAMESPACE\.svc:8620,cluster-pd-2\.cluster-pd\.NAMESPACE\.svc:8620 +- it: retains Server seeds + template: server-deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_PD_PEERS + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8686,cluster-pd-1.cluster-pd.NAMESPACE.svc:8686,cluster-pd-2.cluster-pd.NAMESPACE.svc:8686 +- it: records the original topology + template: bootstrap-configmap.yaml + asserts: + - equal: + path: data.pdReplicas + value: '3' + - equal: + path: data.storeReplicas + value: '3' + - equal: + path: data.initialStoreCount + value: '3' +- it: refuses a bootstrap pin that conflicts with history + template: store-statefulset.yaml + set: + bootstrap.pdReplicas: 5 + asserts: + - failedTemplate: + errorPattern: bootstrap.pdReplicas cannot change +- it: refuses disabling the retained record + template: store-statefulset.yaml + set: + bootstrap.enabled: false + asserts: + - failedTemplate: + errorPattern: bootstrap.enabled cannot be disabled +- it: retains original PD grow guard + template: server-deployment.yaml + set: + pd.replicas: 5 + pd.pdb.minAvailable: 3 + asserts: + - failedTemplate: + errorPattern: pd.replicas cannot grow from 3 to 5 +- it: retains original Store shrink guard + template: server-deployment.yaml + set: + store.replicas: 3 + store.pdb.minAvailable: 2 + asserts: + - failedTemplate: + errorPattern: store.replicas cannot shrink from 4 to 3 diff --git a/helm/hugegraph/tests/bootstrap_record_version_test.yaml b/helm/hugegraph/tests/bootstrap_record_version_test.yaml new file mode 100644 index 0000000000..38d54aa2ad --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_record_version_test.yaml @@ -0,0 +1,128 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap rejects unsupported record versions +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + labels: + app.kubernetes.io/instance: RELEASE-NAME + data: + schemaVersion: '2' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: rejects a record schema it cannot interpret + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: unsupported schemaVersion diff --git a/helm/hugegraph/tests/bootstrap_record_wrong_owner_test.yaml b/helm/hugegraph/tests/bootstrap_record_wrong_owner_test.yaml new file mode 100644 index 0000000000..c9e38de3a0 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_record_wrong_owner_test.yaml @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Bootstrap rejects another release's retained record +templates: +- store-statefulset.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 3 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 2 +release: + upgrade: true +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + labels: + app.kubernetes.io/instance: another-release + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' +tests: +- it: fails closed when retained history belongs to another release + template: store-statefulset.yaml + asserts: + - failedTemplate: + errorPattern: bootstrap ConfigMap belongs to another release diff --git a/helm/hugegraph/tests/bootstrap_reinstall_test.yaml b/helm/hugegraph/tests/bootstrap_reinstall_test.yaml new file mode 100644 index 0000000000..5f5099eac0 --- /dev/null +++ b/helm/hugegraph/tests/bootstrap_reinstall_test.yaml @@ -0,0 +1,127 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Experimental bootstrap refuses implicit recovery +templates: +- pd-statefulset.yaml +- store-statefulset.yaml +- server-deployment.yaml +- bootstrap-configmap.yaml +set: + fullnameOverride: cluster + bootstrap.enabled: true + pd.auth.existingSecret: fixture-pd + server.auth.admin.existingSecret: fixture-admin + server.auth.token.existingSecret: fixture-token + pd.replicas: 3 + store.replicas: 4 + pd.pdb.minAvailable: 2 + store.pdb.minAvailable: 3 +kubernetesProvider: + scheme: + v1/ConfigMap: + gvr: + version: v1 + resource: configmaps + namespaced: true + apps/v1/StatefulSet: + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-bootstrap + namespace: NAMESPACE + data: + schemaVersion: '1' + pdReplicas: '3' + storeReplicas: '3' + initialStoreCount: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-pd + namespace: NAMESPACE + labels: &id001 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + spec: + replicas: '3' + volumeClaimTemplates: + - metadata: + name: pd-data + selector: + matchLabels: *id001 + template: + spec: + containers: + - name: pd + ports: + - name: grpc + containerPort: '8686' + - name: rest + containerPort: '8620' + - name: raft + containerPort: '8610' + env: + - name: HG_PD_RAFT_PEERS_LIST + value: cluster-pd-0.cluster-pd.NAMESPACE.svc:8610,cluster-pd-1.cluster-pd.NAMESPACE.svc:8610,cluster-pd-2.cluster-pd.NAMESPACE.svc:8610 + - name: HG_PD_INITIAL_STORE_LIST + value: cluster-store-0.cluster-store.NAMESPACE.svc:8500,cluster-store-1.cluster-store.NAMESPACE.svc:8500,cluster-store-2.cluster-store.NAMESPACE.svc:8500 + - name: HG_PD_INITIAL_STORE_COUNT + value: '3' + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: cluster-store + namespace: NAMESPACE + labels: &id002 + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: store + spec: + replicas: '4' + volumeClaimTemplates: + - metadata: + name: store-data + selector: + matchLabels: *id002 + template: + spec: + containers: + - name: store + ports: + - name: grpc + containerPort: '8500' + - name: rest + containerPort: '8520' + - name: raft + containerPort: '8510' +tests: +- it: refuses treating retained topology as a fresh install + template: store-statefulset.yaml + set: + pd.replicas: 5 + pd.pdb.minAvailable: 3 + asserts: + - failedTemplate: + errorPattern: bootstrap history already exists on an install diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 05a636c1ef..08328548d2 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -122,6 +122,51 @@ "minLength": 1, "pattern": "^[A-Za-z0-9_.-]+$", "description": "Shared Server/Store PD metadata namespace. Preserve the initialized Server value; changing it does not migrate data." + }, + "bootstrap": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "pdReplicas": { + "oneOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "string", + "pattern": "^$|^[1-9][0-9]*$" + } + ] + }, + "storeReplicas": { + "oneOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "string", + "pattern": "^$|^[1-9][0-9]*$" + } + ] + }, + "initialStoreCount": { + "oneOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "string", + "pattern": "^$|^[1-9][0-9]*$" + } + ] + } + } } }, "definitions": { diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 023f14d12c..329b19ca10 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -29,6 +29,16 @@ global: persistence: enabled: true +# Opt-in stable initialization topology, separate from desired node replicas. +# Online renders capture the first topology in a kept ConfigMap, or import the +# existing PD environment on the first upgrade. Offline upgrades must pin all +# three counts and render with --is-upgrade. This does not change membership. +bootstrap: + enabled: false + pdReplicas: "" + storeReplicas: "" + initialStoreCount: "" + fullnameOverride: "" nameOverride: "" diff --git a/helm/test_helm_bootstrap.py b/helm/test_helm_bootstrap.py new file mode 100644 index 0000000000..19fc2e95b7 --- /dev/null +++ b/helm/test_helm_bootstrap.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compare whole rendered Pod templates across controlled topology changes.""" + +from pathlib import Path +import re +import subprocess +import unittest + + +CHART = Path(__file__).resolve().parent / "hugegraph" + + +class BootstrapLifecycleTest(unittest.TestCase): + def render(self, pd=3, stores=3, initial_pd=3, initial_stores=3, initial_count=3, + upgrade=True): + values = { + "fullnameOverride": "cluster", + "bootstrap.enabled": "true", + "pd.replicas": pd, + "store.replicas": stores, + "pd.pdb.minAvailable": pd // 2 + 1, + "store.pdb.minAvailable": max(stores - 1, 1), + "pd.auth.existingSecret": "fixture-pd", + "server.auth.admin.existingSecret": "fixture-admin", + "server.auth.token.existingSecret": "fixture-token", + } + if upgrade: + values.update({ + "bootstrap.pdReplicas": initial_pd, + "bootstrap.storeReplicas": initial_stores, + "bootstrap.initialStoreCount": initial_count, + }) + command = ["helm", "template", "bootstrap-test", str(CHART), + "--namespace", "bootstrap-test"] + if upgrade: + command.append("--is-upgrade") + for key, value in values.items(): + command.extend(["--set", f"{key}={value}"]) + try: + result = subprocess.run( + command, + check=True, + capture_output=True, + text=True, + timeout=300, + ) + except subprocess.CalledProcessError as error: + self.fail(f"helm template failed (exit {error.returncode}):\n{error.stderr}") + except subprocess.TimeoutExpired as error: + self.fail(f"helm template timed out after {error.timeout}s") + documents = {} + for document in result.stdout.split("---\n"): + source = re.search(r"^# Source: [^\n]+/templates/([^\n]+)$", document, re.M) + if source: + documents[source.group(1)] = document + return documents + + def template(self, documents, component): + name = "server-deployment.yaml" if component == "server" else component + "-statefulset.yaml" + lines = documents[name].splitlines(keepends=True) + start = next(i for i, line in enumerate(lines) if line == " template:\n") + end = start + 1 + while end < len(lines) and (not lines[end].strip() or lines[end].startswith(" ")): + end += 1 + return "".join(lines[start:end]) + + def assert_templates_unchanged(self, before, after): + for component in ("pd", "store", "server"): + with self.subTest(component=component): + self.assertEqual(self.template(before, component), self.template(after, component)) + + def test_store_growth_only_changes_capacity_resources(self): + self.assert_templates_unchanged(self.render(stores=3), self.render(stores=4)) + + def test_store_growth_across_default_rf_boundary_preserves_bootstrap(self): + before = self.render(stores=1, initial_stores=1, initial_count=1) + after = self.render(stores=3, initial_stores=1, initial_count=1) + self.assert_templates_unchanged(before, after) + self.assertIn("-Dpartition.default-shard-count=1", self.template(after, "pd")) + + def test_pd_growth_keeps_initial_peers_and_consumers_stable(self): + self.assert_templates_unchanged(self.render(pd=3), self.render(pd=5)) + + def test_pd_shrink_keeps_recorded_five_peer_bootstrap(self): + before = self.render(pd=5, initial_pd=5) + after = self.render(pd=3, initial_pd=5) + self.assert_templates_unchanged(before, after) + + def test_fresh_topologies_bootstrap_the_requested_members(self): + for count in (1, 3, 5): + with self.subTest(replicas=count): + documents = self.render(pd=count, stores=count, upgrade=False) + pd = documents["pd-statefulset.yaml"] + value = re.search(r'- name: HG_PD_RAFT_PEERS_LIST\n\s+value: "([^"]+)"', pd).group(1) + self.assertEqual(count, len(value.split(","))) + self.assertIn(f' pdReplicas: "{count}"', documents["bootstrap-configmap.yaml"]) + self.assertIn(f' initialStoreCount: "{count}"', documents["bootstrap-configmap.yaml"]) + + +if __name__ == "__main__": + unittest.main()