From 9393c44711b7235ab932baef11027cd2f2b2df4d Mon Sep 17 00:00:00 2001 From: Evgeny Slutsky Date: Thu, 10 Sep 2026 18:28:39 +0200 Subject: [PATCH 1/3] NO-ISSUE: Label primary node and bootstrap OVN SBDB/NBDB RAFT cluster on workers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In MicroShift multinode mode both nodes carry node-role.kubernetes.io/master, so ovnkube-master ran on all nodes — each started its own isolated OVN RAFT cluster with no knowledge of the other node. Two fixes: 1. pkg/node/kubelet.go: the primary node (identified by the absence of a bootstrap kubeconfig) receives the label node.microshift.io/role=primary at kubelet startup. This distinguishes the primary from workers for other consumers without changing the DaemonSet topology. 2. assets/components/ovn/multi-node/master/daemonset.yaml: the sbdb and nbdb containers now detect whether they are running on a worker node by reading the [OvnSouth]/[OvnNorth] address from the mounted ovnkube.conf. When the address points to a different host, they pass --db-sb-cluster-remote-addr (--db-nb-cluster-remote-addr) to join the primary's existing RAFT cluster instead of starting a new one. Each node then gets a local unix socket connected to the shared replicated database — no application-level TCP connection needed from ovnkube-node or ovn-controller. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../ovn/multi-node/master/daemonset.yaml | 30 +++++++++++++++++++ pkg/node/kubelet.go | 3 ++ 2 files changed, 33 insertions(+) diff --git a/assets/components/ovn/multi-node/master/daemonset.yaml b/assets/components/ovn/multi-node/master/daemonset.yaml index fe79c19640..94ca5cf491 100644 --- a/assets/components/ovn/multi-node/master/daemonset.yaml +++ b/assets/components/ovn/multi-node/master/daemonset.yaml @@ -141,6 +141,19 @@ spec: --db-nb-cluster-local-proto=tcp \ --no-monitor" + # On worker nodes the configmap contains [OvnNorth] address= pointing to + # the primary. Join the primary's NBDB RAFT cluster so this node gets a + # local unix socket connected to the shared database. + NB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnNorth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + if [[ "${NB_ADDR}" =~ ^tcp: ]] && [[ "${NB_ADDR}" != *"${K8S_NODE_IP}"* ]]; then + PRIMARY_IP="${NB_ADDR#tcp:}"; PRIMARY_IP="${PRIMARY_IP%%:*}" + OVN_ARGS="${OVN_ARGS} \ + --db-nb-cluster-remote-addr=$(bracketify ${PRIMARY_IP}) \ + --db-nb-cluster-remote-port=9643 \ + --db-nb-cluster-remote-proto=tcp" + fi + rm -f /run/ovn/ovnnb_db.sock echo "$(date -Iseconds) - starting nbdb" @@ -234,6 +247,8 @@ spec: name: run-openvswitch - mountPath: /run/ovn/ name: run-ovn + - mountPath: /run/ovnkube-config/ + name: ovnkube-config - mountPath: /env name: env-overrides resources: @@ -288,6 +303,19 @@ spec: --db-sb-cluster-local-proto=tcp \ --no-monitor" + # On worker nodes the configmap contains [OvnSouth] address= pointing to + # the primary. Join the primary's SBDB RAFT cluster so this node gets a + # local unix socket connected to the shared database. + SB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnSouth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + if [[ "${SB_ADDR}" =~ ^tcp: ]] && [[ "${SB_ADDR}" != *"${K8S_NODE_IP}"* ]]; then + PRIMARY_IP="${SB_ADDR#tcp:}"; PRIMARY_IP="${PRIMARY_IP%%:*}" + OVN_ARGS="${OVN_ARGS} \ + --db-sb-cluster-remote-addr=$(bracketify ${PRIMARY_IP}) \ + --db-sb-cluster-remote-port=9644 \ + --db-sb-cluster-remote-proto=tcp" + fi + rm -f /run/ovn/ovnsb_db.sock echo "$(date -Iseconds) - starting sbdb " @@ -345,6 +373,8 @@ spec: name: run-openvswitch - mountPath: /run/ovn/ name: run-ovn + - mountPath: /run/ovnkube-config/ + name: ovnkube-config - mountPath: /env name: env-overrides resources: diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index 6a0c8ce6d8..54385e4a74 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -89,6 +89,9 @@ func (s *KubeletServer) configure(cfg *config.Config) { kubeletFlags.NodeLabels["node-role.kubernetes.io/worker"] = "" kubeletFlags.NodeLabels["node.openshift.io/os_id"] = osID kubeletFlags.NodeLabels["node.kubernetes.io/instance-type"] = "rhde" + if !cfg.BootstrapKubeConfigExists() { + kubeletFlags.NodeLabels["node.microshift.io/role"] = "primary" + } kubeletConfig, err := loadConfigFile(filepath.Join(config.DataDir, "/resources/kubelet/config/config.yaml")) From 1a3e1350669c33a794c0e0341d0e134994232778 Mon Sep 17 00:00:00 2001 From: Evgeny Slutsky Date: Thu, 10 Sep 2026 18:28:55 +0200 Subject: [PATCH 2/3] NO-ISSUE: Fix OVN multinode SBDB connectivity for primary and worker nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 8a0f4f23eb removed --nb-address/--sb-address from ovnkube --init-node, which were the only mechanism pointing worker nodes at the primary's NB/SB databases. After that removal ovnkube fell back to local unix sockets that do not exist on workers, leaving each node with an isolated OVN RAFT cluster. The replacement per upstream OVN-K design is the config file: adding [OvnNorth]/[OvnSouth] stanzas with the primary's TCP addresses to ovnkube.conf so the binary can connect to the remote databases without explicit CLI flags. Three related fixes: 1. ovnkube.conf (configmap): in multinode mode add [OvnNorth]/[OvnSouth] stanzas with the primary's IP and NB/SB port. The sbdb/nbdb containers also use these addresses to discover the primary RAFT cluster to join. 2. networking.go — configmap guard: only the primary writes the ovnkube-config ConfigMap. A worker writing it would overwrite the correct primary IP with its own, breaking SBDB connectivity and RAFT join for all nodes. 3. networking.go — passes MultiNodeEnabled render param so the configmap template can conditionally emit the stanzas. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- assets/components/ovn/common/configmap.yaml | 8 ++++++ pkg/components/networking.go | 27 +++++++++++++-------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/assets/components/ovn/common/configmap.yaml b/assets/components/ovn/common/configmap.yaml index 7279ce1fb8..1f435a2d7f 100644 --- a/assets/components/ovn/common/configmap.yaml +++ b/assets/components/ovn/common/configmap.yaml @@ -34,3 +34,11 @@ data: election-lease-duration=137 election-renew-deadline=107 election-retry-period=26 +{{- if .MultiNodeEnabled}} + + [OvnNorth] + address=tcp:{{.NodeIP}}:{{.OVN_NB_PORT}} + + [OvnSouth] + address=tcp:{{.NodeIP}}:{{.OVN_SB_PORT}} +{{- end}} diff --git a/pkg/components/networking.go b/pkg/components/networking.go index 2e6ccad927..31606affb4 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -110,17 +110,24 @@ func startCNIPlugin(ctx context.Context, cfg *config.Config, kubeconfigPath stri return err } - // Multinode only params: OVN_NB_PORT, OVN_SB_PORT + // Multinode only params: OVN_NB_PORT, OVN_SB_PORT, MultiNodeEnabled extraParams := assets.RenderParams{ - "OVNConfig": ovnConfig, - "KubeconfigPath": kubeconfigPath, - "KubeconfigDir": filepath.Join(config.DataDir, "/resources/kubeadmin"), - "OVN_NB_PORT": ovn.OVN_NB_PORT, - "OVN_SB_PORT": ovn.OVN_SB_PORT, - } - if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { - klog.Warningf("Failed to apply configMap %v %v", cm, err) - return err + "OVNConfig": ovnConfig, + "KubeconfigPath": kubeconfigPath, + "KubeconfigDir": filepath.Join(config.DataDir, "/resources/kubeadmin"), + "OVN_NB_PORT": ovn.OVN_NB_PORT, + "OVN_SB_PORT": ovn.OVN_SB_PORT, + "MultiNodeEnabled": cfg.MultiNode.Enabled, + } + // In multinode mode the configmap contains [OvnNorth]/[OvnSouth] stanzas + // with the primary's IP. Only the primary may write it; a worker applying + // the configmap would overwrite the primary IP with its own, breaking SBDB + // connectivity for every node that reads the configmap afterwards. + if !cfg.MultiNode.Enabled || !cfg.BootstrapKubeConfigExists() { + if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { + klog.Warningf("Failed to apply configMap %v %v", cm, err) + return err + } } if err := assets.ApplyDaemonSets(ctx, apps, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { klog.Warningf("Failed to apply apps %v %v", apps, err) From c8eef6c78a92eba881e86b7f645489ba3a9669e0 Mon Sep 17 00:00:00 2001 From: Evgeny Slutsky Date: Thu, 10 Sep 2026 18:28:56 +0200 Subject: [PATCH 3/3] NO-ISSUE: Temporarily disable CNCF conformance exclusion for CI validation TODO: Revert before merging. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- test/bin/ci_phase_boot_and_test.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/bin/ci_phase_boot_and_test.sh b/test/bin/ci_phase_boot_and_test.sh index 79bd22e8c6..8cfb4dcf51 100755 --- a/test/bin/ci_phase_boot_and_test.sh +++ b/test/bin/ci_phase_boot_and_test.sh @@ -22,9 +22,12 @@ prepare_scenario_sources() { rm -rf "${SCENARIOS_TO_RUN}" mkdir -p "${SCENARIOS_TO_RUN}" cp "${SCENARIO_SOURCES}"/*.sh "${SCENARIOS_TO_RUN}"/ - if ${EXCLUDE_CNCF_CONFORMANCE}; then - find "${SCENARIOS_TO_RUN}" -name "*cncf-conformance.sh" -delete - fi + # TODO: Temporarily disabled so that the CNCF conformance scenario runs + # unconditionally while the multinode OVN SBDB fix (PR #7344) is validated + # in CI. Revert once the job is confirmed green. + # if ${EXCLUDE_CNCF_CONFORMANCE}; then + # find "${SCENARIOS_TO_RUN}" -name "*cncf-conformance.sh" -delete + # fi } # Log output automatically