From 4bc9f1802f6ff11bc09016ffe1604b415e508502 Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Wed, 23 Sep 2026 17:17:33 -0400 Subject: [PATCH] feat(api): Assign an egress shard's SID in spec A shard's SID still arrives as process configuration, so every shard node needs a hand-written patch on the galactic-nat DaemonSet even after its addresses moved into spec. That patch is the per-node configuration datum-cloud/galactic#581 removes: the shard's process should program its whole identity from its own EgressShard, and it cannot while the SID has nowhere in spec to live. Add spec.shardSID on the same terms as the addresses. It is write-once for the same reason they are: every established flow's reply is re-encapsulated from this SID, so changing it strands them. It stays operator-chosen until the addressing service hands out identifiers of this kind, which is what the status-only position was waiting for; moving it now changes who writes it, not who chooses it. status.shardSID keeps its key and becomes what the datapath is programmed with, matching the address fields. Key changes: - Add spec.shardSID, optional, IPv6-only, write-once, and unable to be unassigned once assigned - Re-specify status.shardSID as the SID the datapath is programmed with - Cover the new key in the JSON and omitempty unit tests - Extend the chainsaw scenario with a changed SID, an unassigned SID, and an IPv4 SID, each rejected Co-Authored-By: Claude Opus 5.5 (1M context) --- api/v1alpha1/egressshard_types.go | 26 +++++-- api/v1alpha1/egressshard_types_test.go | 11 ++- .../network.datumapis.com_egressshards.yaml | 31 ++++++-- docs/api/bgp.md | 3 +- docs/api/gateway.md | 3 +- .../chainsaw-test.yaml | 78 ++++++++++++++++++- 6 files changed, 135 insertions(+), 17 deletions(-) diff --git a/api/v1alpha1/egressshard_types.go b/api/v1alpha1/egressshard_types.go index 6c336d3..cc1b78e 100644 --- a/api/v1alpha1/egressshard_types.go +++ b/api/v1alpha1/egressshard_types.go @@ -205,6 +205,7 @@ type AddressClaimRef struct { // is therefore accepted and is unattributable, which is a fact about the // allocation rather than something a schema can repair. // +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.shardSID) && size(oldSelf.shardSID) > 0) || (has(self.shardSID) && size(self.shardSID) > 0)",message="shardSID cannot be unassigned once assigned" // +kubebuilder:validation:XValidation:rule="(has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0) == (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)",message="shardAddressIPv4 and nat64Prefix must be set together" // +kubebuilder:validation:XValidation:rule="!(has(oldSelf.shardAddressIPv6) && size(oldSelf.shardAddressIPv6) > 0) || (has(self.shardAddressIPv6) && size(self.shardAddressIPv6) > 0)",message="shardAddressIPv6 cannot be unassigned once assigned" // +kubebuilder:validation:XValidation:rule="!(has(oldSelf.shardAddressIPv4) && size(oldSelf.shardAddressIPv4) > 0) || (has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0)",message="shardAddressIPv4 cannot be unassigned once assigned" @@ -216,6 +217,24 @@ type EgressShardSpec struct { // +kubebuilder:validation:Required TargetRef TargetRef `json:"targetRef"` + // ShardSID is this shard's own SRv6 uSID, the outer destination a tenant + // VRF's egress route encapsulates toward. The shard claims a packet by its + // Block and Node-ID alone, the Argument carrying the tenant, so those 64 + // bits must be reserved for this shard and disjoint from every BGPRouter + // locator: a value another node already uses silently diverts that node's + // traffic. + // + // Write-once for the same reason the addresses are: every established + // flow's reply is re-encapsulated from this SID, so changing it strands + // them. Operator-chosen until the addressing service hands out identifiers + // of this kind. + // + // Empty means no SID is assigned yet, and the shard claims no packet. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 6)",message="shardSID must be a valid IPv6 address" + // +kubebuilder:validation:XValidation:rule="self == oldSelf || oldSelf == ''",message="shardSID is immutable once assigned" + ShardSID string `json:"shardSID,omitempty"` + // ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this // shard translates to — every NAT66 masquerade port it allocates lives // within this address, so any node can route a reply to the owning shard @@ -300,11 +319,8 @@ type EgressShardStatus struct { // packet gets is decided from the inner destination, not from a second // SID. // - // Still chosen by an operator and reported here rather than assigned in - // spec, unlike the addresses: a value another node already uses silently - // diverts that node's traffic, so the assignment belongs to the - // addressing service, which does not hand out identifiers of this kind - // yet. + // Reports the SID this node's datapath is programmed with, from + // Spec.ShardSID. // +optional // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 6)",message="shardSID must be a valid IPv6 address" ShardSID string `json:"shardSID,omitempty"` diff --git a/api/v1alpha1/egressshard_types_test.go b/api/v1alpha1/egressshard_types_test.go index c04f06c..4deddcb 100644 --- a/api/v1alpha1/egressshard_types_test.go +++ b/api/v1alpha1/egressshard_types_test.go @@ -24,6 +24,7 @@ func newTestEgressShard() *EgressShard { }, Spec: EgressShardSpec{ TargetRef: TargetRef{Kind: "Node", Name: "node-a"}, + ShardSID: "2001:db8:ff01:2001::", ShardAddressIPv6: "2001:db8:f00d::100", ShardAddressIPv6ClaimRef: &AddressClaimRef{ APIGroup: "ipam.miloapis.com", @@ -42,6 +43,7 @@ func TestEgressShardDeepCopy(t *testing.T) { orig := newTestEgressShard() dup := orig.DeepCopy() + dup.Spec.ShardSID = "2001:db8:ff01:2002::" dup.Spec.ShardAddressIPv6 = "2001:db8:f00d::200" dup.Spec.ShardAddressIPv4 = "198.51.100.7" dup.Spec.NAT64Prefix = "64:ff9b::/96" @@ -49,6 +51,9 @@ func TestEgressShardDeepCopy(t *testing.T) { dup.Spec.ShardAddressIPv6ClaimRef.Name = "some-other-claim" dup.Status.Conditions = append(dup.Status.Conditions, metav1.Condition{Type: ConditionTypeProgrammed}) + if orig.Spec.ShardSID != "2001:db8:ff01:2001::" { + t.Errorf("ShardSID mutated: got %q", orig.Spec.ShardSID) + } if orig.Spec.ShardAddressIPv6 != "2001:db8:f00d::100" { t.Errorf("ShardAddressIPv6 mutated: got %q", orig.Spec.ShardAddressIPv6) } @@ -107,7 +112,8 @@ func TestEgressShardJSONRoundTrip(t *testing.T) { t.Fatalf("Unmarshal: %v", err) } - if got.Spec.ShardAddressIPv6 != orig.Spec.ShardAddressIPv6 || + if got.Spec.ShardSID != orig.Spec.ShardSID || + got.Spec.ShardAddressIPv6 != orig.Spec.ShardAddressIPv6 || got.Spec.ShardAddressIPv4 != orig.Spec.ShardAddressIPv4 || got.Spec.NAT64Prefix != orig.Spec.NAT64Prefix || got.Spec.TargetRef != orig.Spec.TargetRef { @@ -148,6 +154,7 @@ func TestEgressShardSpecFieldNames(t *testing.T) { } want := map[string]string{ + "shardSID": "2001:db8:ff01:2001::", "shardAddressIPv6": "2001:db8:f00d::100", "shardAddressIPv4": "198.51.100.7", "nat64Prefix": "64:ff9b::/96", @@ -196,7 +203,7 @@ func TestEgressShardSpecOmitEmpty(t *testing.T) { } for _, key := range []string{ - "shardAddressIPv6", "shardAddressIPv4", "nat64Prefix", + "shardSID", "shardAddressIPv6", "shardAddressIPv4", "nat64Prefix", "shardAddressIPv6ClaimRef", "shardAddressIPv4ClaimRef", } { if _, ok := m[key]; ok { diff --git a/config/crd/network.datumapis.com_egressshards.yaml b/config/crd/network.datumapis.com_egressshards.yaml index cdb1f27..f9d2e3a 100644 --- a/config/crd/network.datumapis.com_egressshards.yaml +++ b/config/crd/network.datumapis.com_egressshards.yaml @@ -264,6 +264,27 @@ spec: x-kubernetes-validations: - message: shardAddressIPv6ClaimRef is immutable once assigned rule: self == oldSelf + shardSID: + description: |- + ShardSID is this shard's own SRv6 uSID, the outer destination a tenant + VRF's egress route encapsulates toward. The shard claims a packet by its + Block and Node-ID alone, the Argument carrying the tenant, so those 64 + bits must be reserved for this shard and disjoint from every BGPRouter + locator: a value another node already uses silently diverts that node's + traffic. + + Write-once for the same reason the addresses are: every established + flow's reply is re-encapsulated from this SID, so changing it strands + them. Operator-chosen until the addressing service hands out identifiers + of this kind. + + Empty means no SID is assigned yet, and the shard claims no packet. + type: string + x-kubernetes-validations: + - message: shardSID must be a valid IPv6 address + rule: self == '' || (isIP(self) && ip(self).family() == 6) + - message: shardSID is immutable once assigned + rule: self == oldSelf || oldSelf == '' targetRef: description: TargetRef identifies the Node this shard executes on. properties: @@ -283,6 +304,9 @@ spec: - targetRef type: object x-kubernetes-validations: + - message: shardSID cannot be unassigned once assigned + rule: '!(has(oldSelf.shardSID) && size(oldSelf.shardSID) > 0) || (has(self.shardSID) + && size(self.shardSID) > 0)' - message: shardAddressIPv4 and nat64Prefix must be set together rule: (has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0) == (has(self.nat64Prefix) && size(self.nat64Prefix) > 0) @@ -414,11 +438,8 @@ spec: packet gets is decided from the inner destination, not from a second SID. - Still chosen by an operator and reported here rather than assigned in - spec, unlike the addresses: a value another node already uses silently - diverts that node's traffic, so the assignment belongs to the - addressing service, which does not hand out identifiers of this kind - yet. + Reports the SID this node's datapath is programmed with, from + Spec.ShardSID. type: string x-kubernetes-validations: - message: shardSID must be a valid IPv6 address diff --git a/docs/api/bgp.md b/docs/api/bgp.md index 7c4eb1b..07d24b9 100644 --- a/docs/api/bgp.md +++ b/docs/api/bgp.md @@ -1058,6 +1058,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| +| `shardSID` _string_ | ShardSID is this shard's own SRv6 uSID, the outer destination a tenant
VRF's egress route encapsulates toward. The shard claims a packet by its
Block and Node-ID alone, the Argument carrying the tenant, so those 64
bits must be reserved for this shard and disjoint from every BGPRouter
locator: a value another node already uses silently diverts that node's
traffic.
Write-once for the same reason the addresses are: every established
flow's reply is re-encapsulated from this SID, so changing it strands
them. Operator-chosen until the addressing service hands out identifiers
of this kind.
Empty means no SID is assigned yet, and the shard claims no packet. | | | | `shardAddressIPv6` _string_ | ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this
shard translates to — every NAT66 masquerade port it allocates lives
within this address, so any node can route a reply to the owning shard
using ordinary unicast routing on it alone, with no per-flow state
lookup anywhere but that shard.
Empty means no IPv6 address is assigned to this shard. | | | | `shardAddressIPv6ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv6ClaimRef records the addressing-service claim
ShardAddressIPv6 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | | `shardAddressIPv4` _string_ | ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this
shard translates to, and the source an IPv4-only destination sees.
Unlike ShardAddressIPv6, reachability for it is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the underlay or an upstream announcement must attract this
address to this node.
Empty means no IPv4 address is assigned to this shard. | | | @@ -1084,7 +1085,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | -| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID.
Still chosen by an operator and reported here rather than assigned in
spec, unlike the addresses: a value another node already uses silently
diverts that node's traffic, so the assignment belongs to the
addressing service, which does not hand out identifiers of this kind
yet. | | | +| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID.
Reports the SID this node's datapath is programmed with, from
Spec.ShardSID. | | | | `shardAddressIPv6` _string_ | ShardAddressIPv6 is the IPv6 masquerade source this shard's datapath is
programmed with. Empty means it translates no IPv6 flow. | | | | `shardAddressIPv4` _string_ | ShardAddressIPv4 is the IPv4 masquerade source this shard's datapath is
programmed with. Empty means it performs no NAT64. Publishing it is also
what makes the underlay reachability prerequisite in
Spec.ShardAddressIPv4 checkable. | | | | `nat64Prefix` _string_ | NAT64Prefix is the prefix this shard's datapath is programmed to
translate. Empty whenever ShardAddressIPv4 is empty. | | | diff --git a/docs/api/gateway.md b/docs/api/gateway.md index 9cfebfe..cff36ee 100644 --- a/docs/api/gateway.md +++ b/docs/api/gateway.md @@ -159,6 +159,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| +| `shardSID` _string_ | ShardSID is this shard's own SRv6 uSID, the outer destination a tenant
VRF's egress route encapsulates toward. The shard claims a packet by its
Block and Node-ID alone, the Argument carrying the tenant, so those 64
bits must be reserved for this shard and disjoint from every BGPRouter
locator: a value another node already uses silently diverts that node's
traffic.
Write-once for the same reason the addresses are: every established
flow's reply is re-encapsulated from this SID, so changing it strands
them. Operator-chosen until the addressing service hands out identifiers
of this kind.
Empty means no SID is assigned yet, and the shard claims no packet. | | | | `shardAddressIPv6` _string_ | ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this
shard translates to — every NAT66 masquerade port it allocates lives
within this address, so any node can route a reply to the owning shard
using ordinary unicast routing on it alone, with no per-flow state
lookup anywhere but that shard.
Empty means no IPv6 address is assigned to this shard. | | | | `shardAddressIPv6ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv6ClaimRef records the addressing-service claim
ShardAddressIPv6 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | | `shardAddressIPv4` _string_ | ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this
shard translates to, and the source an IPv4-only destination sees.
Unlike ShardAddressIPv6, reachability for it is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the underlay or an upstream announcement must attract this
address to this node.
Empty means no IPv4 address is assigned to this shard. | | | @@ -185,7 +186,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | -| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID.
Still chosen by an operator and reported here rather than assigned in
spec, unlike the addresses: a value another node already uses silently
diverts that node's traffic, so the assignment belongs to the
addressing service, which does not hand out identifiers of this kind
yet. | | | +| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID.
Reports the SID this node's datapath is programmed with, from
Spec.ShardSID. | | | | `shardAddressIPv6` _string_ | ShardAddressIPv6 is the IPv6 masquerade source this shard's datapath is
programmed with. Empty means it translates no IPv6 flow. | | | | `shardAddressIPv4` _string_ | ShardAddressIPv4 is the IPv4 masquerade source this shard's datapath is
programmed with. Empty means it performs no NAT64. Publishing it is also
what makes the underlay reachability prerequisite in
Spec.ShardAddressIPv4 checkable. | | | | `nat64Prefix` _string_ | NAT64Prefix is the prefix this shard's datapath is programmed to
translate. Empty whenever ShardAddressIPv4 is empty. | | | diff --git a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml index 94a7f47..40c61cf 100644 --- a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml +++ b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml @@ -4,8 +4,8 @@ metadata: name: egress-shard-address-immutability spec: description: > - Verify the CEL rules governing an EgressShard's assigned egress addresses - and the claim references that attribute them: a shard is accepted with no + Verify the CEL rules governing an EgressShard's assigned shard SID, egress + addresses, and the claim references that attribute them: a shard is accepted with no address, the cell controller's first write is accepted, rewriting the same value is accepted, and changing or unassigning an assigned value is rejected. Also verifies that a claim reference defaults its apiGroup and @@ -44,7 +44,7 @@ spec: content: | set -e kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' - {"spec":{"shardAddressIPv6":"2001:db8:f00d::100","shardAddressIPv4":"198.51.100.7","nat64Prefix":"64:ff9b::/96", + {"spec":{"shardSID":"2001:db8:ff01:2001::","shardAddressIPv6":"2001:db8:f00d::100","shardAddressIPv4":"198.51.100.7","nat64Prefix":"64:ff9b::/96", "shardAddressIPv6ClaimRef":{"project":"e2e-project","namespace":"egress","name":"e2e-claim-ipv6"}}}' echo "OK: first assignment accepted" - assert: @@ -54,6 +54,7 @@ spec: metadata: name: e2e-egress-shard spec: + shardSID: "2001:db8:ff01:2001::" shardAddressIPv6: 2001:db8:f00d::100 shardAddressIPv4: 198.51.100.7 nat64Prefix: 64:ff9b::/96 @@ -93,6 +94,77 @@ spec: echo "OK: changing an assigned shardAddressIPv6 correctly rejected" echo "Server response: $OUTPUT" + - name: reject-sid-change + try: + - script: + content: | + set +e + OUTPUT=$(kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardSID":"2001:db8:ff01:2002::"}}' 2>&1) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of a changed shardSID but the patch was accepted" + exit 1 + fi + case "$OUTPUT" in + *"immutable once assigned"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: changing an assigned shardSID correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-sid-unassignment + try: + - script: + content: | + set +e + OUTPUT=$(kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardSID":null}}' 2>&1) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of an unassigned shardSID but the patch was accepted" + exit 1 + fi + case "$OUTPUT" in + *"cannot be unassigned once assigned"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: unassigning an assigned shardSID correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-non-ipv6-sid + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard-ipv4-sid + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + shardSID: 198.51.100.9 + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of an IPv4 shardSID but the resource was accepted" + kubectl delete egressshard e2e-egress-shard-ipv4-sid -n "$NAMESPACE" 2>/dev/null || true + exit 1 + fi + case "$OUTPUT" in + *"shardSID must be a valid IPv6 address"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: an IPv4 shardSID correctly rejected" + echo "Server response: $OUTPUT" + - name: reject-prefix-change try: - script: