From 60821de5dd765b44177fe503bd8a93826789560e Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 17 Sep 2026 14:53:52 -0500 Subject: [PATCH 1/4] feat(api): Assign egress shard addresses in spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An egress shard's masquerade sources arrive as process configuration today, and EgressShardStatus exists to publish them: "Every field here is echoed from what the shard's datapath process was actually started with, not derived". That makes an operator the only party who can give a shard an address, puts the value out of reach of any controller, and leaves a cell with no way to claim an address on a shard's behalf. Move the assignment into spec. The controller that owns a cell claims an address per family from the addressing service and writes it here, which keeps that credential off every translating node and keeps the allocation request out of the path that attaches a workload. Status keeps every field and every JSON key it had, re-specified as what the node's datapath is actually programmed with, so a stale datapath and an unclaimed address stay distinguishable and the BGPAdvertisement built from those values needs no change. The shard SID stays in status. It is still chosen by hand, nothing assigns it, and a value another node already uses silently diverts that node's traffic — so its authority belongs to the addressing service, which hands out no identifier of this kind yet. Adding spec.shardSID once it does is an additive change. Selection labels let an API in another group place traffic on these shards without this group referencing, importing, or depending on it: the selector lives entirely on the selecting side, and a shard names nothing that selects it. Key changes: - Add spec.shardAddressIPv6, spec.shardAddressIPv4, and spec.nat64Prefix - Require the IPv4 address and the NAT64 prefix to be set together, a pairing each shard re-checked at startup - Keep every address optional, so a shard exists from the moment its node is labelled and gains its identity afterwards - Add the Programmed condition, which separates "datapath attached" from "translating with the address assigned to it" - Add the network.datumapis.com egress-pool, egress-cell, egress-ipv6, and egress-ipv4 label keys - Cover the spec/status split, the JSON keys, and the label namespace with unit tests Co-Authored-By: Claude Opus 5 (1M context) --- api/v1alpha1/egressshard_types.go | 178 +++++++++++++---- api/v1alpha1/egressshard_types_test.go | 189 ++++++++++++++++++ .../network.datumapis.com_egressshards.yaml | 131 ++++++++---- docs/api/bgp.md | 49 +++-- docs/api/gateway.md | 49 +++-- 5 files changed, 496 insertions(+), 100 deletions(-) create mode 100644 api/v1alpha1/egressshard_types_test.go diff --git a/api/v1alpha1/egressshard_types.go b/api/v1alpha1/egressshard_types.go index 1118259..5ac4fe0 100644 --- a/api/v1alpha1/egressshard_types.go +++ b/api/v1alpha1/egressshard_types.go @@ -16,10 +16,9 @@ import ( // (IPv6 -> IPv4, RFC 6146) are the same function — stateful egress PAT with a // VRF-scoped session table — over different families, so one shard object // describes both rather than there being a second, near-duplicate kind. -// Status.ShardAddressIPv6 and Status.ShardAddressIPv4 are each set only for -// the family this shard actually translates; a shard serving only NAT66 -// leaves the IPv4 field empty and behaves exactly as it did before NAT64 -// existed. +// Spec.ShardAddressIPv6 and Spec.ShardAddressIPv4 are each assigned only for +// the family this shard translates; a shard with no IPv4 address performs no +// NAT64. // // Every shard owns a dedicated, publicly-routable address per family it // serves, and a flow's allocated masquerade port lives within it — so a reply @@ -28,6 +27,21 @@ import ( // can determine the owning shard from the tuple alone" property, satisfied by // construction rather than by a replicated hash table). // +// A shard is told which addresses to translate to; it does not choose them. +// The controller that owns a cell claims one address per family from the +// addressing service and writes it into this spec, which keeps the +// addressing-service credential off every translating node and keeps the +// allocation request out of the path that attaches a workload. Status reports +// what the node's datapath is actually programmed with, so an unclaimed +// address, a stale datapath, and a divergence between the two are each +// distinguishable. +// +// A shard names nothing that selects it. Which shards serve which consumer +// intent is decided entirely by label selectors evaluated on the selecting +// side (see the LabelEgressShard* keys), so no consumer-facing API appears in +// this group and no reference points from a datapath resource back at the +// resource that placed traffic on it. +// // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Namespaced,shortName=egressshard @@ -44,19 +58,123 @@ type EgressShard struct { Status EgressShardStatus `json:"status,omitempty"` } +// Label keys that select the EgressShards serving a given kind of egress. +// They are matched by an ordinary label selector held on the selecting side, +// which is what lets a consumer-facing API in another group place traffic on +// these shards without this group referencing, importing, or depending on +// that API. A shard carrying none of them is selected by nothing and serves +// no traffic. +// +// The pool and cell labels are operator-set placement facts. The family +// labels restate what this spec already assigns, because a selector matches +// labels and cannot read a spec field; whoever writes the addresses writes +// them. +const ( + // LabelEgressShardPool names the operator-defined pool this shard belongs + // to — the unit a selector picks, not an individual shard. Pools exist so + // that adding or draining a node changes no selector. + LabelEgressShardPool string = "network.datumapis.com/egress-pool" + + // LabelEgressShardCell names the cell this shard translates in. Egress is + // realized per cell, so a selector that omits it selects shards in every + // cell and sends a consumer's traffic out of an arbitrary one. + LabelEgressShardCell string = "network.datumapis.com/egress-cell" + + // LabelEgressShardIPv6 marks a shard that translates to IPv6, set to + // LabelValueEgressFamilyServed whenever Spec.ShardAddressIPv6 is assigned. + LabelEgressShardIPv6 string = "network.datumapis.com/egress-ipv6" + + // LabelEgressShardIPv4 marks a shard that translates to IPv4, set to + // LabelValueEgressFamilyServed whenever Spec.ShardAddressIPv4 is assigned. + LabelEgressShardIPv4 string = "network.datumapis.com/egress-ipv4" + + // LabelValueEgressFamilyServed is the only value the family labels carry. + // Absence, not a false value, means the family is not served: a selector + // requiring a family must match on presence so that a shard predating + // these labels never reads as serving one it does not. + LabelValueEgressFamilyServed string = "true" +) + +// ConditionTypeProgrammed indicates whether this node's datapath is +// translating with the addresses this spec assigns. It is distinct from +// Ready, which reports only that the datapath is attached: an attached +// datapath holding no assigned address, or a superseded one, drops or +// mis-sources every flow while reporting Ready. +const ConditionTypeProgrammed string = "Programmed" + +// Programmed sub-reasons — used as Programmed.Reason. +const ( + // ProgrammedReasonAddressesProgrammed indicates the datapath is + // translating with every address this spec assigns. + ProgrammedReasonAddressesProgrammed string = "AddressesProgrammed" + + // ProgrammedReasonAddressUnassigned indicates this spec assigns no + // address for any family, so the shard has nothing to translate to and + // claims no packet. + ProgrammedReasonAddressUnassigned string = "AddressUnassigned" + + // ProgrammedReasonProgrammingFailed indicates the shard could not program + // an assigned address into its datapath. + ProgrammedReasonProgrammingFailed string = "ProgrammingFailed" +) + // EgressShardSpec defines the desired state of an EgressShard. +// +// The address fields are written by the controller that owns the cell, not by +// the shard and not by a consumer. Each is optional: an address the addressing +// service has not yet handed out is absent rather than blank-but-required, so +// a shard object exists from the moment its node is labelled and gains its +// identity afterwards. +// +// +kubebuilder:validation:XValidation:rule="(has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0) == (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)",message="shardAddressIPv4 and nat64Prefix must be set together" type EgressShardSpec struct { // TargetRef identifies the Node this shard executes on. // +kubebuilder:validation:Required TargetRef TargetRef `json:"targetRef"` + + // ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this + // shard translates to — every NAT66 masquerade port it allocates lives + // within this address, so any node can route a reply to the owning shard + // using ordinary unicast routing on it alone, with no per-flow state + // lookup anywhere but that shard. + // + // Empty means no IPv6 address is assigned to this shard. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 6)",message="shardAddressIPv6 must be a valid IPv6 address" + ShardAddressIPv6 string `json:"shardAddressIPv6,omitempty"` + + // ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this + // shard translates to, and the source an IPv4-only destination sees. + // Unlike ShardAddressIPv6, reachability for it is not established by a + // BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the + // internet, so the underlay or an upstream announcement must attract this + // address to this node. + // + // Empty means no IPv4 address is assigned to this shard. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 4)",message="shardAddressIPv4 must be a valid IPv4 address" + ShardAddressIPv4 string `json:"shardAddressIPv4,omitempty"` + + // NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard + // translates to IPv4 — one Datum-operated Network-Specific Prefix, shared + // fabric-wide, never per-tenant. It must be the prefix the resolver + // synthesizes into; a shard translating for a different one is a + // blackhole with no symptom on either side. + // + // Set together with ShardAddressIPv4 or not at all: an address with no + // prefix has nothing to translate for, and a prefix with no address has + // nothing to translate into. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || isCIDR(self)",message="nat64Prefix must be a valid CIDR" + NAT64Prefix string `json:"nat64Prefix,omitempty"` } // EgressShardStatus defines the observed state of an EgressShard. // -// Every field here is echoed from what the shard's datapath process was -// actually started with, not derived: the shard publishes what it is running, -// so a status that disagrees with an operator's intent is a visible -// misconfiguration rather than a silently reconciled one. +// The address and prefix fields report what this node's datapath is programmed +// with, not what it was asked for. A value here that disagrees with the spec is +// a shard that has not converged; a value here with no counterpart in the spec +// is a shard still translating to an address nothing assigns any more. type EgressShardStatus struct { // ObservedGeneration is the .metadata.generation this status was computed from. // +optional @@ -70,50 +188,38 @@ type EgressShardStatus struct { // route against it. One SID serves both families: which translation a // packet gets is decided from the inner destination, not from a second // SID. + // + // Still chosen by an operator and reported here rather than assigned in + // spec, unlike the addresses: a value another node already uses silently + // diverts that node's traffic, so the assignment belongs to the + // addressing service, which does not hand out identifiers of this kind + // yet. // +optional // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 6)",message="shardSID must be a valid IPv6 address" ShardSID string `json:"shardSID,omitempty"` - // ShardAddressIPv6 is this shard's own dedicated, publicly-routable IPv6 - // address — every NAT66 masquerade port this shard allocates lives within - // it, so any node can route a reply to the correct shard using ordinary - // unicast routing on this address alone, with no per-flow state lookup - // anywhere but the owning shard itself. Operator-supplied per shard today - // (no in-cluster derivation mechanism yet — the same gap - // BGPRouter.Spec.SRv6Locator/NodeID assignment has today). - // - // Empty means this shard does not perform IPv6-to-IPv6 translation. + // ShardAddressIPv6 is the IPv6 masquerade source this shard's datapath is + // programmed with. Empty means it translates no IPv6 flow. // +optional // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 6)",message="shardAddressIPv6 must be a valid IPv6 address" ShardAddressIPv6 string `json:"shardAddressIPv6,omitempty"` - // ShardAddressIPv4 is this shard's own dedicated, publicly-routable IPv4 - // address — every NAT64 masquerade port this shard allocates lives within - // it, and it is the source an IPv4-only destination sees. Unlike - // ShardAddressIPv6, reachability for this address is not established by a - // BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the - // internet, so the address must be attracted to this node by the underlay - // or upstream announcement instead. Publishing it here is what makes that - // operator prerequisite checkable. - // - // Empty means this shard does not perform NAT64. + // ShardAddressIPv4 is the IPv4 masquerade source this shard's datapath is + // programmed with. Empty means it performs no NAT64. Publishing it is also + // what makes the underlay reachability prerequisite in + // Spec.ShardAddressIPv4 checkable. // +optional // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 4)",message="shardAddressIPv4 must be a valid IPv4 address" ShardAddressIPv4 string `json:"shardAddressIPv4,omitempty"` - // NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard - // translates to IPv4 — one Datum-operated Network-Specific Prefix, shared - // fabric-wide, never per-tenant. It is echoed here, rather than only - // existing as process configuration, because it is the single fact DNS64 - // synthesis has to agree with: a shard translating for a different prefix - // than the resolver synthesizes into is otherwise a silent blackhole. - // - // Empty whenever ShardAddressIPv4 is empty. + // NAT64Prefix is the prefix this shard's datapath is programmed to + // translate. Empty whenever ShardAddressIPv4 is empty. // +optional // +kubebuilder:validation:XValidation:rule="self == '' || isCIDR(self)",message="nat64Prefix must be a valid CIDR" NAT64Prefix string `json:"nat64Prefix,omitempty"` - // Conditions contains the standard conditions for this resource. + // Conditions contains the standard conditions for this resource, + // including Programmed (see ConditionTypeProgrammed). // // +listType=map // +listMapKey=type diff --git a/api/v1alpha1/egressshard_types_test.go b/api/v1alpha1/egressshard_types_test.go new file mode 100644 index 0000000..86a82c1 --- /dev/null +++ b/api/v1alpha1/egressshard_types_test.go @@ -0,0 +1,189 @@ +package v1alpha1 + +import ( + "encoding/json" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func newTestEgressShard() *EgressShard { + return &EgressShard{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "network.datumapis.com/v1alpha1", + Kind: "EgressShard", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-egress-shard", + Namespace: "galactic-system", + Labels: map[string]string{ + LabelEgressShardPool: "shared-public", + LabelEgressShardCell: "us-east-2-a", + LabelEgressShardIPv6: LabelValueEgressFamilyServed, + }, + }, + Spec: EgressShardSpec{ + TargetRef: TargetRef{Kind: "Node", Name: "node-a"}, + ShardAddressIPv6: "2001:db8:f00d::100", + }, + } +} + +// TestEgressShardDeepCopy verifies that DeepCopy produces an independent +// copy: mutations to the copy must not affect the original. +func TestEgressShardDeepCopy(t *testing.T) { + orig := newTestEgressShard() + dup := orig.DeepCopy() + + dup.Spec.ShardAddressIPv6 = "2001:db8:f00d::200" + dup.Spec.ShardAddressIPv4 = "198.51.100.7" + dup.Spec.NAT64Prefix = "64:ff9b::/96" + dup.Labels[LabelEgressShardPool] = "dedicated-public" + dup.Status.Conditions = append(dup.Status.Conditions, metav1.Condition{Type: ConditionTypeProgrammed}) + + if orig.Spec.ShardAddressIPv6 != "2001:db8:f00d::100" { + t.Errorf("ShardAddressIPv6 mutated: got %q", orig.Spec.ShardAddressIPv6) + } + if orig.Spec.ShardAddressIPv4 != "" { + t.Errorf("ShardAddressIPv4 mutated: got %q", orig.Spec.ShardAddressIPv4) + } + if orig.Spec.NAT64Prefix != "" { + t.Errorf("NAT64Prefix mutated: got %q", orig.Spec.NAT64Prefix) + } + if orig.Labels[LabelEgressShardPool] != "shared-public" { + t.Errorf("pool label mutated: got %q", orig.Labels[LabelEgressShardPool]) + } + if len(orig.Status.Conditions) != 0 { + t.Errorf("Conditions mutated: got %v", orig.Status.Conditions) + } +} + +// TestEgressShardDeepCopyNil verifies DeepCopy on a nil pointer returns nil. +func TestEgressShardDeepCopyNil(t *testing.T) { + var s *EgressShard + if s.DeepCopy() != nil { + t.Error("DeepCopy on nil pointer should return nil") + } +} + +// TestEgressShardJSONRoundTrip verifies that the spec-side addresses and the +// status-side programmed values survive JSON marshal/unmarshal independently +// of each other. +func TestEgressShardJSONRoundTrip(t *testing.T) { + orig := newTestEgressShard() + orig.Spec.ShardAddressIPv4 = "198.51.100.7" + orig.Spec.NAT64Prefix = "64:ff9b::/96" + orig.Status = EgressShardStatus{ + ObservedGeneration: 3, + ShardSID: "2001:db8:ff01::", + ShardAddressIPv6: "2001:db8:f00d::100", + Conditions: []metav1.Condition{ + { + Type: ConditionTypeProgrammed, + Status: metav1.ConditionFalse, + Reason: ProgrammedReasonProgrammingFailed, + }, + }, + } + + data, err := json.Marshal(orig) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + var got EgressShard + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + if got.Spec != orig.Spec { + t.Errorf("Spec: got %+v, want %+v", got.Spec, orig.Spec) + } + if got.Status.ShardSID != orig.Status.ShardSID { + t.Errorf("ShardSID: got %q, want %q", got.Status.ShardSID, orig.Status.ShardSID) + } + if got.Status.ShardAddressIPv6 != orig.Status.ShardAddressIPv6 { + t.Errorf("status ShardAddressIPv6: got %q, want %q", got.Status.ShardAddressIPv6, orig.Status.ShardAddressIPv6) + } + if got.Status.ShardAddressIPv4 != "" { + t.Errorf("status ShardAddressIPv4: got %q, want empty", got.Status.ShardAddressIPv4) + } + if len(got.Status.Conditions) != 1 || got.Status.Conditions[0].Reason != ProgrammedReasonProgrammingFailed { + t.Errorf("Conditions: got %+v", got.Status.Conditions) + } +} + +// TestEgressShardSpecFieldNames verifies the spec JSON keys, which the cell +// controller writes and the CRD's CEL rules name. +func TestEgressShardSpecFieldNames(t *testing.T) { + orig := newTestEgressShard() + orig.Spec.ShardAddressIPv4 = "198.51.100.7" + orig.Spec.NAT64Prefix = "64:ff9b::/96" + + data, err := json.Marshal(orig.Spec) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + var m map[string]any + if err := json.Unmarshal(data, &m); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + want := map[string]string{ + "shardAddressIPv6": "2001:db8:f00d::100", + "shardAddressIPv4": "198.51.100.7", + "nat64Prefix": "64:ff9b::/96", + } + for key, value := range want { + if raw, ok := m[key]; !ok || raw != value { + t.Errorf("%s: got %v, want %q", key, raw, value) + } + } + if _, ok := m["targetRef"]; !ok { + t.Error("expected \"targetRef\" key to be present") + } +} + +// TestEgressShardSpecOmitEmpty verifies that a shard with no address assigned +// yet omits every address key, which is what distinguishes an unclaimed +// address from a blank one. +func TestEgressShardSpecOmitEmpty(t *testing.T) { + spec := EgressShardSpec{TargetRef: TargetRef{Kind: "Node", Name: "node-a"}} + + data, err := json.Marshal(spec) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + var m map[string]any + if err := json.Unmarshal(data, &m); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + for _, key := range []string{"shardAddressIPv6", "shardAddressIPv4", "nat64Prefix"} { + if _, ok := m[key]; ok { + t.Errorf("expected %q key to be absent when empty", key) + } + } +} + +// TestEgressShardLabelKeys verifies the selection label keys stay in this +// API group's own namespace. A key naming another group would make a data +// plane resource carry a reference to the API that selects it. +func TestEgressShardLabelKeys(t *testing.T) { + cases := map[string]string{ + "pool": LabelEgressShardPool, + "cell": LabelEgressShardCell, + "ipv6": LabelEgressShardIPv6, + "ipv4": LabelEgressShardIPv4, + } + for name, key := range cases { + t.Run(name, func(t *testing.T) { + const prefix = "network.datumapis.com/" + if len(key) <= len(prefix) || key[:len(prefix)] != prefix { + t.Errorf("label key %q: want prefix %q", key, prefix) + } + }) + } +} diff --git a/config/crd/network.datumapis.com_egressshards.yaml b/config/crd/network.datumapis.com_egressshards.yaml index eb1e90e..733154c 100644 --- a/config/crd/network.datumapis.com_egressshards.yaml +++ b/config/crd/network.datumapis.com_egressshards.yaml @@ -48,10 +48,9 @@ spec: (IPv6 -> IPv4, RFC 6146) are the same function — stateful egress PAT with a VRF-scoped session table — over different families, so one shard object describes both rather than there being a second, near-duplicate kind. - Status.ShardAddressIPv6 and Status.ShardAddressIPv4 are each set only for - the family this shard actually translates; a shard serving only NAT66 - leaves the IPv4 field empty and behaves exactly as it did before NAT64 - existed. + Spec.ShardAddressIPv6 and Spec.ShardAddressIPv4 are each assigned only for + the family this shard translates; a shard with no IPv4 address performs no + NAT64. Every shard owns a dedicated, publicly-routable address per family it serves, and a flow's allocated masquerade port lives within it — so a reply @@ -59,6 +58,21 @@ spec: no hashing or cross-shard lookup on the return path at all (the "any node can determine the owning shard from the tuple alone" property, satisfied by construction rather than by a replicated hash table). + + A shard is told which addresses to translate to; it does not choose them. + The controller that owns a cell claims one address per family from the + addressing service and writes it into this spec, which keeps the + addressing-service credential off every translating node and keeps the + allocation request out of the path that attaches a workload. Status reports + what the node's datapath is actually programmed with, so an unclaimed + address, a stale datapath, and a divergence between the two are each + distinguishable. + + A shard names nothing that selects it. Which shards serve which consumer + intent is decided entirely by label selectors evaluated on the selecting + side (see the LabelEgressShard* keys), so no consumer-facing API appears in + this group and no reference points from a datapath resource back at the + resource that placed traffic on it. properties: apiVersion: description: |- @@ -78,8 +92,57 @@ spec: metadata: type: object spec: - description: EgressShardSpec defines the desired state of an EgressShard. + description: |- + EgressShardSpec defines the desired state of an EgressShard. + + The address fields are written by the controller that owns the cell, not by + the shard and not by a consumer. Each is optional: an address the addressing + service has not yet handed out is absent rather than blank-but-required, so + a shard object exists from the moment its node is labelled and gains its + identity afterwards. properties: + nat64Prefix: + description: |- + NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard + translates to IPv4 — one Datum-operated Network-Specific Prefix, shared + fabric-wide, never per-tenant. It must be the prefix the resolver + synthesizes into; a shard translating for a different one is a + blackhole with no symptom on either side. + + Set together with ShardAddressIPv4 or not at all: an address with no + prefix has nothing to translate for, and a prefix with no address has + nothing to translate into. + type: string + x-kubernetes-validations: + - message: nat64Prefix must be a valid CIDR + rule: self == '' || isCIDR(self) + shardAddressIPv4: + description: |- + ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this + shard translates to, and the source an IPv4-only destination sees. + Unlike ShardAddressIPv6, reachability for it is not established by a + BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the + internet, so the underlay or an upstream announcement must attract this + address to this node. + + Empty means no IPv4 address is assigned to this shard. + type: string + x-kubernetes-validations: + - message: shardAddressIPv4 must be a valid IPv4 address + rule: self == '' || (isIP(self) && ip(self).family() == 4) + shardAddressIPv6: + description: |- + ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this + shard translates to — every NAT66 masquerade port it allocates lives + within this address, so any node can route a reply to the owning shard + using ordinary unicast routing on it alone, with no per-flow state + lookup anywhere but that shard. + + Empty means no IPv6 address is assigned to this shard. + type: string + x-kubernetes-validations: + - message: shardAddressIPv6 must be a valid IPv6 address + rule: self == '' || (isIP(self) && ip(self).family() == 6) targetRef: description: TargetRef identifies the Node this shard executes on. properties: @@ -98,18 +161,23 @@ spec: required: - targetRef type: object + x-kubernetes-validations: + - message: shardAddressIPv4 and nat64Prefix must be set together + rule: (has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0) + == (has(self.nat64Prefix) && size(self.nat64Prefix) > 0) status: description: |- EgressShardStatus defines the observed state of an EgressShard. - Every field here is echoed from what the shard's datapath process was - actually started with, not derived: the shard publishes what it is running, - so a status that disagrees with an operator's intent is a visible - misconfiguration rather than a silently reconciled one. + The address and prefix fields report what this node's datapath is programmed + with, not what it was asked for. A value here that disagrees with the spec is + a shard that has not converged; a value here with no counterpart in the spec + is a shard still translating to an address nothing assigns any more. properties: conditions: - description: Conditions contains the standard conditions for this - resource. + description: |- + Conditions contains the standard conditions for this resource, + including Programmed (see ConditionTypeProgrammed). items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -170,14 +238,8 @@ spec: x-kubernetes-list-type: map nat64Prefix: description: |- - NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard - translates to IPv4 — one Datum-operated Network-Specific Prefix, shared - fabric-wide, never per-tenant. It is echoed here, rather than only - existing as process configuration, because it is the single fact DNS64 - synthesis has to agree with: a shard translating for a different prefix - than the resolver synthesizes into is otherwise a silent blackhole. - - Empty whenever ShardAddressIPv4 is empty. + NAT64Prefix is the prefix this shard's datapath is programmed to + translate. Empty whenever ShardAddressIPv4 is empty. type: string x-kubernetes-validations: - message: nat64Prefix must be a valid CIDR @@ -189,31 +251,18 @@ spec: type: integer shardAddressIPv4: description: |- - ShardAddressIPv4 is this shard's own dedicated, publicly-routable IPv4 - address — every NAT64 masquerade port this shard allocates lives within - it, and it is the source an IPv4-only destination sees. Unlike - ShardAddressIPv6, reachability for this address is not established by a - BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the - internet, so the address must be attracted to this node by the underlay - or upstream announcement instead. Publishing it here is what makes that - operator prerequisite checkable. - - Empty means this shard does not perform NAT64. + ShardAddressIPv4 is the IPv4 masquerade source this shard's datapath is + programmed with. Empty means it performs no NAT64. Publishing it is also + what makes the underlay reachability prerequisite in + Spec.ShardAddressIPv4 checkable. type: string x-kubernetes-validations: - message: shardAddressIPv4 must be a valid IPv4 address rule: self == '' || (isIP(self) && ip(self).family() == 4) shardAddressIPv6: description: |- - ShardAddressIPv6 is this shard's own dedicated, publicly-routable IPv6 - address — every NAT66 masquerade port this shard allocates lives within - it, so any node can route a reply to the correct shard using ordinary - unicast routing on this address alone, with no per-flow state lookup - anywhere but the owning shard itself. Operator-supplied per shard today - (no in-cluster derivation mechanism yet — the same gap - BGPRouter.Spec.SRv6Locator/NodeID assignment has today). - - Empty means this shard does not perform IPv6-to-IPv6 translation. + ShardAddressIPv6 is the IPv6 masquerade source this shard's datapath is + programmed with. Empty means it translates no IPv6 flow. type: string x-kubernetes-validations: - message: shardAddressIPv6 must be a valid IPv6 address @@ -228,6 +277,12 @@ spec: route against it. One SID serves both families: which translation a packet gets is decided from the inner destination, not from a second SID. + + Still chosen by an operator and reported here rather than assigned in + spec, unlike the addresses: a value another node already uses silently + diverts that node's traffic, so the assignment belongs to the + addressing service, which does not hand out identifiers of this kind + yet. type: string x-kubernetes-validations: - message: shardSID must be a valid IPv6 address diff --git a/docs/api/bgp.md b/docs/api/bgp.md index 09871e7..11d403d 100644 --- a/docs/api/bgp.md +++ b/docs/api/bgp.md @@ -932,10 +932,9 @@ A shard serves one or both address families. NAT66 (IPv6 -> IPv6) and NAT64 (IPv6 -> IPv4, RFC 6146) are the same function — stateful egress PAT with a VRF-scoped session table — over different families, so one shard object describes both rather than there being a second, near-duplicate kind. -Status.ShardAddressIPv6 and Status.ShardAddressIPv4 are each set only for -the family this shard actually translates; a shard serving only NAT66 -leaves the IPv4 field empty and behaves exactly as it did before NAT64 -existed. +Spec.ShardAddressIPv6 and Spec.ShardAddressIPv4 are each assigned only for +the family this shard translates; a shard with no IPv4 address performs no +NAT64. Every shard owns a dedicated, publicly-routable address per family it serves, and a flow's allocated masquerade port lives within it — so a reply @@ -944,6 +943,21 @@ no hashing or cross-shard lookup on the return path at all (the "any node can determine the owning shard from the tuple alone" property, satisfied by construction rather than by a replicated hash table). +A shard is told which addresses to translate to; it does not choose them. +The controller that owns a cell claims one address per family from the +addressing service and writes it into this spec, which keeps the +addressing-service credential off every translating node and keeps the +allocation request out of the path that attaches a workload. Status reports +what the node's datapath is actually programmed with, so an unclaimed +address, a stale datapath, and a divergence between the two are each +distinguishable. + +A shard names nothing that selects it. Which shards serve which consumer +intent is decided entirely by label selectors evaluated on the selecting +side (see the LabelEgressShard* keys), so no consumer-facing API appears in +this group and no reference points from a datapath resource back at the +resource that placed traffic on it. + @@ -965,6 +979,12 @@ construction rather than by a replicated hash table). EgressShardSpec defines the desired state of an EgressShard. +The address fields are written by the controller that owns the cell, not by +the shard and not by a consumer. Each is optional: an address the addressing +service has not yet handed out is absent rather than blank-but-required, so +a shard object exists from the moment its node is labelled and gains its +identity afterwards. + _Appears in:_ @@ -973,6 +993,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| +| `shardAddressIPv6` _string_ | ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this
shard translates to — every NAT66 masquerade port it allocates lives
within this address, so any node can route a reply to the owning shard
using ordinary unicast routing on it alone, with no per-flow state
lookup anywhere but that shard.
Empty means no IPv6 address is assigned to this shard. | | | +| `shardAddressIPv4` _string_ | ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this
shard translates to, and the source an IPv4-only destination sees.
Unlike ShardAddressIPv6, reachability for it is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the underlay or an upstream announcement must attract this
address to this node.
Empty means no IPv4 address is assigned to this shard. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. | | | #### EgressShardStatus @@ -981,10 +1004,10 @@ _Appears in:_ EgressShardStatus defines the observed state of an EgressShard. -Every field here is echoed from what the shard's datapath process was -actually started with, not derived: the shard publishes what it is running, -so a status that disagrees with an operator's intent is a visible -misconfiguration rather than a silently reconciled one. +The address and prefix fields report what this node's datapath is programmed +with, not what it was asked for. A value here that disagrees with the spec is +a shard that has not converged; a value here with no counterpart in the spec +is a shard still translating to an address nothing assigns any more. @@ -994,11 +1017,11 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | -| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID. | | | -| `shardAddressIPv6` _string_ | ShardAddressIPv6 is this shard's own dedicated, publicly-routable IPv6
address — every NAT66 masquerade port this shard allocates lives within
it, so any node can route a reply to the correct shard using ordinary
unicast routing on this address alone, with no per-flow state lookup
anywhere but the owning shard itself. Operator-supplied per shard today
(no in-cluster derivation mechanism yet — the same gap
BGPRouter.Spec.SRv6Locator/NodeID assignment has today).
Empty means this shard does not perform IPv6-to-IPv6 translation. | | | -| `shardAddressIPv4` _string_ | ShardAddressIPv4 is this shard's own dedicated, publicly-routable IPv4
address — every NAT64 masquerade port this shard allocates lives within
it, and it is the source an IPv4-only destination sees. Unlike
ShardAddressIPv6, reachability for this address is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the address must be attracted to this node by the underlay
or upstream announcement instead. Publishing it here is what makes that
operator prerequisite checkable.
Empty means this shard does not perform NAT64. | | | -| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It is echoed here, rather than only
existing as process configuration, because it is the single fact DNS64
synthesis has to agree with: a shard translating for a different prefix
than the resolver synthesizes into is otherwise a silent blackhole.
Empty whenever ShardAddressIPv4 is empty. | | | -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource. | | | +| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID.
Still chosen by an operator and reported here rather than assigned in
spec, unlike the addresses: a value another node already uses silently
diverts that node's traffic, so the assignment belongs to the
addressing service, which does not hand out identifiers of this kind
yet. | | | +| `shardAddressIPv6` _string_ | ShardAddressIPv6 is the IPv6 masquerade source this shard's datapath is
programmed with. Empty means it translates no IPv6 flow. | | | +| `shardAddressIPv4` _string_ | ShardAddressIPv4 is the IPv4 masquerade source this shard's datapath is
programmed with. Empty means it performs no NAT64. Publishing it is also
what makes the underlay reachability prerequisite in
Spec.ShardAddressIPv4 checkable. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the prefix this shard's datapath is programmed to
translate. Empty whenever ShardAddressIPv4 is empty. | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource,
including Programmed (see ConditionTypeProgrammed). | | | #### ExtendedCommunitySet diff --git a/docs/api/gateway.md b/docs/api/gateway.md index e00b33d..5622082 100644 --- a/docs/api/gateway.md +++ b/docs/api/gateway.md @@ -33,10 +33,9 @@ A shard serves one or both address families. NAT66 (IPv6 -> IPv6) and NAT64 (IPv6 -> IPv4, RFC 6146) are the same function — stateful egress PAT with a VRF-scoped session table — over different families, so one shard object describes both rather than there being a second, near-duplicate kind. -Status.ShardAddressIPv6 and Status.ShardAddressIPv4 are each set only for -the family this shard actually translates; a shard serving only NAT66 -leaves the IPv4 field empty and behaves exactly as it did before NAT64 -existed. +Spec.ShardAddressIPv6 and Spec.ShardAddressIPv4 are each assigned only for +the family this shard translates; a shard with no IPv4 address performs no +NAT64. Every shard owns a dedicated, publicly-routable address per family it serves, and a flow's allocated masquerade port lives within it — so a reply @@ -45,6 +44,21 @@ no hashing or cross-shard lookup on the return path at all (the "any node can determine the owning shard from the tuple alone" property, satisfied by construction rather than by a replicated hash table). +A shard is told which addresses to translate to; it does not choose them. +The controller that owns a cell claims one address per family from the +addressing service and writes it into this spec, which keeps the +addressing-service credential off every translating node and keeps the +allocation request out of the path that attaches a workload. Status reports +what the node's datapath is actually programmed with, so an unclaimed +address, a stale datapath, and a divergence between the two are each +distinguishable. + +A shard names nothing that selects it. Which shards serve which consumer +intent is decided entirely by label selectors evaluated on the selecting +side (see the LabelEgressShard* keys), so no consumer-facing API appears in +this group and no reference points from a datapath resource back at the +resource that placed traffic on it. + @@ -66,6 +80,12 @@ construction rather than by a replicated hash table). EgressShardSpec defines the desired state of an EgressShard. +The address fields are written by the controller that owns the cell, not by +the shard and not by a consumer. Each is optional: an address the addressing +service has not yet handed out is absent rather than blank-but-required, so +a shard object exists from the moment its node is labelled and gains its +identity afterwards. + _Appears in:_ @@ -74,6 +94,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| +| `shardAddressIPv6` _string_ | ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this
shard translates to — every NAT66 masquerade port it allocates lives
within this address, so any node can route a reply to the owning shard
using ordinary unicast routing on it alone, with no per-flow state
lookup anywhere but that shard.
Empty means no IPv6 address is assigned to this shard. | | | +| `shardAddressIPv4` _string_ | ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this
shard translates to, and the source an IPv4-only destination sees.
Unlike ShardAddressIPv6, reachability for it is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the underlay or an upstream announcement must attract this
address to this node.
Empty means no IPv4 address is assigned to this shard. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. | | | #### EgressShardStatus @@ -82,10 +105,10 @@ _Appears in:_ EgressShardStatus defines the observed state of an EgressShard. -Every field here is echoed from what the shard's datapath process was -actually started with, not derived: the shard publishes what it is running, -so a status that disagrees with an operator's intent is a visible -misconfiguration rather than a silently reconciled one. +The address and prefix fields report what this node's datapath is programmed +with, not what it was asked for. A value here that disagrees with the spec is +a shard that has not converged; a value here with no counterpart in the spec +is a shard still translating to an address nothing assigns any more. @@ -95,11 +118,11 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | -| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID. | | | -| `shardAddressIPv6` _string_ | ShardAddressIPv6 is this shard's own dedicated, publicly-routable IPv6
address — every NAT66 masquerade port this shard allocates lives within
it, so any node can route a reply to the correct shard using ordinary
unicast routing on this address alone, with no per-flow state lookup
anywhere but the owning shard itself. Operator-supplied per shard today
(no in-cluster derivation mechanism yet — the same gap
BGPRouter.Spec.SRv6Locator/NodeID assignment has today).
Empty means this shard does not perform IPv6-to-IPv6 translation. | | | -| `shardAddressIPv4` _string_ | ShardAddressIPv4 is this shard's own dedicated, publicly-routable IPv4
address — every NAT64 masquerade port this shard allocates lives within
it, and it is the source an IPv4-only destination sees. Unlike
ShardAddressIPv6, reachability for this address is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the address must be attracted to this node by the underlay
or upstream announcement instead. Publishing it here is what makes that
operator prerequisite checkable.
Empty means this shard does not perform NAT64. | | | -| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It is echoed here, rather than only
existing as process configuration, because it is the single fact DNS64
synthesis has to agree with: a shard translating for a different prefix
than the resolver synthesizes into is otherwise a silent blackhole.
Empty whenever ShardAddressIPv4 is empty. | | | -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource. | | | +| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID.
Still chosen by an operator and reported here rather than assigned in
spec, unlike the addresses: a value another node already uses silently
diverts that node's traffic, so the assignment belongs to the
addressing service, which does not hand out identifiers of this kind
yet. | | | +| `shardAddressIPv6` _string_ | ShardAddressIPv6 is the IPv6 masquerade source this shard's datapath is
programmed with. Empty means it translates no IPv6 flow. | | | +| `shardAddressIPv4` _string_ | ShardAddressIPv4 is the IPv4 masquerade source this shard's datapath is
programmed with. Empty means it performs no NAT64. Publishing it is also
what makes the underlay reachability prerequisite in
Spec.ShardAddressIPv4 checkable. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the prefix this shard's datapath is programmed to
translate. Empty whenever ShardAddressIPv4 is empty. | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource,
including Programmed (see ConditionTypeProgrammed). | | | From 45eb71a1eb0c954495ccf624a2c8bba163e71a38 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 17 Sep 2026 15:31:26 -0500 Subject: [PATCH 2/4] feat(api): Make an assigned shard address write-once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A shard's datapath claims a reply by exact match against the address it translates to, so the instant a different address is written, the return traffic of every flow already established through that shard stops being claimed and falls through. There is no drain, no dual-address grace period, and no way for a shard to keep claiming the address it is leaving, so reassignment is a silent break of live connections. Reject the change instead. A shard holding the wrong address is deleted and recreated, which breaks the same flows at a moment someone chose. The alternative that costs nothing — hold old and new at once — is a datapath change this API cannot make, and forbidding the transition now is what keeps the choice open: relaxing an immutability rule later breaks no client, while adding one later breaks any client that had started rewriting. Unassigning is rejected on the same grounds, and because status reports what the datapath programmed: a cleared spec with a populated status is a shard still advertising an address nothing assigns any more. A field-scoped rule cannot see its own absence, so it catches a changed value but not a deleted key. The unassignment rules are therefore scoped to spec, where oldSelf still carries the old object. Status carries no transition rule at all — it tracks reality and has to converge. Key changes: - Reject any change to an assigned shardAddressIPv6, shardAddressIPv4, or nat64Prefix, permitting the first write, which every existing object needs since all of them carry an empty spec - Reject unassigning any of the three once assigned - Add a chainsaw scenario covering first write, same-value rewrite, changed value, unassignment, the unpaired IPv4 address, and a status that still changes - Wait for the EgressShard CRD in the e2e suite, which applied it but never waited for it to be established Co-Authored-By: Claude Opus 5 (1M context) --- api/v1alpha1/egressshard_types.go | 17 +- .../network.datumapis.com_egressshards.yaml | 28 ++- docs/api/bgp.md | 9 +- docs/api/gateway.md | 9 +- test/e2e/Taskfile.yaml | 1 + .../chainsaw-test.yaml | 179 ++++++++++++++++++ 6 files changed, 239 insertions(+), 4 deletions(-) create mode 100644 test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml diff --git a/api/v1alpha1/egressshard_types.go b/api/v1alpha1/egressshard_types.go index 5ac4fe0..307548b 100644 --- a/api/v1alpha1/egressshard_types.go +++ b/api/v1alpha1/egressshard_types.go @@ -126,7 +126,17 @@ const ( // a shard object exists from the moment its node is labelled and gains its // identity afterwards. // +// Each is also write-once, and cannot be unassigned once assigned. The +// datapath claims a reply by exact match against the address it translates to, +// so reassigning one strands the return traffic of every flow already +// established through it, with no drain and no dual-address grace period +// available to cover the change. A shard holding the wrong address is deleted +// and recreated instead, which breaks those flows at a moment someone chose. +// // +kubebuilder:validation:XValidation:rule="(has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0) == (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)",message="shardAddressIPv4 and nat64Prefix must be set together" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.shardAddressIPv6) && size(oldSelf.shardAddressIPv6) > 0) || (has(self.shardAddressIPv6) && size(self.shardAddressIPv6) > 0)",message="shardAddressIPv6 cannot be unassigned once assigned" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.shardAddressIPv4) && size(oldSelf.shardAddressIPv4) > 0) || (has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0)",message="shardAddressIPv4 cannot be unassigned once assigned" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.nat64Prefix) && size(oldSelf.nat64Prefix) > 0) || (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)",message="nat64Prefix cannot be unassigned once assigned" type EgressShardSpec struct { // TargetRef identifies the Node this shard executes on. // +kubebuilder:validation:Required @@ -141,6 +151,7 @@ type EgressShardSpec struct { // Empty means no IPv6 address is assigned to this shard. // +optional // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 6)",message="shardAddressIPv6 must be a valid IPv6 address" + // +kubebuilder:validation:XValidation:rule="self == oldSelf || oldSelf == ''",message="shardAddressIPv6 is immutable once assigned" ShardAddressIPv6 string `json:"shardAddressIPv6,omitempty"` // ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this @@ -153,6 +164,7 @@ type EgressShardSpec struct { // Empty means no IPv4 address is assigned to this shard. // +optional // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 4)",message="shardAddressIPv4 must be a valid IPv4 address" + // +kubebuilder:validation:XValidation:rule="self == oldSelf || oldSelf == ''",message="shardAddressIPv4 is immutable once assigned" ShardAddressIPv4 string `json:"shardAddressIPv4,omitempty"` // NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard @@ -163,9 +175,12 @@ type EgressShardSpec struct { // // Set together with ShardAddressIPv4 or not at all: an address with no // prefix has nothing to translate for, and a prefix with no address has - // nothing to translate into. + // nothing to translate into. Write-once for the same reason the addresses + // are: a shard that starts translating a different prefix blackholes every + // destination the resolver already synthesized into the old one. // +optional // +kubebuilder:validation:XValidation:rule="self == '' || isCIDR(self)",message="nat64Prefix must be a valid CIDR" + // +kubebuilder:validation:XValidation:rule="self == oldSelf || oldSelf == ''",message="nat64Prefix is immutable once assigned" NAT64Prefix string `json:"nat64Prefix,omitempty"` } diff --git a/config/crd/network.datumapis.com_egressshards.yaml b/config/crd/network.datumapis.com_egressshards.yaml index 733154c..fadbf21 100644 --- a/config/crd/network.datumapis.com_egressshards.yaml +++ b/config/crd/network.datumapis.com_egressshards.yaml @@ -100,6 +100,13 @@ spec: service has not yet handed out is absent rather than blank-but-required, so a shard object exists from the moment its node is labelled and gains its identity afterwards. + + Each is also write-once, and cannot be unassigned once assigned. The + datapath claims a reply by exact match against the address it translates to, + so reassigning one strands the return traffic of every flow already + established through it, with no drain and no dual-address grace period + available to cover the change. A shard holding the wrong address is deleted + and recreated instead, which breaks those flows at a moment someone chose. properties: nat64Prefix: description: |- @@ -111,11 +118,15 @@ spec: Set together with ShardAddressIPv4 or not at all: an address with no prefix has nothing to translate for, and a prefix with no address has - nothing to translate into. + nothing to translate into. Write-once for the same reason the addresses + are: a shard that starts translating a different prefix blackholes every + destination the resolver already synthesized into the old one. type: string x-kubernetes-validations: - message: nat64Prefix must be a valid CIDR rule: self == '' || isCIDR(self) + - message: nat64Prefix is immutable once assigned + rule: self == oldSelf || oldSelf == '' shardAddressIPv4: description: |- ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this @@ -130,6 +141,8 @@ spec: x-kubernetes-validations: - message: shardAddressIPv4 must be a valid IPv4 address rule: self == '' || (isIP(self) && ip(self).family() == 4) + - message: shardAddressIPv4 is immutable once assigned + rule: self == oldSelf || oldSelf == '' shardAddressIPv6: description: |- ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this @@ -143,6 +156,8 @@ spec: x-kubernetes-validations: - message: shardAddressIPv6 must be a valid IPv6 address rule: self == '' || (isIP(self) && ip(self).family() == 6) + - message: shardAddressIPv6 is immutable once assigned + rule: self == oldSelf || oldSelf == '' targetRef: description: TargetRef identifies the Node this shard executes on. properties: @@ -165,6 +180,17 @@ spec: - message: shardAddressIPv4 and nat64Prefix must be set together rule: (has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0) == (has(self.nat64Prefix) && size(self.nat64Prefix) > 0) + - message: shardAddressIPv6 cannot be unassigned once assigned + rule: '!(has(oldSelf.shardAddressIPv6) && size(oldSelf.shardAddressIPv6) + > 0) || (has(self.shardAddressIPv6) && size(self.shardAddressIPv6) + > 0)' + - message: shardAddressIPv4 cannot be unassigned once assigned + rule: '!(has(oldSelf.shardAddressIPv4) && size(oldSelf.shardAddressIPv4) + > 0) || (has(self.shardAddressIPv4) && size(self.shardAddressIPv4) + > 0)' + - message: nat64Prefix cannot be unassigned once assigned + rule: '!(has(oldSelf.nat64Prefix) && size(oldSelf.nat64Prefix) > 0) + || (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)' status: description: |- EgressShardStatus defines the observed state of an EgressShard. diff --git a/docs/api/bgp.md b/docs/api/bgp.md index 11d403d..52ea53c 100644 --- a/docs/api/bgp.md +++ b/docs/api/bgp.md @@ -985,6 +985,13 @@ service has not yet handed out is absent rather than blank-but-required, so a shard object exists from the moment its node is labelled and gains its identity afterwards. +Each is also write-once, and cannot be unassigned once assigned. The +datapath claims a reply by exact match against the address it translates to, +so reassigning one strands the return traffic of every flow already +established through it, with no drain and no dual-address grace period +available to cover the change. A shard holding the wrong address is deleted +and recreated instead, which breaks those flows at a moment someone chose. + _Appears in:_ @@ -995,7 +1002,7 @@ _Appears in:_ | `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| | `shardAddressIPv6` _string_ | ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this
shard translates to — every NAT66 masquerade port it allocates lives
within this address, so any node can route a reply to the owning shard
using ordinary unicast routing on it alone, with no per-flow state
lookup anywhere but that shard.
Empty means no IPv6 address is assigned to this shard. | | | | `shardAddressIPv4` _string_ | ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this
shard translates to, and the source an IPv4-only destination sees.
Unlike ShardAddressIPv6, reachability for it is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the underlay or an upstream announcement must attract this
address to this node.
Empty means no IPv4 address is assigned to this shard. | | | -| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | #### EgressShardStatus diff --git a/docs/api/gateway.md b/docs/api/gateway.md index 5622082..7239c04 100644 --- a/docs/api/gateway.md +++ b/docs/api/gateway.md @@ -86,6 +86,13 @@ service has not yet handed out is absent rather than blank-but-required, so a shard object exists from the moment its node is labelled and gains its identity afterwards. +Each is also write-once, and cannot be unassigned once assigned. The +datapath claims a reply by exact match against the address it translates to, +so reassigning one strands the return traffic of every flow already +established through it, with no drain and no dual-address grace period +available to cover the change. A shard holding the wrong address is deleted +and recreated instead, which breaks those flows at a moment someone chose. + _Appears in:_ @@ -96,7 +103,7 @@ _Appears in:_ | `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| | `shardAddressIPv6` _string_ | ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this
shard translates to — every NAT66 masquerade port it allocates lives
within this address, so any node can route a reply to the owning shard
using ordinary unicast routing on it alone, with no per-flow state
lookup anywhere but that shard.
Empty means no IPv6 address is assigned to this shard. | | | | `shardAddressIPv4` _string_ | ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this
shard translates to, and the source an IPv4-only destination sees.
Unlike ShardAddressIPv6, reachability for it is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the underlay or an upstream announcement must attract this
address to this node.
Empty means no IPv4 address is assigned to this shard. | | | -| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | #### EgressShardStatus diff --git a/test/e2e/Taskfile.yaml b/test/e2e/Taskfile.yaml index 79b92e8..2c057fd 100644 --- a/test/e2e/Taskfile.yaml +++ b/test/e2e/Taskfile.yaml @@ -91,6 +91,7 @@ tasks: crd/bgppolicies.network.datumapis.com crd/bgppeers.network.datumapis.com crd/bgpvrfinstances.network.datumapis.com + crd/egressshards.network.datumapis.com --timeout=60s deploy-bgp-routers: diff --git a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml new file mode 100644 index 0000000..487557c --- /dev/null +++ b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml @@ -0,0 +1,179 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: egress-shard-address-immutability +spec: + description: > + Verify the CEL rules governing an EgressShard's assigned egress addresses: + a shard is accepted with no address, the cell controller's first write is + accepted, rewriting the same value is accepted, and changing or unassigning + an assigned value is rejected. Also verifies that status remains writable, + since status reports what the datapath actually programmed and must still + converge. Does not require any operator to be running — every rule under + test is enforced by the API server alone. + steps: + - name: accept-shard-with-no-address + try: + - apply: + resource: + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard + labels: + network.datumapis.com/egress-pool: e2e-shared + network.datumapis.com/egress-cell: e2e-cell + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + - assert: + resource: + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard + (spec.shardAddressIPv6 == null): true + + - name: accept-first-address-assignment + try: + - script: + content: | + set -e + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardAddressIPv6":"2001:db8:f00d::100","shardAddressIPv4":"198.51.100.7","nat64Prefix":"64:ff9b::/96"}}' + echo "OK: first assignment accepted" + - assert: + resource: + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard + spec: + shardAddressIPv6: 2001:db8:f00d::100 + shardAddressIPv4: 198.51.100.7 + nat64Prefix: 64:ff9b::/96 + + - name: accept-same-value-rewrite + try: + - script: + content: | + set -e + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardAddressIPv6":"2001:db8:f00d::100"}}' + echo "OK: rewriting the same address accepted" + + - name: reject-address-change + try: + - script: + content: | + set +e + OUTPUT=$(kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardAddressIPv6":"2001:db8:f00d::200"}}' 2>&1) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of a changed shardAddressIPv6 but the patch was accepted" + exit 1 + fi + case "$OUTPUT" in + *"immutable once assigned"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: changing an assigned shardAddressIPv6 correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-prefix-change + try: + - script: + content: | + set +e + OUTPUT=$(kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"nat64Prefix":"64:ff9b:1::/96"}}' 2>&1) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of a changed nat64Prefix but the patch was accepted" + exit 1 + fi + echo "OK: changing an assigned nat64Prefix correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-address-unassignment + try: + - script: + content: | + set +e + OUTPUT=$(kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardAddressIPv6":null}}' 2>&1) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of an unassigned shardAddressIPv6 but the patch was accepted" + exit 1 + fi + case "$OUTPUT" in + *"cannot be unassigned once assigned"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: unassigning an assigned shardAddressIPv6 correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-ipv4-without-prefix + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -f - 2>&1 <<'EOF' + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard-unpaired + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + shardAddressIPv4: 198.51.100.8 + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of an IPv4 address with no NAT64 prefix but the resource was accepted" + kubectl delete egressshard e2e-egress-shard-unpaired 2>/dev/null || true + exit 1 + fi + case "$OUTPUT" in + *"must be set together"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: an IPv4 address with no NAT64 prefix correctly rejected" + echo "Server response: $OUTPUT" + + - name: accept-status-changes + try: + - script: + content: | + set -e + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" \ + --subresource=status --type=merge -p ' + {"status":{"shardAddressIPv6":"2001:db8:f00d::100","shardSID":"2001:db8:ff01::"}}' + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" \ + --subresource=status --type=merge -p ' + {"status":{"shardAddressIPv6":"2001:db8:f00d::200"}}' + echo "OK: status reports what the datapath programmed and stays mutable" + - assert: + resource: + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard + status: + shardAddressIPv6: 2001:db8:f00d::200 + + finally: + - delete: + ref: + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + name: e2e-egress-shard From affb112b4eea4b48c9b0472426c61005d5ef90c4 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 17 Sep 2026 21:31:40 -0500 Subject: [PATCH 3/4] feat(api): Record the claim an egress address came from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An address in a shard's spec says nothing about where it came from, and the claim it came from cannot say either: the addressing service overwrites a claim's spec.ownerRef on create with the identity that requested it, so a claim a cell controller made is attributed to that controller and not to the shard it was made for. An operator asking which claim holds a shard's address has nothing to follow. Record the claim on the shard, which is the only end of that relationship able to hold it. The reference does not replace the address. The addressing service is an aggregated API served on a project control-plane path and defines no CustomResourceDefinition, so no claim object exists in the cluster a shard runs in; resolving one needs a credential on every translating node, which is what putting the address in spec exists to avoid. The spec therefore carries both: the address the datapath can program, and the claim that accounts for it. A reference is not required with an address, in either direction, and this is deliberate. Requiring the address first would forbid recording a claim whose allocation has not resolved, the window a restarting controller most needs so that it finds the claim it already made rather than making a second one. Requiring the reference first would forbid an address allocated by hand, deciding by validation whether such an address may be recorded at all — a migration question still open. Key changes: - Add AddressClaimRef, an opaque reference carrying apiGroup, kind, project, namespace, and name, importing no addressing-service type - Require the project, because a claim is namespaced within one and this reference is read from outside every project - Default apiGroup and kind to ipam.miloapis.com and IPClaim, so the common case needs three fields - Add spec.shardAddressIPv6ClaimRef and spec.shardAddressIPv4ClaimRef, one per address, each write-once and unable to be unassigned - Extend the chainsaw scenario over the reference rules, the defaults, and a reference recorded before its address resolves Co-Authored-By: Claude Opus 5 (1M context) --- api/v1alpha1/egressshard_types.go | 96 ++++++++++++ api/v1alpha1/egressshard_types_test.go | 41 ++++- api/v1alpha1/zz_generated.deepcopy.go | 27 +++- .../network.datumapis.com_egressshards.yaml | 110 ++++++++++++++ docs/api/bgp.md | 60 ++++++++ docs/api/gateway.md | 60 ++++++++ .../chainsaw-test.yaml | 141 +++++++++++++++++- 7 files changed, 527 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/egressshard_types.go b/api/v1alpha1/egressshard_types.go index 307548b..6c336d3 100644 --- a/api/v1alpha1/egressshard_types.go +++ b/api/v1alpha1/egressshard_types.go @@ -118,6 +118,66 @@ const ( ProgrammedReasonProgrammingFailed string = "ProgrammingFailed" ) +// AddressClaimRef names the addressing-service claim an assigned address came +// from. It is a record, not a lookup: the addressing service is an aggregated +// API served on a project control-plane path and defines no CustomResourceDefinition, +// so no claim object exists in the cluster a shard runs in and nothing here +// resolves one. Reading it means addressing that project's control plane +// directly, with a credential a translating node is deliberately not given. +// +// Every field is a plain string and this group depends on no addressing-service +// type. The shape matches that service's own opaque cross-API reference, plus +// the project, because a claim is namespaced within a project and a namespace +// name alone does not identify one from outside. +// +// Populate it with the coordinates of the claim whose allocation produced the +// address written alongside it: +// +// shardAddressIPv6: 2001:db8:f00d::100 +// shardAddressIPv6ClaimRef: +// project: datum-network-edge +// namespace: egress +// name: egress-shard-node-a-ipv6 +// +// The claim cannot record this relationship itself. The addressing service +// overwrites a claim's spec.ownerRef on create with the identity that requested +// it, so a claim made by a cell controller is attributed to that controller's +// project rather than to the shard it was made for. Attribution therefore runs +// in this direction only, and an operator asking which claim holds a shard's +// address has this field or nothing. +type AddressClaimRef struct { + // APIGroup is the group of the claim resource. + // +optional + // +kubebuilder:default="ipam.miloapis.com" + // +kubebuilder:validation:MinLength=1 + APIGroup string `json:"apiGroup,omitempty"` + + // Kind is the kind of the claim resource. + // +optional + // +kubebuilder:default="IPClaim" + // +kubebuilder:validation:MinLength=1 + Kind string `json:"kind,omitempty"` + + // Project is the project whose control plane serves the claim. Required: + // a claim is namespaced within a project, and this reference is read from + // outside every project. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Project string `json:"project"` + + // Namespace is the namespace holding the claim within Project. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Namespace string `json:"namespace"` + + // Name is the name of the claim. The addressing service derives it from + // the thing being addressed, so it is stable across a shard's lifetime and + // is what a replacement finds again. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` +} + // EgressShardSpec defines the desired state of an EgressShard. // // The address fields are written by the controller that owns the cell, not by @@ -133,10 +193,24 @@ const ( // available to cover the change. A shard holding the wrong address is deleted // and recreated instead, which breaks those flows at a moment someone chose. // +// A claim reference alongside an address records where that address came from, +// and is write-once on the same terms. It is deliberately not required with an +// address, in either direction. Requiring an address before a claim reference +// would forbid recording a claim whose allocation has not resolved yet, which +// is the window a restarting controller most needs to see so that it finds the +// claim it already made instead of making a second one. Requiring a claim +// reference before an address would forbid an address allocated by hand, and +// whether such an address may be recorded here at all is an open migration +// question this validation would answer by fiat. An address with no reference +// is therefore accepted and is unattributable, which is a fact about the +// allocation rather than something a schema can repair. +// // +kubebuilder:validation:XValidation:rule="(has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0) == (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)",message="shardAddressIPv4 and nat64Prefix must be set together" // +kubebuilder:validation:XValidation:rule="!(has(oldSelf.shardAddressIPv6) && size(oldSelf.shardAddressIPv6) > 0) || (has(self.shardAddressIPv6) && size(self.shardAddressIPv6) > 0)",message="shardAddressIPv6 cannot be unassigned once assigned" // +kubebuilder:validation:XValidation:rule="!(has(oldSelf.shardAddressIPv4) && size(oldSelf.shardAddressIPv4) > 0) || (has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0)",message="shardAddressIPv4 cannot be unassigned once assigned" // +kubebuilder:validation:XValidation:rule="!(has(oldSelf.nat64Prefix) && size(oldSelf.nat64Prefix) > 0) || (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)",message="nat64Prefix cannot be unassigned once assigned" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.shardAddressIPv6ClaimRef) || has(self.shardAddressIPv6ClaimRef)",message="shardAddressIPv6ClaimRef cannot be unassigned once assigned" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.shardAddressIPv4ClaimRef) || has(self.shardAddressIPv4ClaimRef)",message="shardAddressIPv4ClaimRef cannot be unassigned once assigned" type EgressShardSpec struct { // TargetRef identifies the Node this shard executes on. // +kubebuilder:validation:Required @@ -154,6 +228,17 @@ type EgressShardSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf || oldSelf == ''",message="shardAddressIPv6 is immutable once assigned" ShardAddressIPv6 string `json:"shardAddressIPv6,omitempty"` + // ShardAddressIPv6ClaimRef records the addressing-service claim + // ShardAddressIPv6 came from — see AddressClaimRef. Set it in the same + // write as the address whenever that address came from a claim. + // + // Nothing here resolves it and no component reads it to program anything; + // it is the only trail from a translating address back to the allocation + // accountable for it. + // +optional + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="shardAddressIPv6ClaimRef is immutable once assigned" + ShardAddressIPv6ClaimRef *AddressClaimRef `json:"shardAddressIPv6ClaimRef,omitempty"` + // ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this // shard translates to, and the source an IPv4-only destination sees. // Unlike ShardAddressIPv6, reachability for it is not established by a @@ -167,6 +252,17 @@ type EgressShardSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf || oldSelf == ''",message="shardAddressIPv4 is immutable once assigned" ShardAddressIPv4 string `json:"shardAddressIPv4,omitempty"` + // ShardAddressIPv4ClaimRef records the addressing-service claim + // ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same + // write as the address whenever that address came from a claim. + // + // Nothing here resolves it and no component reads it to program anything; + // it is the only trail from a translating address back to the allocation + // accountable for it. + // +optional + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="shardAddressIPv4ClaimRef is immutable once assigned" + ShardAddressIPv4ClaimRef *AddressClaimRef `json:"shardAddressIPv4ClaimRef,omitempty"` + // NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard // translates to IPv4 — one Datum-operated Network-Specific Prefix, shared // fabric-wide, never per-tenant. It must be the prefix the resolver diff --git a/api/v1alpha1/egressshard_types_test.go b/api/v1alpha1/egressshard_types_test.go index 86a82c1..c04f06c 100644 --- a/api/v1alpha1/egressshard_types_test.go +++ b/api/v1alpha1/egressshard_types_test.go @@ -25,6 +25,13 @@ func newTestEgressShard() *EgressShard { Spec: EgressShardSpec{ TargetRef: TargetRef{Kind: "Node", Name: "node-a"}, ShardAddressIPv6: "2001:db8:f00d::100", + ShardAddressIPv6ClaimRef: &AddressClaimRef{ + APIGroup: "ipam.miloapis.com", + Kind: "IPClaim", + Project: "datum-network-edge", + Namespace: "egress", + Name: "egress-shard-node-a-ipv6", + }, }, } } @@ -39,6 +46,7 @@ func TestEgressShardDeepCopy(t *testing.T) { dup.Spec.ShardAddressIPv4 = "198.51.100.7" dup.Spec.NAT64Prefix = "64:ff9b::/96" dup.Labels[LabelEgressShardPool] = "dedicated-public" + dup.Spec.ShardAddressIPv6ClaimRef.Name = "some-other-claim" dup.Status.Conditions = append(dup.Status.Conditions, metav1.Condition{Type: ConditionTypeProgrammed}) if orig.Spec.ShardAddressIPv6 != "2001:db8:f00d::100" { @@ -53,6 +61,9 @@ func TestEgressShardDeepCopy(t *testing.T) { if orig.Labels[LabelEgressShardPool] != "shared-public" { t.Errorf("pool label mutated: got %q", orig.Labels[LabelEgressShardPool]) } + if orig.Spec.ShardAddressIPv6ClaimRef.Name != "egress-shard-node-a-ipv6" { + t.Errorf("claim ref aliased by DeepCopy: got %q", orig.Spec.ShardAddressIPv6ClaimRef.Name) + } if len(orig.Status.Conditions) != 0 { t.Errorf("Conditions mutated: got %v", orig.Status.Conditions) } @@ -96,9 +107,15 @@ func TestEgressShardJSONRoundTrip(t *testing.T) { t.Fatalf("Unmarshal: %v", err) } - if got.Spec != orig.Spec { + if got.Spec.ShardAddressIPv6 != orig.Spec.ShardAddressIPv6 || + got.Spec.ShardAddressIPv4 != orig.Spec.ShardAddressIPv4 || + got.Spec.NAT64Prefix != orig.Spec.NAT64Prefix || + got.Spec.TargetRef != orig.Spec.TargetRef { t.Errorf("Spec: got %+v, want %+v", got.Spec, orig.Spec) } + if got.Spec.ShardAddressIPv6ClaimRef == nil || *got.Spec.ShardAddressIPv6ClaimRef != *orig.Spec.ShardAddressIPv6ClaimRef { + t.Errorf("ShardAddressIPv6ClaimRef: got %+v, want %+v", got.Spec.ShardAddressIPv6ClaimRef, orig.Spec.ShardAddressIPv6ClaimRef) + } if got.Status.ShardSID != orig.Status.ShardSID { t.Errorf("ShardSID: got %q, want %q", got.Status.ShardSID, orig.Status.ShardSID) } @@ -143,6 +160,23 @@ func TestEgressShardSpecFieldNames(t *testing.T) { if _, ok := m["targetRef"]; !ok { t.Error("expected \"targetRef\" key to be present") } + + claimRef, ok := m["shardAddressIPv6ClaimRef"].(map[string]any) + if !ok { + t.Fatalf("shardAddressIPv6ClaimRef: got %v, want an object", m["shardAddressIPv6ClaimRef"]) + } + wantRef := map[string]string{ + "apiGroup": "ipam.miloapis.com", + "kind": "IPClaim", + "project": "datum-network-edge", + "namespace": "egress", + "name": "egress-shard-node-a-ipv6", + } + for key, value := range wantRef { + if raw, ok := claimRef[key]; !ok || raw != value { + t.Errorf("shardAddressIPv6ClaimRef.%s: got %v, want %q", key, raw, value) + } + } } // TestEgressShardSpecOmitEmpty verifies that a shard with no address assigned @@ -161,7 +195,10 @@ func TestEgressShardSpecOmitEmpty(t *testing.T) { t.Fatalf("Unmarshal: %v", err) } - for _, key := range []string{"shardAddressIPv6", "shardAddressIPv4", "nat64Prefix"} { + for _, key := range []string{ + "shardAddressIPv6", "shardAddressIPv4", "nat64Prefix", + "shardAddressIPv6ClaimRef", "shardAddressIPv4ClaimRef", + } { if _, ok := m[key]; ok { t.Errorf("expected %q key to be absent when empty", key) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 4a4a01e..17eafb4 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -41,6 +41,21 @@ func (in *ASPathFilter) DeepCopy() *ASPathFilter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddressClaimRef) DeepCopyInto(out *AddressClaimRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddressClaimRef. +func (in *AddressClaimRef) DeepCopy() *AddressClaimRef { + if in == nil { + return nil + } + out := new(AddressClaimRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AddressFamily) DeepCopyInto(out *AddressFamily) { *out = *in @@ -1266,7 +1281,7 @@ func (in *EgressShard) DeepCopyInto(out *EgressShard) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -1324,6 +1339,16 @@ func (in *EgressShardList) DeepCopyObject() runtime.Object { func (in *EgressShardSpec) DeepCopyInto(out *EgressShardSpec) { *out = *in out.TargetRef = in.TargetRef + if in.ShardAddressIPv6ClaimRef != nil { + in, out := &in.ShardAddressIPv6ClaimRef, &out.ShardAddressIPv6ClaimRef + *out = new(AddressClaimRef) + **out = **in + } + if in.ShardAddressIPv4ClaimRef != nil { + in, out := &in.ShardAddressIPv4ClaimRef, &out.ShardAddressIPv4ClaimRef + *out = new(AddressClaimRef) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardSpec. diff --git a/config/crd/network.datumapis.com_egressshards.yaml b/config/crd/network.datumapis.com_egressshards.yaml index fadbf21..cdb1f27 100644 --- a/config/crd/network.datumapis.com_egressshards.yaml +++ b/config/crd/network.datumapis.com_egressshards.yaml @@ -107,6 +107,18 @@ spec: established through it, with no drain and no dual-address grace period available to cover the change. A shard holding the wrong address is deleted and recreated instead, which breaks those flows at a moment someone chose. + + A claim reference alongside an address records where that address came from, + and is write-once on the same terms. It is deliberately not required with an + address, in either direction. Requiring an address before a claim reference + would forbid recording a claim whose allocation has not resolved yet, which + is the window a restarting controller most needs to see so that it finds the + claim it already made instead of making a second one. Requiring a claim + reference before an address would forbid an address allocated by hand, and + whether such an address may be recorded here at all is an open migration + question this validation would answer by fiat. An address with no reference + is therefore accepted and is unattributable, which is a fact about the + allocation rather than something a schema can repair. properties: nat64Prefix: description: |- @@ -143,6 +155,53 @@ spec: rule: self == '' || (isIP(self) && ip(self).family() == 4) - message: shardAddressIPv4 is immutable once assigned rule: self == oldSelf || oldSelf == '' + shardAddressIPv4ClaimRef: + description: |- + ShardAddressIPv4ClaimRef records the addressing-service claim + ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same + write as the address whenever that address came from a claim. + + Nothing here resolves it and no component reads it to program anything; + it is the only trail from a translating address back to the allocation + accountable for it. + properties: + apiGroup: + default: ipam.miloapis.com + description: APIGroup is the group of the claim resource. + minLength: 1 + type: string + kind: + default: IPClaim + description: Kind is the kind of the claim resource. + minLength: 1 + type: string + name: + description: |- + Name is the name of the claim. The addressing service derives it from + the thing being addressed, so it is stable across a shard's lifetime and + is what a replacement finds again. + minLength: 1 + type: string + namespace: + description: Namespace is the namespace holding the claim within + Project. + minLength: 1 + type: string + project: + description: |- + Project is the project whose control plane serves the claim. Required: + a claim is namespaced within a project, and this reference is read from + outside every project. + minLength: 1 + type: string + required: + - name + - namespace + - project + type: object + x-kubernetes-validations: + - message: shardAddressIPv4ClaimRef is immutable once assigned + rule: self == oldSelf shardAddressIPv6: description: |- ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this @@ -158,6 +217,53 @@ spec: rule: self == '' || (isIP(self) && ip(self).family() == 6) - message: shardAddressIPv6 is immutable once assigned rule: self == oldSelf || oldSelf == '' + shardAddressIPv6ClaimRef: + description: |- + ShardAddressIPv6ClaimRef records the addressing-service claim + ShardAddressIPv6 came from — see AddressClaimRef. Set it in the same + write as the address whenever that address came from a claim. + + Nothing here resolves it and no component reads it to program anything; + it is the only trail from a translating address back to the allocation + accountable for it. + properties: + apiGroup: + default: ipam.miloapis.com + description: APIGroup is the group of the claim resource. + minLength: 1 + type: string + kind: + default: IPClaim + description: Kind is the kind of the claim resource. + minLength: 1 + type: string + name: + description: |- + Name is the name of the claim. The addressing service derives it from + the thing being addressed, so it is stable across a shard's lifetime and + is what a replacement finds again. + minLength: 1 + type: string + namespace: + description: Namespace is the namespace holding the claim within + Project. + minLength: 1 + type: string + project: + description: |- + Project is the project whose control plane serves the claim. Required: + a claim is namespaced within a project, and this reference is read from + outside every project. + minLength: 1 + type: string + required: + - name + - namespace + - project + type: object + x-kubernetes-validations: + - message: shardAddressIPv6ClaimRef is immutable once assigned + rule: self == oldSelf targetRef: description: TargetRef identifies the Node this shard executes on. properties: @@ -191,6 +297,10 @@ spec: - message: nat64Prefix cannot be unassigned once assigned rule: '!(has(oldSelf.nat64Prefix) && size(oldSelf.nat64Prefix) > 0) || (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)' + - message: shardAddressIPv6ClaimRef cannot be unassigned once assigned + rule: '!has(oldSelf.shardAddressIPv6ClaimRef) || has(self.shardAddressIPv6ClaimRef)' + - message: shardAddressIPv4ClaimRef cannot be unassigned once assigned + rule: '!has(oldSelf.shardAddressIPv4ClaimRef) || has(self.shardAddressIPv4ClaimRef)' status: description: |- EgressShardStatus defines the observed state of an EgressShard. diff --git a/docs/api/bgp.md b/docs/api/bgp.md index 52ea53c..835937b 100644 --- a/docs/api/bgp.md +++ b/docs/api/bgp.md @@ -74,6 +74,52 @@ _Appears in:_ | `contains` | ASPathMatchContains requires the pattern to match a substring of the AS path.
| +#### AddressClaimRef + + + +AddressClaimRef names the addressing-service claim an assigned address came +from. It is a record, not a lookup: the addressing service is an aggregated +API served on a project control-plane path and defines no CustomResourceDefinition, +so no claim object exists in the cluster a shard runs in and nothing here +resolves one. Reading it means addressing that project's control plane +directly, with a credential a translating node is deliberately not given. + +Every field is a plain string and this group depends on no addressing-service +type. The shape matches that service's own opaque cross-API reference, plus +the project, because a claim is namespaced within a project and a namespace +name alone does not identify one from outside. + +Populate it with the coordinates of the claim whose allocation produced the +address written alongside it: + + shardAddressIPv6: 2001:db8:f00d::100 + shardAddressIPv6ClaimRef: + project: datum-network-edge + namespace: egress + name: egress-shard-node-a-ipv6 + +The claim cannot record this relationship itself. The addressing service +overwrites a claim's spec.ownerRef on create with the identity that requested +it, so a claim made by a cell controller is attributed to that controller's +project rather than to the shard it was made for. Attribution therefore runs +in this direction only, and an operator asking which claim holds a shard's +address has this field or nothing. + + + +_Appears in:_ +- [EgressShardSpec](#egressshardspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiGroup` _string_ | APIGroup is the group of the claim resource. | ipam.miloapis.com | MinLength: 1
| +| `kind` _string_ | Kind is the kind of the claim resource. | IPClaim | MinLength: 1
| +| `project` _string_ | Project is the project whose control plane serves the claim. Required:
a claim is namespaced within a project, and this reference is read from
outside every project. | | MinLength: 1
Required: \{\}
| +| `namespace` _string_ | Namespace is the namespace holding the claim within Project. | | MinLength: 1
Required: \{\}
| +| `name` _string_ | Name is the name of the claim. The addressing service derives it from
the thing being addressed, so it is stable across a shard's lifetime and
is what a replacement finds again. | | MinLength: 1
Required: \{\}
| + + #### AddressFamily @@ -992,6 +1038,18 @@ established through it, with no drain and no dual-address grace period available to cover the change. A shard holding the wrong address is deleted and recreated instead, which breaks those flows at a moment someone chose. +A claim reference alongside an address records where that address came from, +and is write-once on the same terms. It is deliberately not required with an +address, in either direction. Requiring an address before a claim reference +would forbid recording a claim whose allocation has not resolved yet, which +is the window a restarting controller most needs to see so that it finds the +claim it already made instead of making a second one. Requiring a claim +reference before an address would forbid an address allocated by hand, and +whether such an address may be recorded here at all is an open migration +question this validation would answer by fiat. An address with no reference +is therefore accepted and is unattributable, which is a fact about the +allocation rather than something a schema can repair. + _Appears in:_ @@ -1001,7 +1059,9 @@ _Appears in:_ | --- | --- | --- | --- | | `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| | `shardAddressIPv6` _string_ | ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this
shard translates to — every NAT66 masquerade port it allocates lives
within this address, so any node can route a reply to the owning shard
using ordinary unicast routing on it alone, with no per-flow state
lookup anywhere but that shard.
Empty means no IPv6 address is assigned to this shard. | | | +| `shardAddressIPv6ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv6ClaimRef records the addressing-service claim
ShardAddressIPv6 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | | `shardAddressIPv4` _string_ | ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this
shard translates to, and the source an IPv4-only destination sees.
Unlike ShardAddressIPv6, reachability for it is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the underlay or an upstream announcement must attract this
address to this node.
Empty means no IPv4 address is assigned to this shard. | | | +| `shardAddressIPv4ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv4ClaimRef records the addressing-service claim
ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | | `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | diff --git a/docs/api/gateway.md b/docs/api/gateway.md index 7239c04..9cfebfe 100644 --- a/docs/api/gateway.md +++ b/docs/api/gateway.md @@ -17,6 +17,52 @@ Package v1alpha1 contains API Schema definitions for the network.datumapis.com/v +#### AddressClaimRef + + + +AddressClaimRef names the addressing-service claim an assigned address came +from. It is a record, not a lookup: the addressing service is an aggregated +API served on a project control-plane path and defines no CustomResourceDefinition, +so no claim object exists in the cluster a shard runs in and nothing here +resolves one. Reading it means addressing that project's control plane +directly, with a credential a translating node is deliberately not given. + +Every field is a plain string and this group depends on no addressing-service +type. The shape matches that service's own opaque cross-API reference, plus +the project, because a claim is namespaced within a project and a namespace +name alone does not identify one from outside. + +Populate it with the coordinates of the claim whose allocation produced the +address written alongside it: + + shardAddressIPv6: 2001:db8:f00d::100 + shardAddressIPv6ClaimRef: + project: datum-network-edge + namespace: egress + name: egress-shard-node-a-ipv6 + +The claim cannot record this relationship itself. The addressing service +overwrites a claim's spec.ownerRef on create with the identity that requested +it, so a claim made by a cell controller is attributed to that controller's +project rather than to the shard it was made for. Attribution therefore runs +in this direction only, and an operator asking which claim holds a shard's +address has this field or nothing. + + + +_Appears in:_ +- [EgressShardSpec](#egressshardspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiGroup` _string_ | APIGroup is the group of the claim resource. | ipam.miloapis.com | MinLength: 1
| +| `kind` _string_ | Kind is the kind of the claim resource. | IPClaim | MinLength: 1
| +| `project` _string_ | Project is the project whose control plane serves the claim. Required:
a claim is namespaced within a project, and this reference is read from
outside every project. | | MinLength: 1
Required: \{\}
| +| `namespace` _string_ | Namespace is the namespace holding the claim within Project. | | MinLength: 1
Required: \{\}
| +| `name` _string_ | Name is the name of the claim. The addressing service derives it from
the thing being addressed, so it is stable across a shard's lifetime and
is what a replacement finds again. | | MinLength: 1
Required: \{\}
| + + #### EgressShard @@ -93,6 +139,18 @@ established through it, with no drain and no dual-address grace period available to cover the change. A shard holding the wrong address is deleted and recreated instead, which breaks those flows at a moment someone chose. +A claim reference alongside an address records where that address came from, +and is write-once on the same terms. It is deliberately not required with an +address, in either direction. Requiring an address before a claim reference +would forbid recording a claim whose allocation has not resolved yet, which +is the window a restarting controller most needs to see so that it finds the +claim it already made instead of making a second one. Requiring a claim +reference before an address would forbid an address allocated by hand, and +whether such an address may be recorded here at all is an open migration +question this validation would answer by fiat. An address with no reference +is therefore accepted and is unattributable, which is a fact about the +allocation rather than something a schema can repair. + _Appears in:_ @@ -102,7 +160,9 @@ _Appears in:_ | --- | --- | --- | --- | | `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| | `shardAddressIPv6` _string_ | ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this
shard translates to — every NAT66 masquerade port it allocates lives
within this address, so any node can route a reply to the owning shard
using ordinary unicast routing on it alone, with no per-flow state
lookup anywhere but that shard.
Empty means no IPv6 address is assigned to this shard. | | | +| `shardAddressIPv6ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv6ClaimRef records the addressing-service claim
ShardAddressIPv6 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | | `shardAddressIPv4` _string_ | ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this
shard translates to, and the source an IPv4-only destination sees.
Unlike ShardAddressIPv6, reachability for it is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the underlay or an upstream announcement must attract this
address to this node.
Empty means no IPv4 address is assigned to this shard. | | | +| `shardAddressIPv4ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv4ClaimRef records the addressing-service claim
ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | | `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | diff --git a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml index 487557c..51f1b0f 100644 --- a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml +++ b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml @@ -4,10 +4,13 @@ metadata: name: egress-shard-address-immutability spec: description: > - Verify the CEL rules governing an EgressShard's assigned egress addresses: - a shard is accepted with no address, the cell controller's first write is - accepted, rewriting the same value is accepted, and changing or unassigning - an assigned value is rejected. Also verifies that status remains writable, + Verify the CEL rules governing an EgressShard's assigned egress addresses + and the claim references that attribute them: a shard is accepted with no + address, the cell controller's first write is accepted, rewriting the same + value is accepted, and changing or unassigning an assigned value is + rejected. Also verifies that a claim reference defaults its apiGroup and + kind, and that an address and a claim reference are independently + optional. Also verifies that status remains writable, since status reports what the datapath actually programmed and must still converge. Does not require any operator to be running — every rule under test is enforced by the API server alone. @@ -41,7 +44,8 @@ spec: content: | set -e kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' - {"spec":{"shardAddressIPv6":"2001:db8:f00d::100","shardAddressIPv4":"198.51.100.7","nat64Prefix":"64:ff9b::/96"}}' + {"spec":{"shardAddressIPv6":"2001:db8:f00d::100","shardAddressIPv4":"198.51.100.7","nat64Prefix":"64:ff9b::/96", + "shardAddressIPv6ClaimRef":{"project":"e2e-project","namespace":"egress","name":"e2e-claim-ipv6"}}}' echo "OK: first assignment accepted" - assert: resource: @@ -53,6 +57,12 @@ spec: shardAddressIPv6: 2001:db8:f00d::100 shardAddressIPv4: 198.51.100.7 nat64Prefix: 64:ff9b::/96 + shardAddressIPv6ClaimRef: + apiGroup: ipam.miloapis.com + kind: IPClaim + project: e2e-project + namespace: egress + name: e2e-claim-ipv6 - name: accept-same-value-rewrite try: @@ -150,6 +160,127 @@ spec: echo "OK: an IPv4 address with no NAT64 prefix correctly rejected" echo "Server response: $OUTPUT" + - name: reject-claim-ref-change + try: + - script: + content: | + set +e + OUTPUT=$(kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardAddressIPv6ClaimRef":{"project":"e2e-project","namespace":"egress","name":"e2e-claim-other"}}}' 2>&1) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of a changed shardAddressIPv6ClaimRef but the patch was accepted" + exit 1 + fi + case "$OUTPUT" in + *"immutable once assigned"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: changing an assigned claim reference correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-claim-ref-unassignment + try: + - script: + content: | + set +e + OUTPUT=$(kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardAddressIPv6ClaimRef":null}}' 2>&1) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of an unassigned shardAddressIPv6ClaimRef but the patch was accepted" + exit 1 + fi + case "$OUTPUT" in + *"cannot be unassigned once assigned"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: unassigning an assigned claim reference correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-claim-ref-without-project + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -f - 2>&1 <<'EOF' + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard-projectless + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + shardAddressIPv6ClaimRef: + namespace: egress + name: e2e-claim-ipv6 + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of a claim reference with no project but the resource was accepted" + kubectl delete egressshard e2e-egress-shard-projectless 2>/dev/null || true + exit 1 + fi + echo "OK: a claim reference with no project correctly rejected" + echo "Server response: $OUTPUT" + + - name: default-claim-ref-group-and-kind + try: + - script: + content: | + set -e + kubectl apply -f - <<'EOF' + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard-defaults + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + shardAddressIPv6ClaimRef: + project: e2e-project + namespace: egress + name: e2e-claim-ipv6 + EOF + GROUP=$(kubectl get egressshard e2e-egress-shard-defaults -n "$NAMESPACE" \ + -o jsonpath='{.spec.shardAddressIPv6ClaimRef.apiGroup}') + KIND=$(kubectl get egressshard e2e-egress-shard-defaults -n "$NAMESPACE" \ + -o jsonpath='{.spec.shardAddressIPv6ClaimRef.kind}') + kubectl delete egressshard e2e-egress-shard-defaults -n "$NAMESPACE" + if [ "$GROUP" != "ipam.miloapis.com" ] || [ "$KIND" != "IPClaim" ]; then + echo "ERROR: claim reference defaults = $GROUP/$KIND, want ipam.miloapis.com/IPClaim" + exit 1 + fi + echo "OK: claim reference defaulted to $GROUP/$KIND" + + - name: accept-claim-ref-with-no-address + try: + - script: + content: | + set -e + kubectl apply -f - <<'EOF' + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard-pending + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + shardAddressIPv4ClaimRef: + project: e2e-project + namespace: egress + name: e2e-claim-ipv4 + EOF + kubectl delete egressshard e2e-egress-shard-pending -n "$NAMESPACE" + echo "OK: a claim reference whose allocation has not resolved yet is accepted" + - name: accept-status-changes try: - script: From d31482d46c4fc0d3c25d69b7c2435baf220c5fb3 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 17 Sep 2026 21:41:31 -0500 Subject: [PATCH 4/4] test(e2e): Address the egress shard scenario by namespace Two steps of the egress shard scenario created an object with a bare kubectl apply and then read it back with an explicit namespace, so the read looked somewhere the object was never written and the step failed on a resource it had just created. The bare form also left the object outside the namespace chainsaw tears down. Key changes: - Name the namespace on every kubectl call in the scenario, so a step reads and deletes what it created Co-Authored-By: Claude Opus 5 (1M context) --- .../chainsaw-test.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml index 51f1b0f..94a7f47 100644 --- a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml +++ b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml @@ -134,7 +134,7 @@ spec: - script: content: | set +e - OUTPUT=$(kubectl apply -f - 2>&1 <<'EOF' + OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' apiVersion: network.datumapis.com/v1alpha1 kind: EgressShard metadata: @@ -150,7 +150,7 @@ spec: set -e if [ "$EXIT" -eq 0 ]; then echo "ERROR: expected rejection of an IPv4 address with no NAT64 prefix but the resource was accepted" - kubectl delete egressshard e2e-egress-shard-unpaired 2>/dev/null || true + kubectl delete egressshard e2e-egress-shard-unpaired -n "$NAMESPACE" 2>/dev/null || true exit 1 fi case "$OUTPUT" in @@ -205,7 +205,7 @@ spec: - script: content: | set +e - OUTPUT=$(kubectl apply -f - 2>&1 <<'EOF' + OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' apiVersion: network.datumapis.com/v1alpha1 kind: EgressShard metadata: @@ -223,7 +223,7 @@ spec: set -e if [ "$EXIT" -eq 0 ]; then echo "ERROR: expected rejection of a claim reference with no project but the resource was accepted" - kubectl delete egressshard e2e-egress-shard-projectless 2>/dev/null || true + kubectl delete egressshard e2e-egress-shard-projectless -n "$NAMESPACE" 2>/dev/null || true exit 1 fi echo "OK: a claim reference with no project correctly rejected" @@ -234,7 +234,7 @@ spec: - script: content: | set -e - kubectl apply -f - <<'EOF' + kubectl apply -n "$NAMESPACE" -f - <<'EOF' apiVersion: network.datumapis.com/v1alpha1 kind: EgressShard metadata: @@ -264,7 +264,7 @@ spec: - script: content: | set -e - kubectl apply -f - <<'EOF' + kubectl apply -n "$NAMESPACE" -f - <<'EOF' apiVersion: network.datumapis.com/v1alpha1 kind: EgressShard metadata: