diff --git a/api/v1alpha1/egressshard_types.go b/api/v1alpha1/egressshard_types.go index 1118259..6c336d3 100644 --- a/api/v1alpha1/egressshard_types.go +++ b/api/v1alpha1/egressshard_types.go @@ -16,10 +16,9 @@ import ( // (IPv6 -> IPv4, RFC 6146) are the same function — stateful egress PAT with a // VRF-scoped session table — over different families, so one shard object // describes both rather than there being a second, near-duplicate kind. -// Status.ShardAddressIPv6 and Status.ShardAddressIPv4 are each set only for -// the family this shard actually translates; a shard serving only NAT66 -// leaves the IPv4 field empty and behaves exactly as it did before NAT64 -// existed. +// Spec.ShardAddressIPv6 and Spec.ShardAddressIPv4 are each assigned only for +// the family this shard translates; a shard with no IPv4 address performs no +// NAT64. // // Every shard owns a dedicated, publicly-routable address per family it // serves, and a flow's allocated masquerade port lives within it — so a reply @@ -28,6 +27,21 @@ import ( // can determine the owning shard from the tuple alone" property, satisfied by // construction rather than by a replicated hash table). // +// A shard is told which addresses to translate to; it does not choose them. +// The controller that owns a cell claims one address per family from the +// addressing service and writes it into this spec, which keeps the +// addressing-service credential off every translating node and keeps the +// allocation request out of the path that attaches a workload. Status reports +// what the node's datapath is actually programmed with, so an unclaimed +// address, a stale datapath, and a divergence between the two are each +// distinguishable. +// +// A shard names nothing that selects it. Which shards serve which consumer +// intent is decided entirely by label selectors evaluated on the selecting +// side (see the LabelEgressShard* keys), so no consumer-facing API appears in +// this group and no reference points from a datapath resource back at the +// resource that placed traffic on it. +// // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Namespaced,shortName=egressshard @@ -44,19 +58,234 @@ type EgressShard struct { Status EgressShardStatus `json:"status,omitempty"` } +// Label keys that select the EgressShards serving a given kind of egress. +// They are matched by an ordinary label selector held on the selecting side, +// which is what lets a consumer-facing API in another group place traffic on +// these shards without this group referencing, importing, or depending on +// that API. A shard carrying none of them is selected by nothing and serves +// no traffic. +// +// The pool and cell labels are operator-set placement facts. The family +// labels restate what this spec already assigns, because a selector matches +// labels and cannot read a spec field; whoever writes the addresses writes +// them. +const ( + // LabelEgressShardPool names the operator-defined pool this shard belongs + // to — the unit a selector picks, not an individual shard. Pools exist so + // that adding or draining a node changes no selector. + LabelEgressShardPool string = "network.datumapis.com/egress-pool" + + // LabelEgressShardCell names the cell this shard translates in. Egress is + // realized per cell, so a selector that omits it selects shards in every + // cell and sends a consumer's traffic out of an arbitrary one. + LabelEgressShardCell string = "network.datumapis.com/egress-cell" + + // LabelEgressShardIPv6 marks a shard that translates to IPv6, set to + // LabelValueEgressFamilyServed whenever Spec.ShardAddressIPv6 is assigned. + LabelEgressShardIPv6 string = "network.datumapis.com/egress-ipv6" + + // LabelEgressShardIPv4 marks a shard that translates to IPv4, set to + // LabelValueEgressFamilyServed whenever Spec.ShardAddressIPv4 is assigned. + LabelEgressShardIPv4 string = "network.datumapis.com/egress-ipv4" + + // LabelValueEgressFamilyServed is the only value the family labels carry. + // Absence, not a false value, means the family is not served: a selector + // requiring a family must match on presence so that a shard predating + // these labels never reads as serving one it does not. + LabelValueEgressFamilyServed string = "true" +) + +// ConditionTypeProgrammed indicates whether this node's datapath is +// translating with the addresses this spec assigns. It is distinct from +// Ready, which reports only that the datapath is attached: an attached +// datapath holding no assigned address, or a superseded one, drops or +// mis-sources every flow while reporting Ready. +const ConditionTypeProgrammed string = "Programmed" + +// Programmed sub-reasons — used as Programmed.Reason. +const ( + // ProgrammedReasonAddressesProgrammed indicates the datapath is + // translating with every address this spec assigns. + ProgrammedReasonAddressesProgrammed string = "AddressesProgrammed" + + // ProgrammedReasonAddressUnassigned indicates this spec assigns no + // address for any family, so the shard has nothing to translate to and + // claims no packet. + ProgrammedReasonAddressUnassigned string = "AddressUnassigned" + + // ProgrammedReasonProgrammingFailed indicates the shard could not program + // an assigned address into its datapath. + ProgrammedReasonProgrammingFailed string = "ProgrammingFailed" +) + +// AddressClaimRef names the addressing-service claim an assigned address came +// from. It is a record, not a lookup: the addressing service is an aggregated +// API served on a project control-plane path and defines no CustomResourceDefinition, +// so no claim object exists in the cluster a shard runs in and nothing here +// resolves one. Reading it means addressing that project's control plane +// directly, with a credential a translating node is deliberately not given. +// +// Every field is a plain string and this group depends on no addressing-service +// type. The shape matches that service's own opaque cross-API reference, plus +// the project, because a claim is namespaced within a project and a namespace +// name alone does not identify one from outside. +// +// Populate it with the coordinates of the claim whose allocation produced the +// address written alongside it: +// +// shardAddressIPv6: 2001:db8:f00d::100 +// shardAddressIPv6ClaimRef: +// project: datum-network-edge +// namespace: egress +// name: egress-shard-node-a-ipv6 +// +// The claim cannot record this relationship itself. The addressing service +// overwrites a claim's spec.ownerRef on create with the identity that requested +// it, so a claim made by a cell controller is attributed to that controller's +// project rather than to the shard it was made for. Attribution therefore runs +// in this direction only, and an operator asking which claim holds a shard's +// address has this field or nothing. +type AddressClaimRef struct { + // APIGroup is the group of the claim resource. + // +optional + // +kubebuilder:default="ipam.miloapis.com" + // +kubebuilder:validation:MinLength=1 + APIGroup string `json:"apiGroup,omitempty"` + + // Kind is the kind of the claim resource. + // +optional + // +kubebuilder:default="IPClaim" + // +kubebuilder:validation:MinLength=1 + Kind string `json:"kind,omitempty"` + + // Project is the project whose control plane serves the claim. Required: + // a claim is namespaced within a project, and this reference is read from + // outside every project. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Project string `json:"project"` + + // Namespace is the namespace holding the claim within Project. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Namespace string `json:"namespace"` + + // Name is the name of the claim. The addressing service derives it from + // the thing being addressed, so it is stable across a shard's lifetime and + // is what a replacement finds again. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` +} + // EgressShardSpec defines the desired state of an EgressShard. +// +// The address fields are written by the controller that owns the cell, not by +// the shard and not by a consumer. Each is optional: an address the addressing +// service has not yet handed out is absent rather than blank-but-required, so +// a shard object exists from the moment its node is labelled and gains its +// identity afterwards. +// +// Each is also write-once, and cannot be unassigned once assigned. The +// datapath claims a reply by exact match against the address it translates to, +// so reassigning one strands the return traffic of every flow already +// established through it, with no drain and no dual-address grace period +// available to cover the change. A shard holding the wrong address is deleted +// and recreated instead, which breaks those flows at a moment someone chose. +// +// A claim reference alongside an address records where that address came from, +// and is write-once on the same terms. It is deliberately not required with an +// address, in either direction. Requiring an address before a claim reference +// would forbid recording a claim whose allocation has not resolved yet, which +// is the window a restarting controller most needs to see so that it finds the +// claim it already made instead of making a second one. Requiring a claim +// reference before an address would forbid an address allocated by hand, and +// whether such an address may be recorded here at all is an open migration +// question this validation would answer by fiat. An address with no reference +// is therefore accepted and is unattributable, which is a fact about the +// allocation rather than something a schema can repair. +// +// +kubebuilder:validation:XValidation:rule="(has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0) == (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)",message="shardAddressIPv4 and nat64Prefix must be set together" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.shardAddressIPv6) && size(oldSelf.shardAddressIPv6) > 0) || (has(self.shardAddressIPv6) && size(self.shardAddressIPv6) > 0)",message="shardAddressIPv6 cannot be unassigned once assigned" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.shardAddressIPv4) && size(oldSelf.shardAddressIPv4) > 0) || (has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0)",message="shardAddressIPv4 cannot be unassigned once assigned" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.nat64Prefix) && size(oldSelf.nat64Prefix) > 0) || (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)",message="nat64Prefix cannot be unassigned once assigned" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.shardAddressIPv6ClaimRef) || has(self.shardAddressIPv6ClaimRef)",message="shardAddressIPv6ClaimRef cannot be unassigned once assigned" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.shardAddressIPv4ClaimRef) || has(self.shardAddressIPv4ClaimRef)",message="shardAddressIPv4ClaimRef cannot be unassigned once assigned" type EgressShardSpec struct { // TargetRef identifies the Node this shard executes on. // +kubebuilder:validation:Required TargetRef TargetRef `json:"targetRef"` + + // ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this + // shard translates to — every NAT66 masquerade port it allocates lives + // within this address, so any node can route a reply to the owning shard + // using ordinary unicast routing on it alone, with no per-flow state + // lookup anywhere but that shard. + // + // Empty means no IPv6 address is assigned to this shard. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 6)",message="shardAddressIPv6 must be a valid IPv6 address" + // +kubebuilder:validation:XValidation:rule="self == oldSelf || oldSelf == ''",message="shardAddressIPv6 is immutable once assigned" + ShardAddressIPv6 string `json:"shardAddressIPv6,omitempty"` + + // ShardAddressIPv6ClaimRef records the addressing-service claim + // ShardAddressIPv6 came from — see AddressClaimRef. Set it in the same + // write as the address whenever that address came from a claim. + // + // Nothing here resolves it and no component reads it to program anything; + // it is the only trail from a translating address back to the allocation + // accountable for it. + // +optional + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="shardAddressIPv6ClaimRef is immutable once assigned" + ShardAddressIPv6ClaimRef *AddressClaimRef `json:"shardAddressIPv6ClaimRef,omitempty"` + + // ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this + // shard translates to, and the source an IPv4-only destination sees. + // Unlike ShardAddressIPv6, reachability for it is not established by a + // BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the + // internet, so the underlay or an upstream announcement must attract this + // address to this node. + // + // Empty means no IPv4 address is assigned to this shard. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 4)",message="shardAddressIPv4 must be a valid IPv4 address" + // +kubebuilder:validation:XValidation:rule="self == oldSelf || oldSelf == ''",message="shardAddressIPv4 is immutable once assigned" + ShardAddressIPv4 string `json:"shardAddressIPv4,omitempty"` + + // ShardAddressIPv4ClaimRef records the addressing-service claim + // ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same + // write as the address whenever that address came from a claim. + // + // Nothing here resolves it and no component reads it to program anything; + // it is the only trail from a translating address back to the allocation + // accountable for it. + // +optional + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="shardAddressIPv4ClaimRef is immutable once assigned" + ShardAddressIPv4ClaimRef *AddressClaimRef `json:"shardAddressIPv4ClaimRef,omitempty"` + + // NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard + // translates to IPv4 — one Datum-operated Network-Specific Prefix, shared + // fabric-wide, never per-tenant. It must be the prefix the resolver + // synthesizes into; a shard translating for a different one is a + // blackhole with no symptom on either side. + // + // Set together with ShardAddressIPv4 or not at all: an address with no + // prefix has nothing to translate for, and a prefix with no address has + // nothing to translate into. Write-once for the same reason the addresses + // are: a shard that starts translating a different prefix blackholes every + // destination the resolver already synthesized into the old one. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || isCIDR(self)",message="nat64Prefix must be a valid CIDR" + // +kubebuilder:validation:XValidation:rule="self == oldSelf || oldSelf == ''",message="nat64Prefix is immutable once assigned" + NAT64Prefix string `json:"nat64Prefix,omitempty"` } // EgressShardStatus defines the observed state of an EgressShard. // -// Every field here is echoed from what the shard's datapath process was -// actually started with, not derived: the shard publishes what it is running, -// so a status that disagrees with an operator's intent is a visible -// misconfiguration rather than a silently reconciled one. +// The address and prefix fields report what this node's datapath is programmed +// with, not what it was asked for. A value here that disagrees with the spec is +// a shard that has not converged; a value here with no counterpart in the spec +// is a shard still translating to an address nothing assigns any more. type EgressShardStatus struct { // ObservedGeneration is the .metadata.generation this status was computed from. // +optional @@ -70,50 +299,38 @@ type EgressShardStatus struct { // route against it. One SID serves both families: which translation a // packet gets is decided from the inner destination, not from a second // SID. + // + // Still chosen by an operator and reported here rather than assigned in + // spec, unlike the addresses: a value another node already uses silently + // diverts that node's traffic, so the assignment belongs to the + // addressing service, which does not hand out identifiers of this kind + // yet. // +optional // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 6)",message="shardSID must be a valid IPv6 address" ShardSID string `json:"shardSID,omitempty"` - // ShardAddressIPv6 is this shard's own dedicated, publicly-routable IPv6 - // address — every NAT66 masquerade port this shard allocates lives within - // it, so any node can route a reply to the correct shard using ordinary - // unicast routing on this address alone, with no per-flow state lookup - // anywhere but the owning shard itself. Operator-supplied per shard today - // (no in-cluster derivation mechanism yet — the same gap - // BGPRouter.Spec.SRv6Locator/NodeID assignment has today). - // - // Empty means this shard does not perform IPv6-to-IPv6 translation. + // ShardAddressIPv6 is the IPv6 masquerade source this shard's datapath is + // programmed with. Empty means it translates no IPv6 flow. // +optional // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 6)",message="shardAddressIPv6 must be a valid IPv6 address" ShardAddressIPv6 string `json:"shardAddressIPv6,omitempty"` - // ShardAddressIPv4 is this shard's own dedicated, publicly-routable IPv4 - // address — every NAT64 masquerade port this shard allocates lives within - // it, and it is the source an IPv4-only destination sees. Unlike - // ShardAddressIPv6, reachability for this address is not established by a - // BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the - // internet, so the address must be attracted to this node by the underlay - // or upstream announcement instead. Publishing it here is what makes that - // operator prerequisite checkable. - // - // Empty means this shard does not perform NAT64. + // ShardAddressIPv4 is the IPv4 masquerade source this shard's datapath is + // programmed with. Empty means it performs no NAT64. Publishing it is also + // what makes the underlay reachability prerequisite in + // Spec.ShardAddressIPv4 checkable. // +optional // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 4)",message="shardAddressIPv4 must be a valid IPv4 address" ShardAddressIPv4 string `json:"shardAddressIPv4,omitempty"` - // NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard - // translates to IPv4 — one Datum-operated Network-Specific Prefix, shared - // fabric-wide, never per-tenant. It is echoed here, rather than only - // existing as process configuration, because it is the single fact DNS64 - // synthesis has to agree with: a shard translating for a different prefix - // than the resolver synthesizes into is otherwise a silent blackhole. - // - // Empty whenever ShardAddressIPv4 is empty. + // NAT64Prefix is the prefix this shard's datapath is programmed to + // translate. Empty whenever ShardAddressIPv4 is empty. // +optional // +kubebuilder:validation:XValidation:rule="self == '' || isCIDR(self)",message="nat64Prefix must be a valid CIDR" NAT64Prefix string `json:"nat64Prefix,omitempty"` - // Conditions contains the standard conditions for this resource. + // Conditions contains the standard conditions for this resource, + // including Programmed (see ConditionTypeProgrammed). // // +listType=map // +listMapKey=type diff --git a/api/v1alpha1/egressshard_types_test.go b/api/v1alpha1/egressshard_types_test.go new file mode 100644 index 0000000..c04f06c --- /dev/null +++ b/api/v1alpha1/egressshard_types_test.go @@ -0,0 +1,226 @@ +package v1alpha1 + +import ( + "encoding/json" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func newTestEgressShard() *EgressShard { + return &EgressShard{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "network.datumapis.com/v1alpha1", + Kind: "EgressShard", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-egress-shard", + Namespace: "galactic-system", + Labels: map[string]string{ + LabelEgressShardPool: "shared-public", + LabelEgressShardCell: "us-east-2-a", + LabelEgressShardIPv6: LabelValueEgressFamilyServed, + }, + }, + Spec: EgressShardSpec{ + TargetRef: TargetRef{Kind: "Node", Name: "node-a"}, + ShardAddressIPv6: "2001:db8:f00d::100", + ShardAddressIPv6ClaimRef: &AddressClaimRef{ + APIGroup: "ipam.miloapis.com", + Kind: "IPClaim", + Project: "datum-network-edge", + Namespace: "egress", + Name: "egress-shard-node-a-ipv6", + }, + }, + } +} + +// TestEgressShardDeepCopy verifies that DeepCopy produces an independent +// copy: mutations to the copy must not affect the original. +func TestEgressShardDeepCopy(t *testing.T) { + orig := newTestEgressShard() + dup := orig.DeepCopy() + + dup.Spec.ShardAddressIPv6 = "2001:db8:f00d::200" + dup.Spec.ShardAddressIPv4 = "198.51.100.7" + dup.Spec.NAT64Prefix = "64:ff9b::/96" + dup.Labels[LabelEgressShardPool] = "dedicated-public" + dup.Spec.ShardAddressIPv6ClaimRef.Name = "some-other-claim" + dup.Status.Conditions = append(dup.Status.Conditions, metav1.Condition{Type: ConditionTypeProgrammed}) + + if orig.Spec.ShardAddressIPv6 != "2001:db8:f00d::100" { + t.Errorf("ShardAddressIPv6 mutated: got %q", orig.Spec.ShardAddressIPv6) + } + if orig.Spec.ShardAddressIPv4 != "" { + t.Errorf("ShardAddressIPv4 mutated: got %q", orig.Spec.ShardAddressIPv4) + } + if orig.Spec.NAT64Prefix != "" { + t.Errorf("NAT64Prefix mutated: got %q", orig.Spec.NAT64Prefix) + } + if orig.Labels[LabelEgressShardPool] != "shared-public" { + t.Errorf("pool label mutated: got %q", orig.Labels[LabelEgressShardPool]) + } + if orig.Spec.ShardAddressIPv6ClaimRef.Name != "egress-shard-node-a-ipv6" { + t.Errorf("claim ref aliased by DeepCopy: got %q", orig.Spec.ShardAddressIPv6ClaimRef.Name) + } + if len(orig.Status.Conditions) != 0 { + t.Errorf("Conditions mutated: got %v", orig.Status.Conditions) + } +} + +// TestEgressShardDeepCopyNil verifies DeepCopy on a nil pointer returns nil. +func TestEgressShardDeepCopyNil(t *testing.T) { + var s *EgressShard + if s.DeepCopy() != nil { + t.Error("DeepCopy on nil pointer should return nil") + } +} + +// TestEgressShardJSONRoundTrip verifies that the spec-side addresses and the +// status-side programmed values survive JSON marshal/unmarshal independently +// of each other. +func TestEgressShardJSONRoundTrip(t *testing.T) { + orig := newTestEgressShard() + orig.Spec.ShardAddressIPv4 = "198.51.100.7" + orig.Spec.NAT64Prefix = "64:ff9b::/96" + orig.Status = EgressShardStatus{ + ObservedGeneration: 3, + ShardSID: "2001:db8:ff01::", + ShardAddressIPv6: "2001:db8:f00d::100", + Conditions: []metav1.Condition{ + { + Type: ConditionTypeProgrammed, + Status: metav1.ConditionFalse, + Reason: ProgrammedReasonProgrammingFailed, + }, + }, + } + + data, err := json.Marshal(orig) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + var got EgressShard + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + if got.Spec.ShardAddressIPv6 != orig.Spec.ShardAddressIPv6 || + got.Spec.ShardAddressIPv4 != orig.Spec.ShardAddressIPv4 || + got.Spec.NAT64Prefix != orig.Spec.NAT64Prefix || + got.Spec.TargetRef != orig.Spec.TargetRef { + t.Errorf("Spec: got %+v, want %+v", got.Spec, orig.Spec) + } + if got.Spec.ShardAddressIPv6ClaimRef == nil || *got.Spec.ShardAddressIPv6ClaimRef != *orig.Spec.ShardAddressIPv6ClaimRef { + t.Errorf("ShardAddressIPv6ClaimRef: got %+v, want %+v", got.Spec.ShardAddressIPv6ClaimRef, orig.Spec.ShardAddressIPv6ClaimRef) + } + if got.Status.ShardSID != orig.Status.ShardSID { + t.Errorf("ShardSID: got %q, want %q", got.Status.ShardSID, orig.Status.ShardSID) + } + if got.Status.ShardAddressIPv6 != orig.Status.ShardAddressIPv6 { + t.Errorf("status ShardAddressIPv6: got %q, want %q", got.Status.ShardAddressIPv6, orig.Status.ShardAddressIPv6) + } + if got.Status.ShardAddressIPv4 != "" { + t.Errorf("status ShardAddressIPv4: got %q, want empty", got.Status.ShardAddressIPv4) + } + if len(got.Status.Conditions) != 1 || got.Status.Conditions[0].Reason != ProgrammedReasonProgrammingFailed { + t.Errorf("Conditions: got %+v", got.Status.Conditions) + } +} + +// TestEgressShardSpecFieldNames verifies the spec JSON keys, which the cell +// controller writes and the CRD's CEL rules name. +func TestEgressShardSpecFieldNames(t *testing.T) { + orig := newTestEgressShard() + orig.Spec.ShardAddressIPv4 = "198.51.100.7" + orig.Spec.NAT64Prefix = "64:ff9b::/96" + + data, err := json.Marshal(orig.Spec) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + var m map[string]any + if err := json.Unmarshal(data, &m); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + want := map[string]string{ + "shardAddressIPv6": "2001:db8:f00d::100", + "shardAddressIPv4": "198.51.100.7", + "nat64Prefix": "64:ff9b::/96", + } + for key, value := range want { + if raw, ok := m[key]; !ok || raw != value { + t.Errorf("%s: got %v, want %q", key, raw, value) + } + } + if _, ok := m["targetRef"]; !ok { + t.Error("expected \"targetRef\" key to be present") + } + + claimRef, ok := m["shardAddressIPv6ClaimRef"].(map[string]any) + if !ok { + t.Fatalf("shardAddressIPv6ClaimRef: got %v, want an object", m["shardAddressIPv6ClaimRef"]) + } + wantRef := map[string]string{ + "apiGroup": "ipam.miloapis.com", + "kind": "IPClaim", + "project": "datum-network-edge", + "namespace": "egress", + "name": "egress-shard-node-a-ipv6", + } + for key, value := range wantRef { + if raw, ok := claimRef[key]; !ok || raw != value { + t.Errorf("shardAddressIPv6ClaimRef.%s: got %v, want %q", key, raw, value) + } + } +} + +// TestEgressShardSpecOmitEmpty verifies that a shard with no address assigned +// yet omits every address key, which is what distinguishes an unclaimed +// address from a blank one. +func TestEgressShardSpecOmitEmpty(t *testing.T) { + spec := EgressShardSpec{TargetRef: TargetRef{Kind: "Node", Name: "node-a"}} + + data, err := json.Marshal(spec) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + var m map[string]any + if err := json.Unmarshal(data, &m); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + for _, key := range []string{ + "shardAddressIPv6", "shardAddressIPv4", "nat64Prefix", + "shardAddressIPv6ClaimRef", "shardAddressIPv4ClaimRef", + } { + if _, ok := m[key]; ok { + t.Errorf("expected %q key to be absent when empty", key) + } + } +} + +// TestEgressShardLabelKeys verifies the selection label keys stay in this +// API group's own namespace. A key naming another group would make a data +// plane resource carry a reference to the API that selects it. +func TestEgressShardLabelKeys(t *testing.T) { + cases := map[string]string{ + "pool": LabelEgressShardPool, + "cell": LabelEgressShardCell, + "ipv6": LabelEgressShardIPv6, + "ipv4": LabelEgressShardIPv4, + } + for name, key := range cases { + t.Run(name, func(t *testing.T) { + const prefix = "network.datumapis.com/" + if len(key) <= len(prefix) || key[:len(prefix)] != prefix { + t.Errorf("label key %q: want prefix %q", key, prefix) + } + }) + } +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 4a4a01e..17eafb4 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -41,6 +41,21 @@ func (in *ASPathFilter) DeepCopy() *ASPathFilter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddressClaimRef) DeepCopyInto(out *AddressClaimRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddressClaimRef. +func (in *AddressClaimRef) DeepCopy() *AddressClaimRef { + if in == nil { + return nil + } + out := new(AddressClaimRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AddressFamily) DeepCopyInto(out *AddressFamily) { *out = *in @@ -1266,7 +1281,7 @@ func (in *EgressShard) DeepCopyInto(out *EgressShard) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -1324,6 +1339,16 @@ func (in *EgressShardList) DeepCopyObject() runtime.Object { func (in *EgressShardSpec) DeepCopyInto(out *EgressShardSpec) { *out = *in out.TargetRef = in.TargetRef + if in.ShardAddressIPv6ClaimRef != nil { + in, out := &in.ShardAddressIPv6ClaimRef, &out.ShardAddressIPv6ClaimRef + *out = new(AddressClaimRef) + **out = **in + } + if in.ShardAddressIPv4ClaimRef != nil { + in, out := &in.ShardAddressIPv4ClaimRef, &out.ShardAddressIPv4ClaimRef + *out = new(AddressClaimRef) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardSpec. diff --git a/config/crd/network.datumapis.com_egressshards.yaml b/config/crd/network.datumapis.com_egressshards.yaml index eb1e90e..cdb1f27 100644 --- a/config/crd/network.datumapis.com_egressshards.yaml +++ b/config/crd/network.datumapis.com_egressshards.yaml @@ -48,10 +48,9 @@ spec: (IPv6 -> IPv4, RFC 6146) are the same function — stateful egress PAT with a VRF-scoped session table — over different families, so one shard object describes both rather than there being a second, near-duplicate kind. - Status.ShardAddressIPv6 and Status.ShardAddressIPv4 are each set only for - the family this shard actually translates; a shard serving only NAT66 - leaves the IPv4 field empty and behaves exactly as it did before NAT64 - existed. + Spec.ShardAddressIPv6 and Spec.ShardAddressIPv4 are each assigned only for + the family this shard translates; a shard with no IPv4 address performs no + NAT64. Every shard owns a dedicated, publicly-routable address per family it serves, and a flow's allocated masquerade port lives within it — so a reply @@ -59,6 +58,21 @@ spec: no hashing or cross-shard lookup on the return path at all (the "any node can determine the owning shard from the tuple alone" property, satisfied by construction rather than by a replicated hash table). + + A shard is told which addresses to translate to; it does not choose them. + The controller that owns a cell claims one address per family from the + addressing service and writes it into this spec, which keeps the + addressing-service credential off every translating node and keeps the + allocation request out of the path that attaches a workload. Status reports + what the node's datapath is actually programmed with, so an unclaimed + address, a stale datapath, and a divergence between the two are each + distinguishable. + + A shard names nothing that selects it. Which shards serve which consumer + intent is decided entirely by label selectors evaluated on the selecting + side (see the LabelEgressShard* keys), so no consumer-facing API appears in + this group and no reference points from a datapath resource back at the + resource that placed traffic on it. properties: apiVersion: description: |- @@ -78,8 +92,178 @@ spec: metadata: type: object spec: - description: EgressShardSpec defines the desired state of an EgressShard. + description: |- + EgressShardSpec defines the desired state of an EgressShard. + + The address fields are written by the controller that owns the cell, not by + the shard and not by a consumer. Each is optional: an address the addressing + service has not yet handed out is absent rather than blank-but-required, so + a shard object exists from the moment its node is labelled and gains its + identity afterwards. + + Each is also write-once, and cannot be unassigned once assigned. The + datapath claims a reply by exact match against the address it translates to, + so reassigning one strands the return traffic of every flow already + established through it, with no drain and no dual-address grace period + available to cover the change. A shard holding the wrong address is deleted + and recreated instead, which breaks those flows at a moment someone chose. + + A claim reference alongside an address records where that address came from, + and is write-once on the same terms. It is deliberately not required with an + address, in either direction. Requiring an address before a claim reference + would forbid recording a claim whose allocation has not resolved yet, which + is the window a restarting controller most needs to see so that it finds the + claim it already made instead of making a second one. Requiring a claim + reference before an address would forbid an address allocated by hand, and + whether such an address may be recorded here at all is an open migration + question this validation would answer by fiat. An address with no reference + is therefore accepted and is unattributable, which is a fact about the + allocation rather than something a schema can repair. properties: + nat64Prefix: + description: |- + NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard + translates to IPv4 — one Datum-operated Network-Specific Prefix, shared + fabric-wide, never per-tenant. It must be the prefix the resolver + synthesizes into; a shard translating for a different one is a + blackhole with no symptom on either side. + + Set together with ShardAddressIPv4 or not at all: an address with no + prefix has nothing to translate for, and a prefix with no address has + nothing to translate into. Write-once for the same reason the addresses + are: a shard that starts translating a different prefix blackholes every + destination the resolver already synthesized into the old one. + type: string + x-kubernetes-validations: + - message: nat64Prefix must be a valid CIDR + rule: self == '' || isCIDR(self) + - message: nat64Prefix is immutable once assigned + rule: self == oldSelf || oldSelf == '' + shardAddressIPv4: + description: |- + ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this + shard translates to, and the source an IPv4-only destination sees. + Unlike ShardAddressIPv6, reachability for it is not established by a + BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the + internet, so the underlay or an upstream announcement must attract this + address to this node. + + Empty means no IPv4 address is assigned to this shard. + type: string + x-kubernetes-validations: + - message: shardAddressIPv4 must be a valid IPv4 address + rule: self == '' || (isIP(self) && ip(self).family() == 4) + - message: shardAddressIPv4 is immutable once assigned + rule: self == oldSelf || oldSelf == '' + shardAddressIPv4ClaimRef: + description: |- + ShardAddressIPv4ClaimRef records the addressing-service claim + ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same + write as the address whenever that address came from a claim. + + Nothing here resolves it and no component reads it to program anything; + it is the only trail from a translating address back to the allocation + accountable for it. + properties: + apiGroup: + default: ipam.miloapis.com + description: APIGroup is the group of the claim resource. + minLength: 1 + type: string + kind: + default: IPClaim + description: Kind is the kind of the claim resource. + minLength: 1 + type: string + name: + description: |- + Name is the name of the claim. The addressing service derives it from + the thing being addressed, so it is stable across a shard's lifetime and + is what a replacement finds again. + minLength: 1 + type: string + namespace: + description: Namespace is the namespace holding the claim within + Project. + minLength: 1 + type: string + project: + description: |- + Project is the project whose control plane serves the claim. Required: + a claim is namespaced within a project, and this reference is read from + outside every project. + minLength: 1 + type: string + required: + - name + - namespace + - project + type: object + x-kubernetes-validations: + - message: shardAddressIPv4ClaimRef is immutable once assigned + rule: self == oldSelf + shardAddressIPv6: + description: |- + ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this + shard translates to — every NAT66 masquerade port it allocates lives + within this address, so any node can route a reply to the owning shard + using ordinary unicast routing on it alone, with no per-flow state + lookup anywhere but that shard. + + Empty means no IPv6 address is assigned to this shard. + type: string + x-kubernetes-validations: + - message: shardAddressIPv6 must be a valid IPv6 address + rule: self == '' || (isIP(self) && ip(self).family() == 6) + - message: shardAddressIPv6 is immutable once assigned + rule: self == oldSelf || oldSelf == '' + shardAddressIPv6ClaimRef: + description: |- + ShardAddressIPv6ClaimRef records the addressing-service claim + ShardAddressIPv6 came from — see AddressClaimRef. Set it in the same + write as the address whenever that address came from a claim. + + Nothing here resolves it and no component reads it to program anything; + it is the only trail from a translating address back to the allocation + accountable for it. + properties: + apiGroup: + default: ipam.miloapis.com + description: APIGroup is the group of the claim resource. + minLength: 1 + type: string + kind: + default: IPClaim + description: Kind is the kind of the claim resource. + minLength: 1 + type: string + name: + description: |- + Name is the name of the claim. The addressing service derives it from + the thing being addressed, so it is stable across a shard's lifetime and + is what a replacement finds again. + minLength: 1 + type: string + namespace: + description: Namespace is the namespace holding the claim within + Project. + minLength: 1 + type: string + project: + description: |- + Project is the project whose control plane serves the claim. Required: + a claim is namespaced within a project, and this reference is read from + outside every project. + minLength: 1 + type: string + required: + - name + - namespace + - project + type: object + x-kubernetes-validations: + - message: shardAddressIPv6ClaimRef is immutable once assigned + rule: self == oldSelf targetRef: description: TargetRef identifies the Node this shard executes on. properties: @@ -98,18 +282,38 @@ spec: required: - targetRef type: object + x-kubernetes-validations: + - message: shardAddressIPv4 and nat64Prefix must be set together + rule: (has(self.shardAddressIPv4) && size(self.shardAddressIPv4) > 0) + == (has(self.nat64Prefix) && size(self.nat64Prefix) > 0) + - message: shardAddressIPv6 cannot be unassigned once assigned + rule: '!(has(oldSelf.shardAddressIPv6) && size(oldSelf.shardAddressIPv6) + > 0) || (has(self.shardAddressIPv6) && size(self.shardAddressIPv6) + > 0)' + - message: shardAddressIPv4 cannot be unassigned once assigned + rule: '!(has(oldSelf.shardAddressIPv4) && size(oldSelf.shardAddressIPv4) + > 0) || (has(self.shardAddressIPv4) && size(self.shardAddressIPv4) + > 0)' + - message: nat64Prefix cannot be unassigned once assigned + rule: '!(has(oldSelf.nat64Prefix) && size(oldSelf.nat64Prefix) > 0) + || (has(self.nat64Prefix) && size(self.nat64Prefix) > 0)' + - message: shardAddressIPv6ClaimRef cannot be unassigned once assigned + rule: '!has(oldSelf.shardAddressIPv6ClaimRef) || has(self.shardAddressIPv6ClaimRef)' + - message: shardAddressIPv4ClaimRef cannot be unassigned once assigned + rule: '!has(oldSelf.shardAddressIPv4ClaimRef) || has(self.shardAddressIPv4ClaimRef)' status: description: |- EgressShardStatus defines the observed state of an EgressShard. - Every field here is echoed from what the shard's datapath process was - actually started with, not derived: the shard publishes what it is running, - so a status that disagrees with an operator's intent is a visible - misconfiguration rather than a silently reconciled one. + The address and prefix fields report what this node's datapath is programmed + with, not what it was asked for. A value here that disagrees with the spec is + a shard that has not converged; a value here with no counterpart in the spec + is a shard still translating to an address nothing assigns any more. properties: conditions: - description: Conditions contains the standard conditions for this - resource. + description: |- + Conditions contains the standard conditions for this resource, + including Programmed (see ConditionTypeProgrammed). items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -170,14 +374,8 @@ spec: x-kubernetes-list-type: map nat64Prefix: description: |- - NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard - translates to IPv4 — one Datum-operated Network-Specific Prefix, shared - fabric-wide, never per-tenant. It is echoed here, rather than only - existing as process configuration, because it is the single fact DNS64 - synthesis has to agree with: a shard translating for a different prefix - than the resolver synthesizes into is otherwise a silent blackhole. - - Empty whenever ShardAddressIPv4 is empty. + NAT64Prefix is the prefix this shard's datapath is programmed to + translate. Empty whenever ShardAddressIPv4 is empty. type: string x-kubernetes-validations: - message: nat64Prefix must be a valid CIDR @@ -189,31 +387,18 @@ spec: type: integer shardAddressIPv4: description: |- - ShardAddressIPv4 is this shard's own dedicated, publicly-routable IPv4 - address — every NAT64 masquerade port this shard allocates lives within - it, and it is the source an IPv4-only destination sees. Unlike - ShardAddressIPv6, reachability for this address is not established by a - BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the - internet, so the address must be attracted to this node by the underlay - or upstream announcement instead. Publishing it here is what makes that - operator prerequisite checkable. - - Empty means this shard does not perform NAT64. + ShardAddressIPv4 is the IPv4 masquerade source this shard's datapath is + programmed with. Empty means it performs no NAT64. Publishing it is also + what makes the underlay reachability prerequisite in + Spec.ShardAddressIPv4 checkable. type: string x-kubernetes-validations: - message: shardAddressIPv4 must be a valid IPv4 address rule: self == '' || (isIP(self) && ip(self).family() == 4) shardAddressIPv6: description: |- - ShardAddressIPv6 is this shard's own dedicated, publicly-routable IPv6 - address — every NAT66 masquerade port this shard allocates lives within - it, so any node can route a reply to the correct shard using ordinary - unicast routing on this address alone, with no per-flow state lookup - anywhere but the owning shard itself. Operator-supplied per shard today - (no in-cluster derivation mechanism yet — the same gap - BGPRouter.Spec.SRv6Locator/NodeID assignment has today). - - Empty means this shard does not perform IPv6-to-IPv6 translation. + ShardAddressIPv6 is the IPv6 masquerade source this shard's datapath is + programmed with. Empty means it translates no IPv6 flow. type: string x-kubernetes-validations: - message: shardAddressIPv6 must be a valid IPv6 address @@ -228,6 +413,12 @@ spec: route against it. One SID serves both families: which translation a packet gets is decided from the inner destination, not from a second SID. + + Still chosen by an operator and reported here rather than assigned in + spec, unlike the addresses: a value another node already uses silently + diverts that node's traffic, so the assignment belongs to the + addressing service, which does not hand out identifiers of this kind + yet. type: string x-kubernetes-validations: - message: shardSID must be a valid IPv6 address diff --git a/docs/api/bgp.md b/docs/api/bgp.md index 09871e7..835937b 100644 --- a/docs/api/bgp.md +++ b/docs/api/bgp.md @@ -74,6 +74,52 @@ _Appears in:_ | `contains` | ASPathMatchContains requires the pattern to match a substring of the AS path.
| +#### AddressClaimRef + + + +AddressClaimRef names the addressing-service claim an assigned address came +from. It is a record, not a lookup: the addressing service is an aggregated +API served on a project control-plane path and defines no CustomResourceDefinition, +so no claim object exists in the cluster a shard runs in and nothing here +resolves one. Reading it means addressing that project's control plane +directly, with a credential a translating node is deliberately not given. + +Every field is a plain string and this group depends on no addressing-service +type. The shape matches that service's own opaque cross-API reference, plus +the project, because a claim is namespaced within a project and a namespace +name alone does not identify one from outside. + +Populate it with the coordinates of the claim whose allocation produced the +address written alongside it: + + shardAddressIPv6: 2001:db8:f00d::100 + shardAddressIPv6ClaimRef: + project: datum-network-edge + namespace: egress + name: egress-shard-node-a-ipv6 + +The claim cannot record this relationship itself. The addressing service +overwrites a claim's spec.ownerRef on create with the identity that requested +it, so a claim made by a cell controller is attributed to that controller's +project rather than to the shard it was made for. Attribution therefore runs +in this direction only, and an operator asking which claim holds a shard's +address has this field or nothing. + + + +_Appears in:_ +- [EgressShardSpec](#egressshardspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiGroup` _string_ | APIGroup is the group of the claim resource. | ipam.miloapis.com | MinLength: 1
| +| `kind` _string_ | Kind is the kind of the claim resource. | IPClaim | MinLength: 1
| +| `project` _string_ | Project is the project whose control plane serves the claim. Required:
a claim is namespaced within a project, and this reference is read from
outside every project. | | MinLength: 1
Required: \{\}
| +| `namespace` _string_ | Namespace is the namespace holding the claim within Project. | | MinLength: 1
Required: \{\}
| +| `name` _string_ | Name is the name of the claim. The addressing service derives it from
the thing being addressed, so it is stable across a shard's lifetime and
is what a replacement finds again. | | MinLength: 1
Required: \{\}
| + + #### AddressFamily @@ -932,10 +978,9 @@ A shard serves one or both address families. NAT66 (IPv6 -> IPv6) and NAT64 (IPv6 -> IPv4, RFC 6146) are the same function — stateful egress PAT with a VRF-scoped session table — over different families, so one shard object describes both rather than there being a second, near-duplicate kind. -Status.ShardAddressIPv6 and Status.ShardAddressIPv4 are each set only for -the family this shard actually translates; a shard serving only NAT66 -leaves the IPv4 field empty and behaves exactly as it did before NAT64 -existed. +Spec.ShardAddressIPv6 and Spec.ShardAddressIPv4 are each assigned only for +the family this shard translates; a shard with no IPv4 address performs no +NAT64. Every shard owns a dedicated, publicly-routable address per family it serves, and a flow's allocated masquerade port lives within it — so a reply @@ -944,6 +989,21 @@ no hashing or cross-shard lookup on the return path at all (the "any node can determine the owning shard from the tuple alone" property, satisfied by construction rather than by a replicated hash table). +A shard is told which addresses to translate to; it does not choose them. +The controller that owns a cell claims one address per family from the +addressing service and writes it into this spec, which keeps the +addressing-service credential off every translating node and keeps the +allocation request out of the path that attaches a workload. Status reports +what the node's datapath is actually programmed with, so an unclaimed +address, a stale datapath, and a divergence between the two are each +distinguishable. + +A shard names nothing that selects it. Which shards serve which consumer +intent is decided entirely by label selectors evaluated on the selecting +side (see the LabelEgressShard* keys), so no consumer-facing API appears in +this group and no reference points from a datapath resource back at the +resource that placed traffic on it. + @@ -965,6 +1025,31 @@ construction rather than by a replicated hash table). EgressShardSpec defines the desired state of an EgressShard. +The address fields are written by the controller that owns the cell, not by +the shard and not by a consumer. Each is optional: an address the addressing +service has not yet handed out is absent rather than blank-but-required, so +a shard object exists from the moment its node is labelled and gains its +identity afterwards. + +Each is also write-once, and cannot be unassigned once assigned. The +datapath claims a reply by exact match against the address it translates to, +so reassigning one strands the return traffic of every flow already +established through it, with no drain and no dual-address grace period +available to cover the change. A shard holding the wrong address is deleted +and recreated instead, which breaks those flows at a moment someone chose. + +A claim reference alongside an address records where that address came from, +and is write-once on the same terms. It is deliberately not required with an +address, in either direction. Requiring an address before a claim reference +would forbid recording a claim whose allocation has not resolved yet, which +is the window a restarting controller most needs to see so that it finds the +claim it already made instead of making a second one. Requiring a claim +reference before an address would forbid an address allocated by hand, and +whether such an address may be recorded here at all is an open migration +question this validation would answer by fiat. An address with no reference +is therefore accepted and is unattributable, which is a fact about the +allocation rather than something a schema can repair. + _Appears in:_ @@ -973,6 +1058,11 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| +| `shardAddressIPv6` _string_ | ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this
shard translates to — every NAT66 masquerade port it allocates lives
within this address, so any node can route a reply to the owning shard
using ordinary unicast routing on it alone, with no per-flow state
lookup anywhere but that shard.
Empty means no IPv6 address is assigned to this shard. | | | +| `shardAddressIPv6ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv6ClaimRef records the addressing-service claim
ShardAddressIPv6 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | +| `shardAddressIPv4` _string_ | ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this
shard translates to, and the source an IPv4-only destination sees.
Unlike ShardAddressIPv6, reachability for it is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the underlay or an upstream announcement must attract this
address to this node.
Empty means no IPv4 address is assigned to this shard. | | | +| `shardAddressIPv4ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv4ClaimRef records the addressing-service claim
ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | #### EgressShardStatus @@ -981,10 +1071,10 @@ _Appears in:_ EgressShardStatus defines the observed state of an EgressShard. -Every field here is echoed from what the shard's datapath process was -actually started with, not derived: the shard publishes what it is running, -so a status that disagrees with an operator's intent is a visible -misconfiguration rather than a silently reconciled one. +The address and prefix fields report what this node's datapath is programmed +with, not what it was asked for. A value here that disagrees with the spec is +a shard that has not converged; a value here with no counterpart in the spec +is a shard still translating to an address nothing assigns any more. @@ -994,11 +1084,11 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | -| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID. | | | -| `shardAddressIPv6` _string_ | ShardAddressIPv6 is this shard's own dedicated, publicly-routable IPv6
address — every NAT66 masquerade port this shard allocates lives within
it, so any node can route a reply to the correct shard using ordinary
unicast routing on this address alone, with no per-flow state lookup
anywhere but the owning shard itself. Operator-supplied per shard today
(no in-cluster derivation mechanism yet — the same gap
BGPRouter.Spec.SRv6Locator/NodeID assignment has today).
Empty means this shard does not perform IPv6-to-IPv6 translation. | | | -| `shardAddressIPv4` _string_ | ShardAddressIPv4 is this shard's own dedicated, publicly-routable IPv4
address — every NAT64 masquerade port this shard allocates lives within
it, and it is the source an IPv4-only destination sees. Unlike
ShardAddressIPv6, reachability for this address is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the address must be attracted to this node by the underlay
or upstream announcement instead. Publishing it here is what makes that
operator prerequisite checkable.
Empty means this shard does not perform NAT64. | | | -| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It is echoed here, rather than only
existing as process configuration, because it is the single fact DNS64
synthesis has to agree with: a shard translating for a different prefix
than the resolver synthesizes into is otherwise a silent blackhole.
Empty whenever ShardAddressIPv4 is empty. | | | -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource. | | | +| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID.
Still chosen by an operator and reported here rather than assigned in
spec, unlike the addresses: a value another node already uses silently
diverts that node's traffic, so the assignment belongs to the
addressing service, which does not hand out identifiers of this kind
yet. | | | +| `shardAddressIPv6` _string_ | ShardAddressIPv6 is the IPv6 masquerade source this shard's datapath is
programmed with. Empty means it translates no IPv6 flow. | | | +| `shardAddressIPv4` _string_ | ShardAddressIPv4 is the IPv4 masquerade source this shard's datapath is
programmed with. Empty means it performs no NAT64. Publishing it is also
what makes the underlay reachability prerequisite in
Spec.ShardAddressIPv4 checkable. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the prefix this shard's datapath is programmed to
translate. Empty whenever ShardAddressIPv4 is empty. | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource,
including Programmed (see ConditionTypeProgrammed). | | | #### ExtendedCommunitySet diff --git a/docs/api/gateway.md b/docs/api/gateway.md index e00b33d..9cfebfe 100644 --- a/docs/api/gateway.md +++ b/docs/api/gateway.md @@ -17,6 +17,52 @@ Package v1alpha1 contains API Schema definitions for the network.datumapis.com/v +#### AddressClaimRef + + + +AddressClaimRef names the addressing-service claim an assigned address came +from. It is a record, not a lookup: the addressing service is an aggregated +API served on a project control-plane path and defines no CustomResourceDefinition, +so no claim object exists in the cluster a shard runs in and nothing here +resolves one. Reading it means addressing that project's control plane +directly, with a credential a translating node is deliberately not given. + +Every field is a plain string and this group depends on no addressing-service +type. The shape matches that service's own opaque cross-API reference, plus +the project, because a claim is namespaced within a project and a namespace +name alone does not identify one from outside. + +Populate it with the coordinates of the claim whose allocation produced the +address written alongside it: + + shardAddressIPv6: 2001:db8:f00d::100 + shardAddressIPv6ClaimRef: + project: datum-network-edge + namespace: egress + name: egress-shard-node-a-ipv6 + +The claim cannot record this relationship itself. The addressing service +overwrites a claim's spec.ownerRef on create with the identity that requested +it, so a claim made by a cell controller is attributed to that controller's +project rather than to the shard it was made for. Attribution therefore runs +in this direction only, and an operator asking which claim holds a shard's +address has this field or nothing. + + + +_Appears in:_ +- [EgressShardSpec](#egressshardspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiGroup` _string_ | APIGroup is the group of the claim resource. | ipam.miloapis.com | MinLength: 1
| +| `kind` _string_ | Kind is the kind of the claim resource. | IPClaim | MinLength: 1
| +| `project` _string_ | Project is the project whose control plane serves the claim. Required:
a claim is namespaced within a project, and this reference is read from
outside every project. | | MinLength: 1
Required: \{\}
| +| `namespace` _string_ | Namespace is the namespace holding the claim within Project. | | MinLength: 1
Required: \{\}
| +| `name` _string_ | Name is the name of the claim. The addressing service derives it from
the thing being addressed, so it is stable across a shard's lifetime and
is what a replacement finds again. | | MinLength: 1
Required: \{\}
| + + #### EgressShard @@ -33,10 +79,9 @@ A shard serves one or both address families. NAT66 (IPv6 -> IPv6) and NAT64 (IPv6 -> IPv4, RFC 6146) are the same function — stateful egress PAT with a VRF-scoped session table — over different families, so one shard object describes both rather than there being a second, near-duplicate kind. -Status.ShardAddressIPv6 and Status.ShardAddressIPv4 are each set only for -the family this shard actually translates; a shard serving only NAT66 -leaves the IPv4 field empty and behaves exactly as it did before NAT64 -existed. +Spec.ShardAddressIPv6 and Spec.ShardAddressIPv4 are each assigned only for +the family this shard translates; a shard with no IPv4 address performs no +NAT64. Every shard owns a dedicated, publicly-routable address per family it serves, and a flow's allocated masquerade port lives within it — so a reply @@ -45,6 +90,21 @@ no hashing or cross-shard lookup on the return path at all (the "any node can determine the owning shard from the tuple alone" property, satisfied by construction rather than by a replicated hash table). +A shard is told which addresses to translate to; it does not choose them. +The controller that owns a cell claims one address per family from the +addressing service and writes it into this spec, which keeps the +addressing-service credential off every translating node and keeps the +allocation request out of the path that attaches a workload. Status reports +what the node's datapath is actually programmed with, so an unclaimed +address, a stale datapath, and a divergence between the two are each +distinguishable. + +A shard names nothing that selects it. Which shards serve which consumer +intent is decided entirely by label selectors evaluated on the selecting +side (see the LabelEgressShard* keys), so no consumer-facing API appears in +this group and no reference points from a datapath resource back at the +resource that placed traffic on it. + @@ -66,6 +126,31 @@ construction rather than by a replicated hash table). EgressShardSpec defines the desired state of an EgressShard. +The address fields are written by the controller that owns the cell, not by +the shard and not by a consumer. Each is optional: an address the addressing +service has not yet handed out is absent rather than blank-but-required, so +a shard object exists from the moment its node is labelled and gains its +identity afterwards. + +Each is also write-once, and cannot be unassigned once assigned. The +datapath claims a reply by exact match against the address it translates to, +so reassigning one strands the return traffic of every flow already +established through it, with no drain and no dual-address grace period +available to cover the change. A shard holding the wrong address is deleted +and recreated instead, which breaks those flows at a moment someone chose. + +A claim reference alongside an address records where that address came from, +and is write-once on the same terms. It is deliberately not required with an +address, in either direction. Requiring an address before a claim reference +would forbid recording a claim whose allocation has not resolved yet, which +is the window a restarting controller most needs to see so that it finds the +claim it already made instead of making a second one. Requiring a claim +reference before an address would forbid an address allocated by hand, and +whether such an address may be recorded here at all is an open migration +question this validation would answer by fiat. An address with no reference +is therefore accepted and is unattributable, which is a fact about the +allocation rather than something a schema can repair. + _Appears in:_ @@ -74,6 +159,11 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| +| `shardAddressIPv6` _string_ | ShardAddressIPv6 is the dedicated, publicly-routable IPv6 address this
shard translates to — every NAT66 masquerade port it allocates lives
within this address, so any node can route a reply to the owning shard
using ordinary unicast routing on it alone, with no per-flow state
lookup anywhere but that shard.
Empty means no IPv6 address is assigned to this shard. | | | +| `shardAddressIPv6ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv6ClaimRef records the addressing-service claim
ShardAddressIPv6 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | +| `shardAddressIPv4` _string_ | ShardAddressIPv4 is the dedicated, publicly-routable IPv4 address this
shard translates to, and the source an IPv4-only destination sees.
Unlike ShardAddressIPv6, reachability for it is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the underlay or an upstream announcement must attract this
address to this node.
Empty means no IPv4 address is assigned to this shard. | | | +| `shardAddressIPv4ClaimRef` _[AddressClaimRef](#addressclaimref)_ | ShardAddressIPv4ClaimRef records the addressing-service claim
ShardAddressIPv4 came from — see AddressClaimRef. Set it in the same
write as the address whenever that address came from a claim.
Nothing here resolves it and no component reads it to program anything;
it is the only trail from a translating address back to the allocation
accountable for it. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It must be the prefix the resolver
synthesizes into; a shard translating for a different one is a
blackhole with no symptom on either side.
Set together with ShardAddressIPv4 or not at all: an address with no
prefix has nothing to translate for, and a prefix with no address has
nothing to translate into. Write-once for the same reason the addresses
are: a shard that starts translating a different prefix blackholes every
destination the resolver already synthesized into the old one. | | | #### EgressShardStatus @@ -82,10 +172,10 @@ _Appears in:_ EgressShardStatus defines the observed state of an EgressShard. -Every field here is echoed from what the shard's datapath process was -actually started with, not derived: the shard publishes what it is running, -so a status that disagrees with an operator's intent is a visible -misconfiguration rather than a silently reconciled one. +The address and prefix fields report what this node's datapath is programmed +with, not what it was asked for. A value here that disagrees with the spec is +a shard that has not converged; a value here with no counterpart in the spec +is a shard still translating to an address nothing assigns any more. @@ -95,11 +185,11 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | -| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID. | | | -| `shardAddressIPv6` _string_ | ShardAddressIPv6 is this shard's own dedicated, publicly-routable IPv6
address — every NAT66 masquerade port this shard allocates lives within
it, so any node can route a reply to the correct shard using ordinary
unicast routing on this address alone, with no per-flow state lookup
anywhere but the owning shard itself. Operator-supplied per shard today
(no in-cluster derivation mechanism yet — the same gap
BGPRouter.Spec.SRv6Locator/NodeID assignment has today).
Empty means this shard does not perform IPv6-to-IPv6 translation. | | | -| `shardAddressIPv4` _string_ | ShardAddressIPv4 is this shard's own dedicated, publicly-routable IPv4
address — every NAT64 masquerade port this shard allocates lives within
it, and it is the source an IPv4-only destination sees. Unlike
ShardAddressIPv6, reachability for this address is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the address must be attracted to this node by the underlay
or upstream announcement instead. Publishing it here is what makes that
operator prerequisite checkable.
Empty means this shard does not perform NAT64. | | | -| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It is echoed here, rather than only
existing as process configuration, because it is the single fact DNS64
synthesis has to agree with: a shard translating for a different prefix
than the resolver synthesizes into is otherwise a silent blackhole.
Empty whenever ShardAddressIPv4 is empty. | | | -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource. | | | +| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID.
Still chosen by an operator and reported here rather than assigned in
spec, unlike the addresses: a value another node already uses silently
diverts that node's traffic, so the assignment belongs to the
addressing service, which does not hand out identifiers of this kind
yet. | | | +| `shardAddressIPv6` _string_ | ShardAddressIPv6 is the IPv6 masquerade source this shard's datapath is
programmed with. Empty means it translates no IPv6 flow. | | | +| `shardAddressIPv4` _string_ | ShardAddressIPv4 is the IPv4 masquerade source this shard's datapath is
programmed with. Empty means it performs no NAT64. Publishing it is also
what makes the underlay reachability prerequisite in
Spec.ShardAddressIPv4 checkable. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the prefix this shard's datapath is programmed to
translate. Empty whenever ShardAddressIPv4 is empty. | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource,
including Programmed (see ConditionTypeProgrammed). | | | diff --git a/test/e2e/Taskfile.yaml b/test/e2e/Taskfile.yaml index 79b92e8..2c057fd 100644 --- a/test/e2e/Taskfile.yaml +++ b/test/e2e/Taskfile.yaml @@ -91,6 +91,7 @@ tasks: crd/bgppolicies.network.datumapis.com crd/bgppeers.network.datumapis.com crd/bgpvrfinstances.network.datumapis.com + crd/egressshards.network.datumapis.com --timeout=60s deploy-bgp-routers: diff --git a/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml new file mode 100644 index 0000000..94a7f47 --- /dev/null +++ b/test/e2e/tests/egress-shard-address-immutability/chainsaw-test.yaml @@ -0,0 +1,310 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: egress-shard-address-immutability +spec: + description: > + Verify the CEL rules governing an EgressShard's assigned egress addresses + and the claim references that attribute them: a shard is accepted with no + address, the cell controller's first write is accepted, rewriting the same + value is accepted, and changing or unassigning an assigned value is + rejected. Also verifies that a claim reference defaults its apiGroup and + kind, and that an address and a claim reference are independently + optional. Also verifies that status remains writable, + since status reports what the datapath actually programmed and must still + converge. Does not require any operator to be running — every rule under + test is enforced by the API server alone. + steps: + - name: accept-shard-with-no-address + try: + - apply: + resource: + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard + labels: + network.datumapis.com/egress-pool: e2e-shared + network.datumapis.com/egress-cell: e2e-cell + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + - assert: + resource: + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard + (spec.shardAddressIPv6 == null): true + + - name: accept-first-address-assignment + try: + - script: + content: | + set -e + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardAddressIPv6":"2001:db8:f00d::100","shardAddressIPv4":"198.51.100.7","nat64Prefix":"64:ff9b::/96", + "shardAddressIPv6ClaimRef":{"project":"e2e-project","namespace":"egress","name":"e2e-claim-ipv6"}}}' + echo "OK: first assignment accepted" + - assert: + resource: + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard + spec: + shardAddressIPv6: 2001:db8:f00d::100 + shardAddressIPv4: 198.51.100.7 + nat64Prefix: 64:ff9b::/96 + shardAddressIPv6ClaimRef: + apiGroup: ipam.miloapis.com + kind: IPClaim + project: e2e-project + namespace: egress + name: e2e-claim-ipv6 + + - name: accept-same-value-rewrite + try: + - script: + content: | + set -e + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardAddressIPv6":"2001:db8:f00d::100"}}' + echo "OK: rewriting the same address accepted" + + - name: reject-address-change + try: + - script: + content: | + set +e + OUTPUT=$(kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardAddressIPv6":"2001:db8:f00d::200"}}' 2>&1) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of a changed shardAddressIPv6 but the patch was accepted" + exit 1 + fi + case "$OUTPUT" in + *"immutable once assigned"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: changing an assigned shardAddressIPv6 correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-prefix-change + try: + - script: + content: | + set +e + OUTPUT=$(kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"nat64Prefix":"64:ff9b:1::/96"}}' 2>&1) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of a changed nat64Prefix but the patch was accepted" + exit 1 + fi + echo "OK: changing an assigned nat64Prefix correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-address-unassignment + try: + - script: + content: | + set +e + OUTPUT=$(kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardAddressIPv6":null}}' 2>&1) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of an unassigned shardAddressIPv6 but the patch was accepted" + exit 1 + fi + case "$OUTPUT" in + *"cannot be unassigned once assigned"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: unassigning an assigned shardAddressIPv6 correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-ipv4-without-prefix + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard-unpaired + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + shardAddressIPv4: 198.51.100.8 + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of an IPv4 address with no NAT64 prefix but the resource was accepted" + kubectl delete egressshard e2e-egress-shard-unpaired -n "$NAMESPACE" 2>/dev/null || true + exit 1 + fi + case "$OUTPUT" in + *"must be set together"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: an IPv4 address with no NAT64 prefix correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-claim-ref-change + try: + - script: + content: | + set +e + OUTPUT=$(kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardAddressIPv6ClaimRef":{"project":"e2e-project","namespace":"egress","name":"e2e-claim-other"}}}' 2>&1) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of a changed shardAddressIPv6ClaimRef but the patch was accepted" + exit 1 + fi + case "$OUTPUT" in + *"immutable once assigned"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: changing an assigned claim reference correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-claim-ref-unassignment + try: + - script: + content: | + set +e + OUTPUT=$(kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" --type=merge -p ' + {"spec":{"shardAddressIPv6ClaimRef":null}}' 2>&1) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of an unassigned shardAddressIPv6ClaimRef but the patch was accepted" + exit 1 + fi + case "$OUTPUT" in + *"cannot be unassigned once assigned"*) ;; + *) echo "ERROR: rejected for the wrong reason: $OUTPUT"; exit 1 ;; + esac + echo "OK: unassigning an assigned claim reference correctly rejected" + echo "Server response: $OUTPUT" + + - name: reject-claim-ref-without-project + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard-projectless + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + shardAddressIPv6ClaimRef: + namespace: egress + name: e2e-claim-ipv6 + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: expected rejection of a claim reference with no project but the resource was accepted" + kubectl delete egressshard e2e-egress-shard-projectless -n "$NAMESPACE" 2>/dev/null || true + exit 1 + fi + echo "OK: a claim reference with no project correctly rejected" + echo "Server response: $OUTPUT" + + - name: default-claim-ref-group-and-kind + try: + - script: + content: | + set -e + kubectl apply -n "$NAMESPACE" -f - <<'EOF' + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard-defaults + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + shardAddressIPv6ClaimRef: + project: e2e-project + namespace: egress + name: e2e-claim-ipv6 + EOF + GROUP=$(kubectl get egressshard e2e-egress-shard-defaults -n "$NAMESPACE" \ + -o jsonpath='{.spec.shardAddressIPv6ClaimRef.apiGroup}') + KIND=$(kubectl get egressshard e2e-egress-shard-defaults -n "$NAMESPACE" \ + -o jsonpath='{.spec.shardAddressIPv6ClaimRef.kind}') + kubectl delete egressshard e2e-egress-shard-defaults -n "$NAMESPACE" + if [ "$GROUP" != "ipam.miloapis.com" ] || [ "$KIND" != "IPClaim" ]; then + echo "ERROR: claim reference defaults = $GROUP/$KIND, want ipam.miloapis.com/IPClaim" + exit 1 + fi + echo "OK: claim reference defaulted to $GROUP/$KIND" + + - name: accept-claim-ref-with-no-address + try: + - script: + content: | + set -e + kubectl apply -n "$NAMESPACE" -f - <<'EOF' + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard-pending + spec: + targetRef: + kind: Node + name: bgp-e2e-worker + shardAddressIPv4ClaimRef: + project: e2e-project + namespace: egress + name: e2e-claim-ipv4 + EOF + kubectl delete egressshard e2e-egress-shard-pending -n "$NAMESPACE" + echo "OK: a claim reference whose allocation has not resolved yet is accepted" + + - name: accept-status-changes + try: + - script: + content: | + set -e + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" \ + --subresource=status --type=merge -p ' + {"status":{"shardAddressIPv6":"2001:db8:f00d::100","shardSID":"2001:db8:ff01::"}}' + kubectl patch egressshard e2e-egress-shard -n "$NAMESPACE" \ + --subresource=status --type=merge -p ' + {"status":{"shardAddressIPv6":"2001:db8:f00d::200"}}' + echo "OK: status reports what the datapath programmed and stays mutable" + - assert: + resource: + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + metadata: + name: e2e-egress-shard + status: + shardAddressIPv6: 2001:db8:f00d::200 + + finally: + - delete: + ref: + apiVersion: network.datumapis.com/v1alpha1 + kind: EgressShard + name: e2e-egress-shard