diff --git a/api/v1alpha1/egresspolicy_types.go b/api/v1alpha1/egresspolicy_types.go index 4cdc839..f084b5a 100644 --- a/api/v1alpha1/egresspolicy_types.go +++ b/api/v1alpha1/egresspolicy_types.go @@ -5,8 +5,8 @@ import ( ) // NetworkEgressPolicy enables internet egress for a single tenant -// VPC/VPCAttachment, served by the sharded, stateful galactic-nat66 tier -// (see NAT66Shard). Unlike NetworkRule, it carries no VIP/backend/port: +// VPC/VPCAttachment, served by the sharded, stateful galactic-nat tier +// (see EgressShard). Unlike NetworkRule, it carries no VIP/backend/port: // egress is on or off for a (vpcRef, vpcAttachmentRef) pair, // existence-implies-enabled, not a per-flow rule — because the destination // of an egress flow is an arbitrary internet address, not a pre-configured @@ -20,12 +20,12 @@ import ( // accepted — see the Accepted condition. // // Presence of an accepted NetworkEgressPolicy resolves only *enablement* -// (should this tenant's VRF get a default route toward the shared NAT66 -// tier at all) — unlike this type's original design (superseded), there is -// no single "assigned gateway node" to compute or pin: any NAT66Shard may +// (should this tenant's VRF get an egress route toward the shared +// translation tier at all) — unlike this type's original design (superseded), there is +// no single "assigned gateway node" to compute or pin: any EgressShard may // serve any tenant's flow, chosen by the shard-placement consistent-hash // ring (internal/maglev, keyed on (tenant VRFID, backend, destination) — -// see NAT66Shard's doc comment), not by a per-tenant node assignment stored +// see EgressShard's doc comment), not by a per-tenant node assignment stored // here. *Isolation* (preventing two tenants with colliding ULA source // addresses from colliding in the egress connection table) is a separate, // datapath-level concern resolved by tagging each flow with the VRFID diff --git a/api/v1alpha1/egresspolicy_types_test.go b/api/v1alpha1/egresspolicy_types_test.go index 6ae6857..108407a 100644 --- a/api/v1alpha1/egresspolicy_types_test.go +++ b/api/v1alpha1/egresspolicy_types_test.go @@ -130,8 +130,8 @@ func TestNetworkEgressPolicyFieldNames(t *testing.T) { // TestNetworkEgressPolicyStatusHasNoAssignedGatewayNode is a regression // test: the earlier design pinned a policy to a single gateway node's -// masquerade datapath. The sharded galactic-nat66 tier has no such fixed -// assignment — any NAT66Shard may serve any tenant's flow, chosen by the +// masquerade datapath. The sharded galactic-nat tier has no such fixed +// assignment — any EgressShard may serve any tenant's flow, chosen by the // shard-placement consistent-hash ring, not a per-tenant node stored here. func TestNetworkEgressPolicyStatusHasNoAssignedGatewayNode(t *testing.T) { orig := newTestEgressPolicy() diff --git a/api/v1alpha1/egressshard_types.go b/api/v1alpha1/egressshard_types.go new file mode 100644 index 0000000..1118259 --- /dev/null +++ b/api/v1alpha1/egressshard_types.go @@ -0,0 +1,134 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EgressShard marks a single node as a member of the sharded, stateful egress +// translation tier (galactic-nat) — a component deliberately kept off the +// ingress load-balancer's own consistent-hash ring (see NetworkGateway): +// tenant egress traffic (backend -> arbitrary internet destination) is a +// different traffic pattern from ingress (fixed VIP, fixed backend pool) +// and needs its own placement ring, own per-flow state, and its own +// self-routing return path, entirely independent of any NetworkGateway node. +// +// A shard serves one or both address families. NAT66 (IPv6 -> IPv6) and NAT64 +// (IPv6 -> IPv4, RFC 6146) are the same function — stateful egress PAT with a +// VRF-scoped session table — over different families, so one shard object +// describes both rather than there being a second, near-duplicate kind. +// Status.ShardAddressIPv6 and Status.ShardAddressIPv4 are each set only for +// the family this shard actually translates; a shard serving only NAT66 +// leaves the IPv4 field empty and behaves exactly as it did before NAT64 +// existed. +// +// Every shard owns a dedicated, publicly-routable address per family it +// serves, and a flow's allocated masquerade port lives within it — so a reply +// is delivered to the correct shard by ordinary unicast routing alone, with +// no hashing or cross-shard lookup on the return path at all (the "any node +// can determine the owning shard from the tuple alone" property, satisfied by +// construction rather than by a replicated hash table). +// +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced,shortName=egressshard +// +kubebuilder:printcolumn:name="TARGET",type="string",JSONPath=".spec.targetRef.name" +// +kubebuilder:printcolumn:name="SHARD-SID",type="string",JSONPath=".status.shardSID" +// +kubebuilder:printcolumn:name="IPV6",type="string",JSONPath=".status.shardAddressIPv6" +// +kubebuilder:printcolumn:name="IPV4",type="string",JSONPath=".status.shardAddressIPv4" +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" +type EgressShard struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec EgressShardSpec `json:"spec,omitempty"` + Status EgressShardStatus `json:"status,omitempty"` +} + +// EgressShardSpec defines the desired state of an EgressShard. +type EgressShardSpec struct { + // TargetRef identifies the Node this shard executes on. + // +kubebuilder:validation:Required + TargetRef TargetRef `json:"targetRef"` +} + +// EgressShardStatus defines the observed state of an EgressShard. +// +// Every field here is echoed from what the shard's datapath process was +// actually started with, not derived: the shard publishes what it is running, +// so a status that disagrees with an operator's intent is a visible +// misconfiguration rather than a silently reconciled one. +type EgressShardStatus struct { + // ObservedGeneration is the .metadata.generation this status was computed from. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike + // the ShardAddress fields, which are plain routable addresses), + // advertised into BGP the same way any other node-reachability route is + // (a /128 BGPAdvertisement, no VRFID/Function) so every other node learns + // a kernel SEG6 route toward it before installing a tenant VRF's egress + // route against it. One SID serves both families: which translation a + // packet gets is decided from the inner destination, not from a second + // SID. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 6)",message="shardSID must be a valid IPv6 address" + ShardSID string `json:"shardSID,omitempty"` + + // ShardAddressIPv6 is this shard's own dedicated, publicly-routable IPv6 + // address — every NAT66 masquerade port this shard allocates lives within + // it, so any node can route a reply to the correct shard using ordinary + // unicast routing on this address alone, with no per-flow state lookup + // anywhere but the owning shard itself. Operator-supplied per shard today + // (no in-cluster derivation mechanism yet — the same gap + // BGPRouter.Spec.SRv6Locator/NodeID assignment has today). + // + // Empty means this shard does not perform IPv6-to-IPv6 translation. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 6)",message="shardAddressIPv6 must be a valid IPv6 address" + ShardAddressIPv6 string `json:"shardAddressIPv6,omitempty"` + + // ShardAddressIPv4 is this shard's own dedicated, publicly-routable IPv4 + // address — every NAT64 masquerade port this shard allocates lives within + // it, and it is the source an IPv4-only destination sees. Unlike + // ShardAddressIPv6, reachability for this address is not established by a + // BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the + // internet, so the address must be attracted to this node by the underlay + // or upstream announcement instead. Publishing it here is what makes that + // operator prerequisite checkable. + // + // Empty means this shard does not perform NAT64. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || (isIP(self) && ip(self).family() == 4)",message="shardAddressIPv4 must be a valid IPv4 address" + ShardAddressIPv4 string `json:"shardAddressIPv4,omitempty"` + + // NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard + // translates to IPv4 — one Datum-operated Network-Specific Prefix, shared + // fabric-wide, never per-tenant. It is echoed here, rather than only + // existing as process configuration, because it is the single fact DNS64 + // synthesis has to agree with: a shard translating for a different prefix + // than the resolver synthesizes into is otherwise a silent blackhole. + // + // Empty whenever ShardAddressIPv4 is empty. + // +optional + // +kubebuilder:validation:XValidation:rule="self == '' || isCIDR(self)",message="nat64Prefix must be a valid CIDR" + NAT64Prefix string `json:"nat64Prefix,omitempty"` + + // Conditions contains the standard conditions for this resource. + // + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// EgressShardList is a list of EgressShard resources. +// +kubebuilder:object:root=true +type EgressShardList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EgressShard `json:"items"` +} + +func init() { + SchemeBuilder.Register(&EgressShard{}, &EgressShardList{}) +} diff --git a/api/v1alpha1/nat66shard_types.go b/api/v1alpha1/nat66shard_types.go deleted file mode 100644 index 094bf3e..0000000 --- a/api/v1alpha1/nat66shard_types.go +++ /dev/null @@ -1,89 +0,0 @@ -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// NAT66Shard marks a single node as a member of the sharded, stateful NAT66 -// egress tier (galactic-nat66) — a component deliberately kept off the -// ingress load-balancer's own consistent-hash ring (see NetworkGateway): -// tenant egress traffic (backend -> arbitrary internet destination) is a -// different traffic pattern from ingress (fixed VIP, fixed backend pool) -// and needs its own placement ring, own per-flow state, and its own -// self-routing return path, entirely independent of any NetworkGateway node. -// -// Every shard owns a dedicated, BGP-advertised public IPv6 address -// (Status.ShardAddress) that a flow's allocated masquerade port lives -// within — so a reply is delivered to the correct shard by ordinary -// unicast SRv6/BGP routing alone, with no hashing or cross-shard lookup on -// the return path at all (the "any node can determine the owning shard from -// the tuple alone" property, satisfied by construction rather than by a -// replicated hash table). -// -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +kubebuilder:resource:scope=Namespaced,shortName=nat66shard -// +kubebuilder:printcolumn:name="TARGET",type="string",JSONPath=".spec.targetRef.name" -// +kubebuilder:printcolumn:name="SHARD-ADDRESS",type="string",JSONPath=".status.shardAddress" -// +kubebuilder:printcolumn:name="SHARD-SID",type="string",JSONPath=".status.shardSID" -// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" -type NAT66Shard struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec NAT66ShardSpec `json:"spec,omitempty"` - Status NAT66ShardStatus `json:"status,omitempty"` -} - -// NAT66ShardSpec defines the desired state of a NAT66Shard. -type NAT66ShardSpec struct { - // TargetRef identifies the Node this shard executes on. - // +kubebuilder:validation:Required - TargetRef TargetRef `json:"targetRef"` -} - -// NAT66ShardStatus defines the observed state of a NAT66Shard. -type NAT66ShardStatus struct { - // ObservedGeneration is the .metadata.generation this status was computed from. - // +optional - ObservedGeneration int64 `json:"observedGeneration,omitempty"` - - // ShardAddress is this shard's own dedicated, publicly-routable IPv6 - // address — every masquerade port this shard allocates lives within it, - // so any node can route a reply to the correct shard using ordinary - // unicast routing on this address alone, with no per-flow state lookup - // anywhere but the owning shard itself. Operator-supplied per shard - // today (no in-cluster derivation mechanism yet — the same gap - // BGPRouter.Spec.SRv6Locator/NodeID assignment has today). - // +optional - // +kubebuilder:validation:XValidation:rule="self == '' || isIP(self)",message="shardAddress must be a valid IPv6 address" - ShardAddress string `json:"shardAddress,omitempty"` - - // ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike - // ShardAddress, a plain routable address), advertised into BGP the same - // way any other node-reachability route is (a /128 BGPAdvertisement, no - // VRFID/Function) so every other node learns a kernel SEG6 route toward - // it before installing a tenant VRF's default egress route against it. - // +optional - // +kubebuilder:validation:XValidation:rule="self == '' || isIP(self)",message="shardSID must be a valid IPv6 address" - ShardSID string `json:"shardSID,omitempty"` - - // Conditions contains the standard conditions for this resource. - // - // +listType=map - // +listMapKey=type - // +optional - Conditions []metav1.Condition `json:"conditions,omitempty"` -} - -// NAT66ShardList is a list of NAT66Shard resources. -// +kubebuilder:object:root=true -type NAT66ShardList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []NAT66Shard `json:"items"` -} - -func init() { - SchemeBuilder.Register(&NAT66Shard{}, &NAT66ShardList{}) -} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3991830..4a4a01e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1262,47 +1262,7 @@ func (in *CommunitySet) DeepCopy() *CommunitySet { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExtendedCommunitySet) DeepCopyInto(out *ExtendedCommunitySet) { - *out = *in - if in.Add != nil { - in, out := &in.Add, &out.Add - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Remove != nil { - in, out := &in.Remove, &out.Remove - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtendedCommunitySet. -func (in *ExtendedCommunitySet) DeepCopy() *ExtendedCommunitySet { - if in == nil { - return nil - } - out := new(ExtendedCommunitySet) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LocalSecretRef) DeepCopyInto(out *LocalSecretRef) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalSecretRef. -func (in *LocalSecretRef) DeepCopy() *LocalSecretRef { - if in == nil { - return nil - } - out := new(LocalSecretRef) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NAT66Shard) DeepCopyInto(out *NAT66Shard) { +func (in *EgressShard) DeepCopyInto(out *EgressShard) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) @@ -1310,18 +1270,18 @@ func (in *NAT66Shard) DeepCopyInto(out *NAT66Shard) { in.Status.DeepCopyInto(&out.Status) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NAT66Shard. -func (in *NAT66Shard) DeepCopy() *NAT66Shard { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShard. +func (in *EgressShard) DeepCopy() *EgressShard { if in == nil { return nil } - out := new(NAT66Shard) + out := new(EgressShard) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *NAT66Shard) DeepCopyObject() runtime.Object { +func (in *EgressShard) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -1329,31 +1289,31 @@ func (in *NAT66Shard) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NAT66ShardList) DeepCopyInto(out *NAT66ShardList) { +func (in *EgressShardList) DeepCopyInto(out *EgressShardList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]NAT66Shard, len(*in)) + *out = make([]EgressShard, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NAT66ShardList. -func (in *NAT66ShardList) DeepCopy() *NAT66ShardList { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardList. +func (in *EgressShardList) DeepCopy() *EgressShardList { if in == nil { return nil } - out := new(NAT66ShardList) + out := new(EgressShardList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *NAT66ShardList) DeepCopyObject() runtime.Object { +func (in *EgressShardList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -1361,23 +1321,23 @@ func (in *NAT66ShardList) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NAT66ShardSpec) DeepCopyInto(out *NAT66ShardSpec) { +func (in *EgressShardSpec) DeepCopyInto(out *EgressShardSpec) { *out = *in out.TargetRef = in.TargetRef } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NAT66ShardSpec. -func (in *NAT66ShardSpec) DeepCopy() *NAT66ShardSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardSpec. +func (in *EgressShardSpec) DeepCopy() *EgressShardSpec { if in == nil { return nil } - out := new(NAT66ShardSpec) + out := new(EgressShardSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NAT66ShardStatus) DeepCopyInto(out *NAT66ShardStatus) { +func (in *EgressShardStatus) DeepCopyInto(out *EgressShardStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions @@ -1388,12 +1348,52 @@ func (in *NAT66ShardStatus) DeepCopyInto(out *NAT66ShardStatus) { } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NAT66ShardStatus. -func (in *NAT66ShardStatus) DeepCopy() *NAT66ShardStatus { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardStatus. +func (in *EgressShardStatus) DeepCopy() *EgressShardStatus { if in == nil { return nil } - out := new(NAT66ShardStatus) + out := new(EgressShardStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExtendedCommunitySet) DeepCopyInto(out *ExtendedCommunitySet) { + *out = *in + if in.Add != nil { + in, out := &in.Add, &out.Add + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Remove != nil { + in, out := &in.Remove, &out.Remove + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtendedCommunitySet. +func (in *ExtendedCommunitySet) DeepCopy() *ExtendedCommunitySet { + if in == nil { + return nil + } + out := new(ExtendedCommunitySet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocalSecretRef) DeepCopyInto(out *LocalSecretRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalSecretRef. +func (in *LocalSecretRef) DeepCopy() *LocalSecretRef { + if in == nil { + return nil + } + out := new(LocalSecretRef) in.DeepCopyInto(out) return out } diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 3554f72..ea068d4 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -7,8 +7,8 @@ resources: - network.datumapis.com_bgppolicies.yaml - network.datumapis.com_bgprouters.yaml - network.datumapis.com_bgpvrfinstances.yaml + - network.datumapis.com_egressshards.yaml - network.datumapis.com_networkegresspolicies.yaml - network.datumapis.com_networkgateways.yaml - network.datumapis.com_networkrules.yaml - network.datumapis.com_servicevipbindings.yaml - - network.datumapis.com_nat66shards.yaml diff --git a/config/crd/network.datumapis.com_nat66shards.yaml b/config/crd/network.datumapis.com_egressshards.yaml similarity index 56% rename from config/crd/network.datumapis.com_nat66shards.yaml rename to config/crd/network.datumapis.com_egressshards.yaml index e299012..eb1e90e 100644 --- a/config/crd/network.datumapis.com_nat66shards.yaml +++ b/config/crd/network.datumapis.com_egressshards.yaml @@ -4,28 +4,31 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.18.0 - name: nat66shards.network.datumapis.com + name: egressshards.network.datumapis.com spec: group: network.datumapis.com names: - kind: NAT66Shard - listKind: NAT66ShardList - plural: nat66shards + kind: EgressShard + listKind: EgressShardList + plural: egressshards shortNames: - - nat66shard - singular: nat66shard + - egressshard + singular: egressshard scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .spec.targetRef.name name: TARGET type: string - - jsonPath: .status.shardAddress - name: SHARD-ADDRESS - type: string - jsonPath: .status.shardSID name: SHARD-SID type: string + - jsonPath: .status.shardAddressIPv6 + name: IPV6 + type: string + - jsonPath: .status.shardAddressIPv4 + name: IPV4 + type: string - jsonPath: .metadata.creationTimestamp name: AGE type: date @@ -33,21 +36,29 @@ spec: schema: openAPIV3Schema: description: |- - NAT66Shard marks a single node as a member of the sharded, stateful NAT66 - egress tier (galactic-nat66) — a component deliberately kept off the + EgressShard marks a single node as a member of the sharded, stateful egress + translation tier (galactic-nat) — a component deliberately kept off the ingress load-balancer's own consistent-hash ring (see NetworkGateway): tenant egress traffic (backend -> arbitrary internet destination) is a different traffic pattern from ingress (fixed VIP, fixed backend pool) and needs its own placement ring, own per-flow state, and its own self-routing return path, entirely independent of any NetworkGateway node. - Every shard owns a dedicated, BGP-advertised public IPv6 address - (Status.ShardAddress) that a flow's allocated masquerade port lives - within — so a reply is delivered to the correct shard by ordinary - unicast SRv6/BGP routing alone, with no hashing or cross-shard lookup on - the return path at all (the "any node can determine the owning shard from - the tuple alone" property, satisfied by construction rather than by a - replicated hash table). + A shard serves one or both address families. NAT66 (IPv6 -> IPv6) and NAT64 + (IPv6 -> IPv4, RFC 6146) are the same function — stateful egress PAT with a + VRF-scoped session table — over different families, so one shard object + describes both rather than there being a second, near-duplicate kind. + Status.ShardAddressIPv6 and Status.ShardAddressIPv4 are each set only for + the family this shard actually translates; a shard serving only NAT66 + leaves the IPv4 field empty and behaves exactly as it did before NAT64 + existed. + + Every shard owns a dedicated, publicly-routable address per family it + serves, and a flow's allocated masquerade port lives within it — so a reply + is delivered to the correct shard by ordinary unicast routing alone, with + no hashing or cross-shard lookup on the return path at all (the "any node + can determine the owning shard from the tuple alone" property, satisfied by + construction rather than by a replicated hash table). properties: apiVersion: description: |- @@ -67,7 +78,7 @@ spec: metadata: type: object spec: - description: NAT66ShardSpec defines the desired state of a NAT66Shard. + description: EgressShardSpec defines the desired state of an EgressShard. properties: targetRef: description: TargetRef identifies the Node this shard executes on. @@ -88,7 +99,13 @@ spec: - targetRef type: object status: - description: NAT66ShardStatus defines the observed state of a NAT66Shard. + description: |- + EgressShardStatus defines the observed state of an EgressShard. + + Every field here is echoed from what the shard's datapath process was + actually started with, not derived: the shard publishes what it is running, + so a status that disagrees with an operator's intent is a visible + misconfiguration rather than a silently reconciled one. properties: conditions: description: Conditions contains the standard conditions for this @@ -151,35 +168,70 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + nat64Prefix: + description: |- + NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard + translates to IPv4 — one Datum-operated Network-Specific Prefix, shared + fabric-wide, never per-tenant. It is echoed here, rather than only + existing as process configuration, because it is the single fact DNS64 + synthesis has to agree with: a shard translating for a different prefix + than the resolver synthesizes into is otherwise a silent blackhole. + + Empty whenever ShardAddressIPv4 is empty. + type: string + x-kubernetes-validations: + - message: nat64Prefix must be a valid CIDR + rule: self == '' || isCIDR(self) observedGeneration: description: ObservedGeneration is the .metadata.generation this status was computed from. format: int64 type: integer - shardAddress: + shardAddressIPv4: description: |- - ShardAddress is this shard's own dedicated, publicly-routable IPv6 - address — every masquerade port this shard allocates lives within it, - so any node can route a reply to the correct shard using ordinary + ShardAddressIPv4 is this shard's own dedicated, publicly-routable IPv4 + address — every NAT64 masquerade port this shard allocates lives within + it, and it is the source an IPv4-only destination sees. Unlike + ShardAddressIPv6, reachability for this address is not established by a + BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the + internet, so the address must be attracted to this node by the underlay + or upstream announcement instead. Publishing it here is what makes that + operator prerequisite checkable. + + Empty means this shard does not perform NAT64. + type: string + x-kubernetes-validations: + - message: shardAddressIPv4 must be a valid IPv4 address + rule: self == '' || (isIP(self) && ip(self).family() == 4) + shardAddressIPv6: + description: |- + ShardAddressIPv6 is this shard's own dedicated, publicly-routable IPv6 + address — every NAT66 masquerade port this shard allocates lives within + it, so any node can route a reply to the correct shard using ordinary unicast routing on this address alone, with no per-flow state lookup - anywhere but the owning shard itself. Operator-supplied per shard - today (no in-cluster derivation mechanism yet — the same gap + anywhere but the owning shard itself. Operator-supplied per shard today + (no in-cluster derivation mechanism yet — the same gap BGPRouter.Spec.SRv6Locator/NodeID assignment has today). + + Empty means this shard does not perform IPv6-to-IPv6 translation. type: string x-kubernetes-validations: - - message: shardAddress must be a valid IPv6 address - rule: self == '' || isIP(self) + - message: shardAddressIPv6 must be a valid IPv6 address + rule: self == '' || (isIP(self) && ip(self).family() == 6) shardSID: description: |- ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike - ShardAddress, a plain routable address), advertised into BGP the same - way any other node-reachability route is (a /128 BGPAdvertisement, no - VRFID/Function) so every other node learns a kernel SEG6 route toward - it before installing a tenant VRF's default egress route against it. + the ShardAddress fields, which are plain routable addresses), + advertised into BGP the same way any other node-reachability route is + (a /128 BGPAdvertisement, no VRFID/Function) so every other node learns + a kernel SEG6 route toward it before installing a tenant VRF's egress + route against it. One SID serves both families: which translation a + packet gets is decided from the inner destination, not from a second + SID. type: string x-kubernetes-validations: - message: shardSID must be a valid IPv6 address - rule: self == '' || isIP(self) + rule: self == '' || (isIP(self) && ip(self).family() == 6) type: object type: object served: true diff --git a/config/crd/network.datumapis.com_networkegresspolicies.yaml b/config/crd/network.datumapis.com_networkegresspolicies.yaml index 2ce5517..a20527b 100644 --- a/config/crd/network.datumapis.com_networkegresspolicies.yaml +++ b/config/crd/network.datumapis.com_networkegresspolicies.yaml @@ -31,8 +31,8 @@ spec: openAPIV3Schema: description: |- NetworkEgressPolicy enables internet egress for a single tenant - VPC/VPCAttachment, served by the sharded, stateful galactic-nat66 tier - (see NAT66Shard). Unlike NetworkRule, it carries no VIP/backend/port: + VPC/VPCAttachment, served by the sharded, stateful galactic-nat tier + (see EgressShard). Unlike NetworkRule, it carries no VIP/backend/port: egress is on or off for a (vpcRef, vpcAttachmentRef) pair, existence-implies-enabled, not a per-flow rule — because the destination of an egress flow is an arbitrary internet address, not a pre-configured @@ -46,12 +46,12 @@ spec: accepted — see the Accepted condition. Presence of an accepted NetworkEgressPolicy resolves only *enablement* - (should this tenant's VRF get a default route toward the shared NAT66 - tier at all) — unlike this type's original design (superseded), there is - no single "assigned gateway node" to compute or pin: any NAT66Shard may + (should this tenant's VRF get an egress route toward the shared + translation tier at all) — unlike this type's original design (superseded), there is + no single "assigned gateway node" to compute or pin: any EgressShard may serve any tenant's flow, chosen by the shard-placement consistent-hash ring (internal/maglev, keyed on (tenant VRFID, backend, destination) — - see NAT66Shard's doc comment), not by a per-tenant node assignment stored + see EgressShard's doc comment), not by a per-tenant node assignment stored here. *Isolation* (preventing two tenants with colliding ULA source addresses from colliding in the egress connection table) is a separate, datapath-level concern resolved by tagging each flow with the VRFID diff --git a/docs/api/bgp.md b/docs/api/bgp.md index 6be5722..09871e7 100644 --- a/docs/api/bgp.md +++ b/docs/api/bgp.md @@ -15,7 +15,7 @@ Package v1alpha1 contains API Schema definitions for the network.datumapis.com/v - [BGPPolicy](#bgppolicy) - [BGPRouter](#bgprouter) - [BGPVRFInstance](#bgpvrfinstance) -- [NAT66Shard](#nat66shard) +- [EgressShard](#egressshard) - [ServiceVIPBinding](#servicevipbinding) @@ -916,128 +916,143 @@ _Appears in:_ | `iPv6PrefixAdvertisement` | EVPNRouteTypeIPv6PrefixAdvertisement is Type-5: IPv6 Prefix Advertisement route.
| -#### ExtendedCommunitySet +#### EgressShard -ExtendedCommunitySet defines extended community add and remove operations. +EgressShard marks a single node as a member of the sharded, stateful egress +translation tier (galactic-nat) — a component deliberately kept off the +ingress load-balancer's own consistent-hash ring (see NetworkGateway): +tenant egress traffic (backend -> arbitrary internet destination) is a +different traffic pattern from ingress (fixed VIP, fixed backend pool) +and needs its own placement ring, own per-flow state, and its own +self-routing return path, entirely independent of any NetworkGateway node. + +A shard serves one or both address families. NAT66 (IPv6 -> IPv6) and NAT64 +(IPv6 -> IPv4, RFC 6146) are the same function — stateful egress PAT with a +VRF-scoped session table — over different families, so one shard object +describes both rather than there being a second, near-duplicate kind. +Status.ShardAddressIPv6 and Status.ShardAddressIPv4 are each set only for +the family this shard actually translates; a shard serving only NAT66 +leaves the IPv4 field empty and behaves exactly as it did before NAT64 +existed. + +Every shard owns a dedicated, publicly-routable address per family it +serves, and a flow's allocated masquerade port lives within it — so a reply +is delivered to the correct shard by ordinary unicast routing alone, with +no hashing or cross-shard lookup on the return path at all (the "any node +can determine the owning shard from the tuple alone" property, satisfied by +construction rather than by a replicated hash table). + -_Appears in:_ -- [BGPPolicySetActions](#bgppolicysetactions) | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `add` _string array_ | Add is a list of extended communities to attach. | | MaxItems: 32
items:MaxLength: 64
| -| `remove` _string array_ | Remove is a list of extended communities to strip. | | MaxItems: 32
items:MaxLength: 64
| +| `apiVersion` _string_ | `network.datumapis.com/v1alpha1` | | | +| `kind` _string_ | `EgressShard` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[EgressShardSpec](#egressshardspec)_ | | | | +| `status` _[EgressShardStatus](#egressshardstatus)_ | | | | -#### LocalSecretRef +#### EgressShardSpec -LocalSecretRef references a Secret within the same namespace. -Cross-namespace references are not supported. +EgressShardSpec defines the desired state of an EgressShard. _Appears in:_ -- [BGPPeerSpec](#bgppeerspec) +- [EgressShard](#egressshard) | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | Name is the name of the Secret. | | MinLength: 1
| +| `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| -#### MaxPrefixShutdownAction +#### EgressShardStatus -_Underlying type:_ _string_ -MaxPrefixShutdownAction defines the action taken when a peer exceeds its -maximum-prefix limit. -_Validation:_ -- Enum: [warning-only restart shutdown] +EgressShardStatus defines the observed state of an EgressShard. -_Appears in:_ -- [BGPMaximumPrefix](#bgpmaximumprefix) +Every field here is echoed from what the shard's datapath process was +actually started with, not derived: the shard publishes what it is running, +so a status that disagrees with an operator's intent is a visible +misconfiguration rather than a silently reconciled one. -| Field | Description | -| --- | --- | -| `warning-only` | MaxPrefixShutdownActionWarningOnly logs a warning but keeps the session up.
| -| `restart` | MaxPrefixShutdownActionRestart resets the BGP session when the limit is exceeded.
| -| `shutdown` | MaxPrefixShutdownActionShutdown tears down the BGP session when the limit is exceeded.
| -#### NAT66Shard +_Appears in:_ +- [EgressShard](#egressshard) +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | +| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID. | | | +| `shardAddressIPv6` _string_ | ShardAddressIPv6 is this shard's own dedicated, publicly-routable IPv6
address — every NAT66 masquerade port this shard allocates lives within
it, so any node can route a reply to the correct shard using ordinary
unicast routing on this address alone, with no per-flow state lookup
anywhere but the owning shard itself. Operator-supplied per shard today
(no in-cluster derivation mechanism yet — the same gap
BGPRouter.Spec.SRv6Locator/NodeID assignment has today).
Empty means this shard does not perform IPv6-to-IPv6 translation. | | | +| `shardAddressIPv4` _string_ | ShardAddressIPv4 is this shard's own dedicated, publicly-routable IPv4
address — every NAT64 masquerade port this shard allocates lives within
it, and it is the source an IPv4-only destination sees. Unlike
ShardAddressIPv6, reachability for this address is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the address must be attracted to this node by the underlay
or upstream announcement instead. Publishing it here is what makes that
operator prerequisite checkable.
Empty means this shard does not perform NAT64. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It is echoed here, rather than only
existing as process configuration, because it is the single fact DNS64
synthesis has to agree with: a shard translating for a different prefix
than the resolver synthesizes into is otherwise a silent blackhole.
Empty whenever ShardAddressIPv4 is empty. | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource. | | | -NAT66Shard marks a single node as a member of the sharded, stateful NAT66 -egress tier (galactic-nat66) — a component deliberately kept off the -ingress load-balancer's own consistent-hash ring (see NetworkGateway): -tenant egress traffic (backend -> arbitrary internet destination) is a -different traffic pattern from ingress (fixed VIP, fixed backend pool) -and needs its own placement ring, own per-flow state, and its own -self-routing return path, entirely independent of any NetworkGateway node. +#### ExtendedCommunitySet -Every shard owns a dedicated, BGP-advertised public IPv6 address -(Status.ShardAddress) that a flow's allocated masquerade port lives -within — so a reply is delivered to the correct shard by ordinary -unicast SRv6/BGP routing alone, with no hashing or cross-shard lookup on -the return path at all (the "any node can determine the owning shard from -the tuple alone" property, satisfied by construction rather than by a -replicated hash table). +ExtendedCommunitySet defines extended community add and remove operations. +_Appears in:_ +- [BGPPolicySetActions](#bgppolicysetactions) + | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `apiVersion` _string_ | `network.datumapis.com/v1alpha1` | | | -| `kind` _string_ | `NAT66Shard` | | | -| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | -| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | -| `spec` _[NAT66ShardSpec](#nat66shardspec)_ | | | | -| `status` _[NAT66ShardStatus](#nat66shardstatus)_ | | | | +| `add` _string array_ | Add is a list of extended communities to attach. | | MaxItems: 32
items:MaxLength: 64
| +| `remove` _string array_ | Remove is a list of extended communities to strip. | | MaxItems: 32
items:MaxLength: 64
| -#### NAT66ShardSpec +#### LocalSecretRef -NAT66ShardSpec defines the desired state of a NAT66Shard. +LocalSecretRef references a Secret within the same namespace. +Cross-namespace references are not supported. _Appears in:_ -- [NAT66Shard](#nat66shard) +- [BGPPeerSpec](#bgppeerspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| - - -#### NAT66ShardStatus +| `name` _string_ | Name is the name of the Secret. | | MinLength: 1
| +#### MaxPrefixShutdownAction -NAT66ShardStatus defines the observed state of a NAT66Shard. +_Underlying type:_ _string_ +MaxPrefixShutdownAction defines the action taken when a peer exceeds its +maximum-prefix limit. +_Validation:_ +- Enum: [warning-only restart shutdown] _Appears in:_ -- [NAT66Shard](#nat66shard) +- [BGPMaximumPrefix](#bgpmaximumprefix) -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | -| `shardAddress` _string_ | ShardAddress is this shard's own dedicated, publicly-routable IPv6
address — every masquerade port this shard allocates lives within it,
so any node can route a reply to the correct shard using ordinary
unicast routing on this address alone, with no per-flow state lookup
anywhere but the owning shard itself. Operator-supplied per shard
today (no in-cluster derivation mechanism yet — the same gap
BGPRouter.Spec.SRv6Locator/NodeID assignment has today). | | | -| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
ShardAddress, a plain routable address), advertised into BGP the same
way any other node-reachability route is (a /128 BGPAdvertisement, no
VRFID/Function) so every other node learns a kernel SEG6 route toward
it before installing a tenant VRF's default egress route against it. | | | -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource. | | | +| Field | Description | +| --- | --- | +| `warning-only` | MaxPrefixShutdownActionWarningOnly logs a warning but keeps the session up.
| +| `restart` | MaxPrefixShutdownActionRestart resets the BGP session when the limit is exceeded.
| +| `shutdown` | MaxPrefixShutdownActionShutdown tears down the BGP session when the limit is exceeded.
| #### NPTv6Spec @@ -1313,19 +1328,27 @@ information (galactic-gateway's usidresolver.go), one object per (node, VIP, backend) triple; consumed by a per-node reconciler running inside galactic-router's tenant role. -EgressKind decides which of two entirely different backend mechanisms -this object drives, mirroring the same veth/tap fork the SRv6 uSID decap -datapath already has (internal/plumbing/ebpf/usidmap's egress_kind field): - - - veth (container backend): the node binds VIPAddress on its own - galactic-vip0 dummy interface and the backend answers on it from - inside its own pod netns — internal/plumbing/vip's Bind/Unbind/Verify. +EgressKind decides which of two backend mechanisms this object drives, +mirroring the same veth/tap fork the SRv6 uSID decap datapath already +has (internal/plumbing/ebpf/usidmap's egress_kind field). Both mechanisms +now converge on the same VIP-boundary substitution +(BackendAddress:BackendPort for VIPAddress:Port at the SRv6 uSID TC-BPF +boundary, usid_ingress's inbound half / usid_egress's outbound half) — +required for both, not just tap, since a decapsulated ingress packet is +delivered into the owning tenant's own VRF routing table, which has no +route to an address bound outside that VRF (found live: see galactic's +ServiceVIPBindingReconciler doc comment): + + - veth (container backend): the node ALSO binds VIPAddress on its own + galactic-vip0 dummy interface (internal/plumbing/vip's + Bind/Unbind/Verify in galactic) — this alone does not deliver + anything to the backend pod (galactic-vip0 lives in the node's root + namespace, not the tenant's VRF), but still lets the node itself + verifiably answer on the VIP. - tap (VM backend): there is no guest-side configuration capability in - this repo by design (internal/cnitap's own doc comment) — instead the - node transparently substitutes VIPAddress:Port for - BackendAddress:BackendPort at the SRv6 uSID TC-BPF boundary - (usid_ingress's inbound half, a new usid_egress program's outbound - half), so the guest OS never needs to know the VIP exists at all. + this repo by design (internal/cnitap's own doc comment), so the + substitution above is this kind's *only* delivery mechanism — the + guest OS never needs to know the VIP exists at all. @@ -1377,8 +1400,8 @@ _Appears in:_ | `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this binding applies to. | | Required: \{\}
| | `vipAddress` _string_ | VIPAddress is the service VIP the backend must be reachable on. | | Required: \{\}
| | `port` _integer_ | Port is the VIP-facing port traffic arrives on. | | Maximum: 65535
Minimum: 1
Required: \{\}
| -| `backendAddress` _string_ | BackendAddress is the backend's own real address (its pod-netns
address for a veth backend, or its actual guest-facing address for a
tap backend). Required only for EgressKindTap, where it is the
substitution target; a veth binding's backend answers on VIPAddress
itself once bound, so this field is ignored for that kind. | | | -| `backendPort` _integer_ | BackendPort is the backend's own real port, paired with
BackendAddress for the tap-translation case. Ignored for veth. | | Maximum: 65535
Minimum: 1
| +| `backendAddress` _string_ | BackendAddress is the backend's own real address (its pod-netns
address for a veth backend, or its actual guest-facing address for a
tap backend) — the VIP-boundary substitution target for both kinds
now (see EgressKind's own doc comment for why veth needs this too,
not just tap). +optional at the API level for the same reason
EgressKind itself carries no matching CEL requirement; the
reconciler validates it's set for either kind before doing anything. | | | +| `backendPort` _integer_ | BackendPort is the backend's own real port, paired with
BackendAddress — required for both kinds, see that field's doc
comment. | | Maximum: 65535
Minimum: 1
| | `egressKind` _[ServiceVIPBindingEgressKind](#servicevipbindingegresskind)_ | EgressKind selects which backend mechanism this binding drives. | | Enum: [veth tap]
Required: \{\}
| @@ -1410,7 +1433,7 @@ Supported values for kind: Node. _Appears in:_ - [BGPRouterSpec](#bgprouterspec) -- [NAT66ShardSpec](#nat66shardspec) +- [EgressShardSpec](#egressshardspec) - [ServiceVIPBindingSpec](#servicevipbindingspec) | Field | Description | Default | Validation | diff --git a/docs/api/gateway.md b/docs/api/gateway.md index 836e3d6..e00b33d 100644 --- a/docs/api/gateway.md +++ b/docs/api/gateway.md @@ -9,7 +9,7 @@ Package v1alpha1 contains API Schema definitions for the network.datumapis.com/v1alpha1 API group. ### Resource Types -- [NAT66Shard](#nat66shard) +- [EgressShard](#egressshard) - [NetworkEgressPolicy](#networkegresspolicy) - [NetworkGateway](#networkgateway) - [NetworkRule](#networkrule) @@ -17,25 +17,33 @@ Package v1alpha1 contains API Schema definitions for the network.datumapis.com/v -#### NAT66Shard +#### EgressShard -NAT66Shard marks a single node as a member of the sharded, stateful NAT66 -egress tier (galactic-nat66) — a component deliberately kept off the +EgressShard marks a single node as a member of the sharded, stateful egress +translation tier (galactic-nat) — a component deliberately kept off the ingress load-balancer's own consistent-hash ring (see NetworkGateway): tenant egress traffic (backend -> arbitrary internet destination) is a different traffic pattern from ingress (fixed VIP, fixed backend pool) and needs its own placement ring, own per-flow state, and its own self-routing return path, entirely independent of any NetworkGateway node. -Every shard owns a dedicated, BGP-advertised public IPv6 address -(Status.ShardAddress) that a flow's allocated masquerade port lives -within — so a reply is delivered to the correct shard by ordinary -unicast SRv6/BGP routing alone, with no hashing or cross-shard lookup on -the return path at all (the "any node can determine the owning shard from -the tuple alone" property, satisfied by construction rather than by a -replicated hash table). +A shard serves one or both address families. NAT66 (IPv6 -> IPv6) and NAT64 +(IPv6 -> IPv4, RFC 6146) are the same function — stateful egress PAT with a +VRF-scoped session table — over different families, so one shard object +describes both rather than there being a second, near-duplicate kind. +Status.ShardAddressIPv6 and Status.ShardAddressIPv4 are each set only for +the family this shard actually translates; a shard serving only NAT66 +leaves the IPv4 field empty and behaves exactly as it did before NAT64 +existed. + +Every shard owns a dedicated, publicly-routable address per family it +serves, and a flow's allocated masquerade port lives within it — so a reply +is delivered to the correct shard by ordinary unicast routing alone, with +no hashing or cross-shard lookup on the return path at all (the "any node +can determine the owning shard from the tuple alone" property, satisfied by +construction rather than by a replicated hash table). @@ -44,46 +52,53 @@ replicated hash table). | Field | Description | Default | Validation | | --- | --- | --- | --- | | `apiVersion` _string_ | `network.datumapis.com/v1alpha1` | | | -| `kind` _string_ | `NAT66Shard` | | | +| `kind` _string_ | `EgressShard` | | | | `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | | `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | | `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | -| `spec` _[NAT66ShardSpec](#nat66shardspec)_ | | | | -| `status` _[NAT66ShardStatus](#nat66shardstatus)_ | | | | +| `spec` _[EgressShardSpec](#egressshardspec)_ | | | | +| `status` _[EgressShardStatus](#egressshardstatus)_ | | | | -#### NAT66ShardSpec +#### EgressShardSpec -NAT66ShardSpec defines the desired state of a NAT66Shard. +EgressShardSpec defines the desired state of an EgressShard. _Appears in:_ -- [NAT66Shard](#nat66shard) +- [EgressShard](#egressshard) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `targetRef` _[TargetRef](#targetref)_ | TargetRef identifies the Node this shard executes on. | | Required: \{\}
| -#### NAT66ShardStatus +#### EgressShardStatus + +EgressShardStatus defines the observed state of an EgressShard. -NAT66ShardStatus defines the observed state of a NAT66Shard. +Every field here is echoed from what the shard's datapath process was +actually started with, not derived: the shard publishes what it is running, +so a status that disagrees with an operator's intent is a visible +misconfiguration rather than a silently reconciled one. _Appears in:_ -- [NAT66Shard](#nat66shard) +- [EgressShard](#egressshard) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `observedGeneration` _integer_ | ObservedGeneration is the .metadata.generation this status was computed from. | | | -| `shardAddress` _string_ | ShardAddress is this shard's own dedicated, publicly-routable IPv6
address — every masquerade port this shard allocates lives within it,
so any node can route a reply to the correct shard using ordinary
unicast routing on this address alone, with no per-flow state lookup
anywhere but the owning shard itself. Operator-supplied per shard
today (no in-cluster derivation mechanism yet — the same gap
BGPRouter.Spec.SRv6Locator/NodeID assignment has today). | | | -| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
ShardAddress, a plain routable address), advertised into BGP the same
way any other node-reachability route is (a /128 BGPAdvertisement, no
VRFID/Function) so every other node learns a kernel SEG6 route toward
it before installing a tenant VRF's default egress route against it. | | | +| `shardSID` _string_ | ShardSID is this shard's own uSID locator — a real SRv6 uSID (unlike
the ShardAddress fields, which are plain routable addresses),
advertised into BGP the same way any other node-reachability route is
(a /128 BGPAdvertisement, no VRFID/Function) so every other node learns
a kernel SEG6 route toward it before installing a tenant VRF's egress
route against it. One SID serves both families: which translation a
packet gets is decided from the inner destination, not from a second
SID. | | | +| `shardAddressIPv6` _string_ | ShardAddressIPv6 is this shard's own dedicated, publicly-routable IPv6
address — every NAT66 masquerade port this shard allocates lives within
it, so any node can route a reply to the correct shard using ordinary
unicast routing on this address alone, with no per-flow state lookup
anywhere but the owning shard itself. Operator-supplied per shard today
(no in-cluster derivation mechanism yet — the same gap
BGPRouter.Spec.SRv6Locator/NodeID assignment has today).
Empty means this shard does not perform IPv6-to-IPv6 translation. | | | +| `shardAddressIPv4` _string_ | ShardAddressIPv4 is this shard's own dedicated, publicly-routable IPv4
address — every NAT64 masquerade port this shard allocates lives within
it, and it is the source an IPv4-only destination sees. Unlike
ShardAddressIPv6, reachability for this address is not established by a
BGPAdvertisement into the EVPN fabric: an IPv4 reply arrives from the
internet, so the address must be attracted to this node by the underlay
or upstream announcement instead. Publishing it here is what makes that
operator prerequisite checkable.
Empty means this shard does not perform NAT64. | | | +| `nat64Prefix` _string_ | NAT64Prefix is the IPv6 prefix whose synthesized addresses this shard
translates to IPv4 — one Datum-operated Network-Specific Prefix, shared
fabric-wide, never per-tenant. It is echoed here, rather than only
existing as process configuration, because it is the single fact DNS64
synthesis has to agree with: a shard translating for a different prefix
than the resolver synthesizes into is otherwise a silent blackhole.
Empty whenever ShardAddressIPv4 is empty. | | | | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | Conditions contains the standard conditions for this resource. | | | @@ -94,8 +109,8 @@ _Appears in:_ NetworkEgressPolicy enables internet egress for a single tenant -VPC/VPCAttachment, served by the sharded, stateful galactic-nat66 tier -(see NAT66Shard). Unlike NetworkRule, it carries no VIP/backend/port: +VPC/VPCAttachment, served by the sharded, stateful galactic-nat tier +(see EgressShard). Unlike NetworkRule, it carries no VIP/backend/port: egress is on or off for a (vpcRef, vpcAttachmentRef) pair, existence-implies-enabled, not a per-flow rule — because the destination of an egress flow is an arbitrary internet address, not a pre-configured @@ -109,12 +124,12 @@ the requester is authorized for vpcRef/vpcAttachmentRef before a policy is accepted — see the Accepted condition. Presence of an accepted NetworkEgressPolicy resolves only *enablement* -(should this tenant's VRF get a default route toward the shared NAT66 -tier at all) — unlike this type's original design (superseded), there is -no single "assigned gateway node" to compute or pin: any NAT66Shard may +(should this tenant's VRF get an egress route toward the shared +translation tier at all) — unlike this type's original design (superseded), there is +no single "assigned gateway node" to compute or pin: any EgressShard may serve any tenant's flow, chosen by the shard-placement consistent-hash ring (internal/maglev, keyed on (tenant VRFID, backend, destination) — -see NAT66Shard's doc comment), not by a per-tenant node assignment stored +see EgressShard's doc comment), not by a per-tenant node assignment stored here. *Isolation* (preventing two tenants with colliding ULA source addresses from colliding in the egress connection table) is a separate, datapath-level concern resolved by tagging each flow with the VRFID @@ -370,19 +385,27 @@ information (galactic-gateway's usidresolver.go), one object per (node, VIP, backend) triple; consumed by a per-node reconciler running inside galactic-router's tenant role. -EgressKind decides which of two entirely different backend mechanisms -this object drives, mirroring the same veth/tap fork the SRv6 uSID decap -datapath already has (internal/plumbing/ebpf/usidmap's egress_kind field): - - - veth (container backend): the node binds VIPAddress on its own - galactic-vip0 dummy interface and the backend answers on it from - inside its own pod netns — internal/plumbing/vip's Bind/Unbind/Verify. +EgressKind decides which of two backend mechanisms this object drives, +mirroring the same veth/tap fork the SRv6 uSID decap datapath already +has (internal/plumbing/ebpf/usidmap's egress_kind field). Both mechanisms +now converge on the same VIP-boundary substitution +(BackendAddress:BackendPort for VIPAddress:Port at the SRv6 uSID TC-BPF +boundary, usid_ingress's inbound half / usid_egress's outbound half) — +required for both, not just tap, since a decapsulated ingress packet is +delivered into the owning tenant's own VRF routing table, which has no +route to an address bound outside that VRF (found live: see galactic's +ServiceVIPBindingReconciler doc comment): + + - veth (container backend): the node ALSO binds VIPAddress on its own + galactic-vip0 dummy interface (internal/plumbing/vip's + Bind/Unbind/Verify in galactic) — this alone does not deliver + anything to the backend pod (galactic-vip0 lives in the node's root + namespace, not the tenant's VRF), but still lets the node itself + verifiably answer on the VIP. - tap (VM backend): there is no guest-side configuration capability in - this repo by design (internal/cnitap's own doc comment) — instead the - node transparently substitutes VIPAddress:Port for - BackendAddress:BackendPort at the SRv6 uSID TC-BPF boundary - (usid_ingress's inbound half, a new usid_egress program's outbound - half), so the guest OS never needs to know the VIP exists at all. + this repo by design (internal/cnitap's own doc comment), so the + substitution above is this kind's *only* delivery mechanism — the + guest OS never needs to know the VIP exists at all. @@ -435,8 +458,8 @@ _Appears in:_ | `vipAddress` _string_ | VIPAddress is the service VIP the backend must be reachable on. | | Required: \{\}
| | `port` _integer_ | Port is the VIP-facing port traffic arrives on. | | Maximum: 65535
Minimum: 1
Required: \{\}
| | `protocol` _[NetworkRuleProtocol](#networkruleprotocol)_ | Protocol is the transport protocol this binding applies to. | | Enum: [tcp udp]
Required: \{\}
| -| `backendAddress` _string_ | BackendAddress is the backend's own real address (its pod-netns
address for a veth backend, or its actual guest-facing address for a
tap backend). Required only for EgressKindTap, where it is the
substitution target; a veth binding's backend answers on VIPAddress
itself once bound, so this field is ignored for that kind. | | | -| `backendPort` _integer_ | BackendPort is the backend's own real port, paired with
BackendAddress for the tap-translation case. Ignored for veth. | | Maximum: 65535
Minimum: 1
| +| `backendAddress` _string_ | BackendAddress is the backend's own real address (its pod-netns
address for a veth backend, or its actual guest-facing address for a
tap backend) — the VIP-boundary substitution target for both kinds
now (see EgressKind's own doc comment for why veth needs this too,
not just tap). +optional at the API level for the same reason
EgressKind itself carries no matching CEL requirement; the
reconciler validates it's set for either kind before doing anything. | | | +| `backendPort` _integer_ | BackendPort is the backend's own real port, paired with
BackendAddress — required for both kinds, see that field's doc
comment. | | Maximum: 65535
Minimum: 1
| | `egressKind` _[ServiceVIPBindingEgressKind](#servicevipbindingegresskind)_ | EgressKind selects which backend mechanism this binding drives. | | Enum: [veth tap]
Required: \{\}
| @@ -467,7 +490,7 @@ Supported values for kind: Node. _Appears in:_ -- [NAT66ShardSpec](#nat66shardspec) +- [EgressShardSpec](#egressshardspec) - [NetworkGatewaySpec](#networkgatewayspec) - [ServiceVIPBindingSpec](#servicevipbindingspec)