From e75eb05d2ace0debf5ab273e409f9fcbf77e5183 Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Tue, 3 Nov 2026 18:09:45 -0800 Subject: [PATCH 1/3] fix(nicinfo): skip lo and reject routes; expand bond/vlan uplinks to NICs DefaultUplinks took every ::/0 in /proc/net/ipv6_route, including the kernel's "unreachable ::/0 dev lo" (RTF_REJECT, not RTF_UP) that every host has, so prod records carried UPLINK2=lo. It also returned bond0 itself, a virtual device with no "device" entry, so the NIC columns were empty and the LLDP neighbours (keyed by the physical ports) never matched. Default routes now count only when up, not reject, and not on lo or "*" (v4 unreachable/blackhole). The v4 mask must be 0 too. A virtual device holding the route is replaced by the physical NICs under its lower_ sysfs links (bond, team, vlan, bridge; cycles and depth bounded); one with no NIC beneath it is kept as before. Co-Authored-By: Claude Opus 5.5 --- pkg/nicinfo/uplinks.go | 142 ++++++++++++++++++++-------- pkg/nicinfo/uplinks_test.go | 179 ++++++++++++++++++++++++++++++++++++ 2 files changed, 282 insertions(+), 39 deletions(-) create mode 100644 pkg/nicinfo/uplinks_test.go diff --git a/pkg/nicinfo/uplinks.go b/pkg/nicinfo/uplinks.go index 68473c760..7eac25456 100644 --- a/pkg/nicinfo/uplinks.go +++ b/pkg/nicinfo/uplinks.go @@ -6,15 +6,35 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" ) +// Route flags (include/uapi/linux/route.h, ipv6_route.h). The same bits in +// /proc/net/route and /proc/net/ipv6_route. +const ( + rtfUp = 0x0001 + rtfReject = 0x0200 +) + +// maxLowerDepth bounds the upper->lower walk (vlan on bond on NIC is 2). +const maxLowerDepth = 8 + // DefaultUplinks returns up to maxN physical interface names that carry the // default IPv4/IPv6 routes (best-effort; reads /proc/net/route + // /proc/net/ipv6_route). Used when the operator doesn't pin uplink interfaces // explicitly. On any read error it returns what it has so far (possibly empty); // it never fails. +// +// Only usable routes count: lo and reject routes are skipped (the kernel's +// "unreachable ::/0 dev lo" is in every ipv6_route), as are routes not up. A +// virtual device holding the route (bond0, a vlan, a bridge) is replaced by the +// physical NICs beneath it, which is where the NIC and LLDP columns live. func DefaultUplinks(maxN int) []string { + return defaultUplinks("/proc", "/sys", maxN) +} + +func defaultUplinks(procRoot, sysfsRoot string, maxN int) []string { if maxN <= 0 { return nil } @@ -33,44 +53,110 @@ func DefaultUplinks(maxN int) []string { return len(out) < maxN } - for _, name := range defaultRouteIfacesV4() { - if !add(name) { - return out - } + var routes []string + if b, err := os.ReadFile(filepath.Join(procRoot, "net", "route")); err == nil { + routes = append(routes, parseDefaultRoutesV4(b)...) } - for _, name := range defaultRouteIfacesV6() { - if !add(name) { - return out + if b, err := os.ReadFile(filepath.Join(procRoot, "net", "ipv6_route")); err == nil { + routes = append(routes, parseDefaultRoutesV6(b)...) + } + for _, route := range routes { + for _, name := range physicalLowers(sysfsRoot, route) { + if !add(name) { + return out + } } } return out } -// defaultRouteIfacesV4 parses /proc/net/route for interfaces holding the -// default route (destination 0.0.0.0). The file is whitespace-separated with a -// header line; fields are: Iface Destination Gateway Flags ... -func defaultRouteIfacesV4() []string { - b, err := os.ReadFile("/proc/net/route") - if err != nil { - return nil +// usableRoute reports whether a default route's interface and hex flags name +// an uplink: up, not a reject route, and not lo ("*" is v4's no-device). +func usableRoute(iface, flags string) bool { + if iface == "" || iface == "lo" || iface == "*" { + return false } + f, err := strconv.ParseUint(flags, 16, 32) + return err == nil && f&rtfUp != 0 && f&rtfReject == 0 +} + +// parseDefaultRoutesV4 returns the interfaces holding a usable default route in +// /proc/net/route: a header line, then whitespace-separated fields +// Iface Destination Gateway Flags RefCnt Use Metric Mask ... +func parseDefaultRoutesV4(b []byte) []string { var out []string for i, line := range strings.Split(string(b), "\n") { if i == 0 { // header continue } f := strings.Fields(line) - if len(f) < 2 { + if len(f) < 8 { continue } - // Destination "00000000" is the default route (0.0.0.0). - if f[1] == "00000000" { + // Destination and Mask 0.0.0.0 is the default route. + if f[1] == "00000000" && f[7] == "00000000" && usableRoute(f[0], f[3]) { out = append(out, f[0]) } } return out } +// parseDefaultRoutesV6 returns the interfaces holding a usable default route in +// /proc/net/ipv6_route. Each line has ten fields: dest_net dest_prefixlen +// src_net src_prefixlen next_hop metric refcnt use flags iface. +func parseDefaultRoutesV6(b []byte) []string { + var out []string + for _, line := range strings.Split(string(b), "\n") { + f := strings.Fields(line) + if len(f) != 10 { + continue + } + // dest_net all zeros and prefix length 0 => default route (::/0). + if f[0] == "00000000000000000000000000000000" && f[1] == "00" && usableRoute(f[9], f[8]) { + out = append(out, f[9]) + } + } + return out +} + +// physicalLowers returns the physical NICs beneath name, following the +// /class/net//lower_ links a bond, team, vlan or bridge has to +// its lower devices, sorted per level. A NIC (it has a "device" entry) is its +// own answer. A virtual device with no physical NIC beneath it (a bridge of +// veths, a tunnel) or with an unreadable sysfs is returned as itself, as +// before this walk existed. +func physicalLowers(sysfsRoot, name string) []string { + netDir := filepath.Join(sysfsRoot, "class", "net") + var out []string + visited := make(map[string]bool) + var walk func(name string, depth int) + walk = func(name string, depth int) { + if visited[name] || depth > maxLowerDepth { + return + } + visited[name] = true + dir := filepath.Join(netDir, name) + if _, err := os.Lstat(filepath.Join(dir, "device")); err == nil { + out = append(out, name) + return + } + entries, err := os.ReadDir(dir) + if err != nil { + return + } + for _, e := range entries { // ReadDir sorts by name + if lower, ok := strings.CutPrefix(e.Name(), "lower_"); ok && lower != "" { + walk(lower, depth+1) + } + } + } + walk(name, 0) + if len(out) == 0 { + return []string{name} + } + return out +} + // PhysicalInterfaces returns up to maxN physical interface names discovered by // scanning /class/net for entries that have a "device" symlink (a // real NIC — virtio_net, mlx5, ice, … — as opposed to lo/veth*/docker0/bridges, @@ -109,25 +195,3 @@ func PhysicalInterfaces(sysfsRoot string, maxN int) []string { } return names } - -// defaultRouteIfacesV6 parses /proc/net/ipv6_route for interfaces holding the -// default route (::/0). Each line has: dest_net(32 hex) dest_prefixlen ... -// with the interface name as the last field. -func defaultRouteIfacesV6() []string { - b, err := os.ReadFile("/proc/net/ipv6_route") - if err != nil { - return nil - } - var out []string - for _, line := range strings.Split(string(b), "\n") { - f := strings.Fields(line) - if len(f) < 10 { - continue - } - // dest_net all zeros and prefix length 0 => default route (::/0). - if f[0] == "00000000000000000000000000000000" && f[1] == "00" { - out = append(out, f[len(f)-1]) - } - } - return out -} diff --git a/pkg/nicinfo/uplinks_test.go b/pkg/nicinfo/uplinks_test.go new file mode 100644 index 000000000..5587bf57f --- /dev/null +++ b/pkg/nicinfo/uplinks_test.go @@ -0,0 +1,179 @@ +//go:build linux + +package nicinfo + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +const routeV4Header = "Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\n" + +// routeV4 is one /proc/net/route line. +func routeV4(iface, dest, flags, mask string) string { + return iface + "\t" + dest + "\t0100A8C0\t" + flags + "\t0\t0\t0\t" + mask + "\t0\t0\t0\n" +} + +// routeV6 is one /proc/net/ipv6_route line. +func routeV6(dest, prefix, flags, iface string) string { + zero := strings.Repeat("0", 32) + return dest + " " + prefix + " " + zero + " 00 " + zero + " 00000400 00000001 00000000 " + flags + " " + iface + "\n" +} + +var ( + zeroV6 = strings.Repeat("0", 32) + // The kernel's "unreachable ::/0 dev lo" (RTF_NONEXTHOP|RTF_REJECT, not + // RTF_UP): in every ipv6_route, and what made UPLINK2 lo on prod. + unreachableV6 = zeroV6 + " 00 " + zeroV6 + " 00 " + zeroV6 + " ffffffff 00000001 00000000 00200200 lo\n" +) + +func TestParseDefaultRoutesV4(t *testing.T) { + for _, tc := range []struct { + name, category, description string + input string + expected []string + }{ + {"gateway", "positive", "default via a gateway on eth0 (UG)", routeV4Header + routeV4("eth0", "00000000", "0003", "00000000"), []string{"eth0"}}, + {"bond", "positive", "default on bond0, the prod shape", routeV4Header + routeV4("bond0", "00000000", "0003", "00000000"), []string{"bond0"}}, + {"unreachable", "negative", "unreachable default: no device, RTF_REJECT", routeV4Header + routeV4("*", "00000000", "0201", "00000000"), nil}, + {"blackhole", "negative", "blackhole default: no device (*), up, not reject", routeV4Header + routeV4("*", "00000000", "0001", "00000000"), nil}, + {"reject-on-dev", "negative", "reject flag on a real device", routeV4Header + routeV4("eth0", "00000000", "0203", "00000000"), nil}, + {"not-up", "negative", "default route without RTF_UP", routeV4Header + routeV4("eth0", "00000000", "0002", "00000000"), nil}, + {"lo", "negative", "an up default route on lo", routeV4Header + routeV4("lo", "00000000", "0001", "00000000"), nil}, + {"subnet", "negative", "a subnet route, not the default", routeV4Header + routeV4("eth0", "0000A8C0", "0001", "00FFFFFF"), nil}, + {"mask", "boundary", "destination 0 with a /8 mask is 0.0.0.0/8, not default", routeV4Header + routeV4("eth0", "00000000", "0001", "000000FF"), nil}, + {"header-only", "boundary", "a table with only the header", routeV4Header, nil}, + {"two", "corner", "two defaults by metric, kept in table order", routeV4Header + routeV4("eth1", "00000000", "0003", "00000000") + routeV4("eth0", "00000000", "0003", "00000000"), []string{"eth1", "eth0"}}, + {"garbage", "corner", "short lines and non-hex flags", routeV4Header + "eth0 00000000\n" + routeV4("eth0", "00000000", "zz", "00000000"), nil}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Logf("%s: %s", tc.category, tc.description) + if got := parseDefaultRoutesV4([]byte(tc.input)); !slices.Equal(got, tc.expected) { + t.Fatalf("got %v, expected %v", got, tc.expected) + } + }) + } +} + +func TestParseDefaultRoutesV6(t *testing.T) { + for _, tc := range []struct { + name, category, description string + input string + expected []string + }{ + {"static", "positive", "static default via eth0 (UP|GATEWAY)", routeV6(zeroV6, "00", "00000003", "eth0"), []string{"eth0"}}, + {"ra", "positive", "RA-learned default (UP|GATEWAY|DEFAULT|ADDRCONF|EXPIRES)", routeV6(zeroV6, "00", "00450003", "bond0"), []string{"bond0"}}, + {"unreachable-lo", "negative", "the kernel's unreachable ::/0 dev lo, the UPLINK2=lo bug", unreachableV6, nil}, + {"up-reject", "negative", "an up reject route on a real device", routeV6(zeroV6, "00", "00000201", "eth0"), nil}, + {"lo-up", "negative", "an up default on lo", routeV6(zeroV6, "00", "00000001", "lo"), nil}, + {"prefix", "boundary", "dest :: with a /64 prefix, not default", routeV6(zeroV6, "40", "00000001", "eth0"), nil}, + {"empty", "boundary", "an empty table", "", nil}, + {"with-unreachable", "corner", "a real default beside the unreachable one", unreachableV6 + routeV6(zeroV6, "00", "00000003", "eth0"), []string{"eth0"}}, + {"garbage", "corner", "nine fields, and non-hex flags", zeroV6 + " 00 x x x x x x x\n" + routeV6(zeroV6, "00", "zz", "eth0"), nil}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Logf("%s: %s", tc.category, tc.description) + if got := parseDefaultRoutesV6([]byte(tc.input)); !slices.Equal(got, tc.expected) { + t.Fatalf("got %v, expected %v", got, tc.expected) + } + }) + } +} + +// sysNet builds /class/net: nics get a device entry, uppers get one +// lower_ link per lower (a symlink, as on sysfs), virtual ifaces get +// neither. +func sysNet(t *testing.T, nics []string, uppers map[string][]string, virtual []string) string { + t.Helper() + root := t.TempDir() + net := filepath.Join(root, "class", "net") + mkdir := func(p string) { + if err := os.MkdirAll(p, 0o750); err != nil { + t.Fatal(err) + } + } + for _, n := range nics { + mkdir(filepath.Join(net, n, "device")) + } + for _, v := range virtual { + mkdir(filepath.Join(net, v)) + } + for upper, lowers := range uppers { + mkdir(filepath.Join(net, upper)) + for _, l := range lowers { + if err := os.Symlink(filepath.Join("..", l), filepath.Join(net, upper, "lower_"+l)); err != nil { + t.Fatal(err) + } + } + } + return root +} + +func TestPhysicalLowers(t *testing.T) { + for _, tc := range []struct { + name, category, description string + nics []string + uppers map[string][]string + virtual []string + input string + expected []string + }{ + {"nic", "positive", "a NIC is its own answer", []string{"eth0"}, nil, nil, "eth0", []string{"eth0"}}, + {"bond", "positive", "bond0 over two NICs, the prod shape", []string{"eth0", "eth1"}, map[string][]string{"bond0": {"eth1", "eth0"}}, nil, "bond0", []string{"eth0", "eth1"}}, + {"vlan-on-bond", "positive", "a vlan on a bond on two NICs", []string{"eth0", "eth1"}, map[string][]string{"bond0.100": {"bond0"}, "bond0": {"eth0", "eth1"}}, nil, "bond0.100", []string{"eth0", "eth1"}}, + {"bridge-veths", "negative", "a bridge of veths has no NIC beneath it", nil, map[string][]string{"br0": {"veth1"}}, []string{"veth1"}, "br0", []string{"br0"}}, + {"missing", "negative", "an interface not in sysfs", nil, nil, nil, "gone0", []string{"gone0"}}, + {"empty-bond", "boundary", "a bond with no members", nil, map[string][]string{"bond0": nil}, nil, "bond0", []string{"bond0"}}, + {"bridge-mixed", "corner", "a bridge over a NIC and a veth: only the NIC", []string{"eth2"}, map[string][]string{"br0": {"eth2", "veth1"}}, []string{"veth1"}, "br0", []string{"eth2"}}, + {"cycle", "corner", "a lower_ loop terminates", nil, map[string][]string{"a": {"b"}, "b": {"a"}}, nil, "a", []string{"a"}}, + {"shared-lower", "corner", "two paths to one NIC count it once", []string{"eth0"}, map[string][]string{"br0": {"bond0", "vlan5"}, "bond0": {"eth0"}, "vlan5": {"eth0"}}, nil, "br0", []string{"eth0"}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Logf("%s: %s", tc.category, tc.description) + root := sysNet(t, tc.nics, tc.uppers, tc.virtual) + if got := physicalLowers(root, tc.input); !slices.Equal(got, tc.expected) { + t.Fatalf("got %v, expected %v", got, tc.expected) + } + }) + } +} + +func TestDefaultUplinksFixture(t *testing.T) { + bond := map[string][]string{"bond0": {"eth0", "eth1"}} + for _, tc := range []struct { + name, category, description string + v4, v6 string + max int + expected []string + }{ + {"prod", "positive", "v4 default on bond0, v6 only the unreachable lo route: was [bond0 lo]", routeV4Header + routeV4("bond0", "00000000", "0003", "00000000"), unreachableV6, 2, []string{"eth0", "eth1"}}, + {"dual-stack", "positive", "v4 and v6 defaults on one bond: members once", routeV4Header + routeV4("bond0", "00000000", "0003", "00000000"), routeV6(zeroV6, "00", "00450003", "bond0"), 2, []string{"eth0", "eth1"}}, + {"only-lo", "negative", "no usable default route at all", routeV4Header, unreachableV6, 2, nil}, + {"no-proc", "negative", "route files unreadable", "", "", 2, nil}, + {"cap-one", "boundary", "max 1 with a two-member bond", routeV4Header + routeV4("bond0", "00000000", "0003", "00000000"), "", 1, []string{"eth0"}}, + {"cap-zero", "boundary", "max 0", routeV4Header + routeV4("bond0", "00000000", "0003", "00000000"), "", 0, nil}, + {"v6-member", "corner", "v4 on eth2, v6 on bond0: v4 first, then the members", routeV4Header + routeV4("eth2", "00000000", "0003", "00000000"), routeV6(zeroV6, "00", "00000003", "bond0"), 4, []string{"eth2", "eth0", "eth1"}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Logf("%s: %s", tc.category, tc.description) + sys := sysNet(t, []string{"eth0", "eth1", "eth2"}, bond, nil) + proc := t.TempDir() + if tc.v4 != "" || tc.v6 != "" { + if err := os.MkdirAll(filepath.Join(proc, "net"), 0o750); err != nil { + t.Fatal(err) + } + for file, body := range map[string]string{"route": tc.v4, "ipv6_route": tc.v6} { + if err := os.WriteFile(filepath.Join(proc, "net", file), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + } + if got := defaultUplinks(proc, sys, tc.max); !slices.Equal(got, tc.expected) { + t.Fatalf("got %v, expected %v", got, tc.expected) + } + }) + } +} From e13287e7bc67db037ae427c89728bed44fa6c95d Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Tue, 3 Nov 2026 18:12:56 -0800 Subject: [PATCH 2/3] feat(xtcp): warn when a netns dir is a private mount On prod v2.0.0, ansible-host mounted /run/netns and /run/docker/netns with Docker's default rprivate propagation, so namespaces created after xtcp2 started never appeared in the container: their sockets went uncollected (setns EINVAL on the stale file) and their container name/image was empty, with runtime falling back to cgroupfs. One host saw 10 of its 17 netns. At startup xtcp2 reads /proc/self/mountinfo, finds the mount holding each netns dir, and if it neither shares nor slaves (no shared:N / master:N) logs a warning naming the rslave fix and sets xtcp_gauges{function="checkNetnsPropagation",variable="private",type=} to 1 (0 otherwise). The deploy fix is runpod/ansible-host#1120. Docs: the docker example mounts ro,rslave; metrics table row; design doc. Co-Authored-By: Claude Opus 5.5 --- docs/design-metadata-enrichment.md | 14 +++- docs/metrics.md | 1 + docs/output-and-destinations.md | 4 +- pkg/xtcp/init.go | 3 + pkg/xtcp/netns_propagation.go | 106 +++++++++++++++++++++++++++++ pkg/xtcp/netns_propagation_test.go | 102 +++++++++++++++++++++++++++ 6 files changed, 227 insertions(+), 3 deletions(-) create mode 100644 pkg/xtcp/netns_propagation.go create mode 100644 pkg/xtcp/netns_propagation_test.go diff --git a/docs/design-metadata-enrichment.md b/docs/design-metadata-enrichment.md index 5726a4b24..a1514a8da 100644 --- a/docs/design-metadata-enrichment.md +++ b/docs/design-metadata-enrichment.md @@ -77,7 +77,11 @@ every record on the hot path (they dictionary-compress to ~nothing downstream). Topology is modeled as **two fixed uplink slots** (`uplink1_*`, `uplink2_*`) since hosts are dual-homed. Uplinks default to the interfaces on the default IPv4/IPv6 -routes (`nicinfo.DefaultUplinks`), overridable with `-uplinkInterfaces`. +routes (`nicinfo.DefaultUplinks`), overridable with `-uplinkInterfaces`. Only +usable routes count (up, not reject, not on `lo`: every host has the kernel's +`unreachable ::/0 dev lo`), and a virtual device holding the route (bond, vlan, +bridge) is replaced by the physical NICs under its `lower_*` sysfs links, since +those carry the NIC attributes and the LLDP neighbours. ## 3. Config: flags / env / config protobuf @@ -125,6 +129,14 @@ current deploy **already** mounts `/run/netns:ro` and `/run/docker/netns:ro`, so the discovery fix (§1) restores `netns`/`container_id` with **no deploy change** once the new image ships. +Those mounts must be `ro,rslave`, not Docker's default `rprivate`: a pod's netns +is a bind mount made under them after xtcp2 starts, and a private mount never +shows it. On prod v2.0.0 one host's xtcp2 saw 10 of its 17 netns; the newer +pods' sockets went uncollected (setns EINVAL on the stale file) and their +container name/image was empty. xtcp2 warns at startup and sets +`xtcp_gauges{function="checkNetnsPropagation",variable="private"}` to 1 for +each such dir. + To enable the *new* enrichers, add (recommended as **opt-in** ansible vars, default off, mirroring `xtcp2_resolve_container_id`): diff --git a/docs/metrics.md b/docs/metrics.md index 26ded927c..263d3ba11 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -283,6 +283,7 @@ All are in `xtcp_counts`. | `xtcp_counts` | `Run` | `start` | counter | the daemon started | | `xtcp_gauge` | | | | namespaces tracked, set by `mapCountReporter` | | `xtcp_gauges` | `InitPromethus` | `compiledInEnrichers` | enricher name | 1 if the enricher is built into this binary, 0 if not | +| `xtcp_gauges` | `checkNetnsPropagation` | `private` | netns dir | 1 if the dir is a private mount, so namespaces created after startup are invisible (mount it `rslave`); 0 if it receives them | ## Link monitor diff --git a/docs/output-and-destinations.md b/docs/output-and-destinations.md index 59d3af58b..b17983f67 100644 --- a/docs/output-and-destinations.md +++ b/docs/output-and-destinations.md @@ -112,7 +112,7 @@ sudo ./result/bin/xtcp2 -dest tcp:127.0.0.1:9000 -marshal jsonl -d 1 sudo ./result/bin/xtcp2 -dest http://127.0.0.1:8080/ingest -marshal jsonl -d 1 ``` -**In Docker** (mount the host namespace dirs so the daemon sees real sockets; records on `stdout`/`stderr` appear in `docker logs`): +**In Docker** (mount the host namespace dirs so the daemon sees real sockets; `rslave` so it also sees namespaces created after it starts; records on `stdout`/`stderr` appear in `docker logs`): ```sh # one-time: expose the host root netns as a named netns @@ -121,7 +121,7 @@ sudo mkdir -p /run/netns && sudo ip netns attach xtcp2host 1 # CSV to docker logs docker run -d --name xtcp2 \ --cap-add NET_ADMIN --cap-add SYS_ADMIN \ - -v /run/netns:/run/netns:ro -v /run/docker/netns:/run/docker/netns:ro \ + -v /run/netns:/run/netns:ro,rslave -v /run/docker/netns:/run/docker/netns:ro,rslave \ xtcp2:min -dest stdout -marshal csv -d 1 -columns hostname,inetDiagMsgSocketSource,inetDiagMsgState docker logs xtcp2 # header + humanized rows ``` diff --git a/pkg/xtcp/init.go b/pkg/xtcp/init.go index 4e5b4bdb0..3f0a15dbe 100644 --- a/pkg/xtcp/init.go +++ b/pkg/xtcp/init.go @@ -92,6 +92,9 @@ func (x *XTCP) Init(ctx context.Context) { wg.Wait() + // After InitPromethus, like the enrichers below: it sets a gauge. + x.checkNetnsPropagation() + // initEnrichers runs AFTER the wait, not before it. Every enricher reports // its outcome through x.pC / x.pGV, which InitPromethus creates on one of // the goroutines above; called earlier it can bump a counter on a nil diff --git a/pkg/xtcp/netns_propagation.go b/pkg/xtcp/netns_propagation.go new file mode 100644 index 000000000..a94ff59d4 --- /dev/null +++ b/pkg/xtcp/netns_propagation.go @@ -0,0 +1,106 @@ +package xtcp + +import ( + "log" + "os" + "path/filepath" + "strings" +) + +// netns bind mounts appear under /run/netns and /run/docker/netns as pods +// start. When xtcp2 runs in a container, those dirs are bind mounts too, and +// with Docker's default rprivate propagation the container sees only the +// namespaces that existed when it started: newer ones are not collected +// (setns EINVAL on the stale file) and their container name/image enrichment +// is empty. Prod v2.0.0 ran this way (one host saw 10 of 17). The fix is +// deployment side (`:ro,rslave`); this check makes it visible. + +// mountPropagation reports whether the mount holding dir receives mounts made +// elsewhere: true for shared or slave (shared:N / master:N in mountinfo), +// false for private. found is false when no mount in mountinfo holds dir. +// +// mountinfo lines are: id parent major:minor root mountpoint options +// [optional fields...] - fstype source superoptions. The mount holding dir is +// the one with the longest mount point that is dir or a parent of it; of +// equal mount points the last listed is on top. +func mountPropagation(mountinfo []byte, dir string) (receives, found bool) { + dir = filepath.Clean(dir) + best := -1 + for _, line := range strings.Split(string(mountinfo), "\n") { + f := strings.Fields(line) + if len(f) < 7 { + continue + } + point := unescapeMountinfo(f[4]) + if !pathWithin(dir, point) || len(point) < best { + continue + } + best = len(point) + found = true + receives = false + for _, opt := range f[6:] { + if opt == "-" { + break + } + if strings.HasPrefix(opt, "shared:") || strings.HasPrefix(opt, "master:") { + receives = true + } + } + } + return receives, found +} + +// pathWithin reports whether dir is point or below it. +func pathWithin(dir, point string) bool { + if point == "/" || dir == point { + return true + } + return strings.HasPrefix(dir, point+"/") +} + +// unescapeMountinfo undoes the kernel's octal escapes (\040 space, \011 tab, +// \012 newline, \134 backslash) in a mountinfo path. +func unescapeMountinfo(s string) string { + if !strings.Contains(s, `\`) { + return s + } + var b strings.Builder + for i := 0; i < len(s); i++ { + if s[i] == '\\' && i+3 < len(s) && isOctal(s[i+1]) && isOctal(s[i+2]) && isOctal(s[i+3]) { + b.WriteByte((s[i+1]-'0')<<6 | (s[i+2]-'0')<<3 | (s[i+3] - '0')) + i += 3 + continue + } + b.WriteByte(s[i]) + } + return b.String() +} + +func isOctal(c byte) bool { return c >= '0' && c <= '7' } + +// checkNetnsPropagation warns, and sets +// xtcp_gauges{function="checkNetnsPropagation",variable="private",type=} +// to 1, for each netns dir on a private mount. Runs after InitPromethus. +func (x *XTCP) checkNetnsPropagation() { + mountinfo, err := os.ReadFile(procRoot + "/self/mountinfo") + if err != nil { + if x.debugLevel > 10 { + log.Printf("checkNetnsPropagation: %v", err) + } + return + } + for _, dir := range netNsCandidateDirs { + if _, ok := x.netNsDirs.Load(dir); !ok { + continue + } + clean := filepath.Clean(dir) + receives, found := mountPropagation(mountinfo, clean) + private := found && !receives + value := 0.0 + if private { + value = 1 + log.Printf("WARN: netns dir %s is a private mount: network namespaces created after xtcp2 started are invisible to it (not collected, no container name/image); mount it with rslave propagation, e.g. docker -v %s:%s:ro,rslave", clean, clean, clean) + } + x.pGV.WithLabelValues("checkNetnsPropagation", "private", clean).Set(value) + } +} diff --git a/pkg/xtcp/netns_propagation_test.go b/pkg/xtcp/netns_propagation_test.go new file mode 100644 index 000000000..b4a1ef1a1 --- /dev/null +++ b/pkg/xtcp/netns_propagation_test.go @@ -0,0 +1,102 @@ +package xtcp + +import ( + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// mountinfo lines as the kernel writes them. +const ( + miRoot = "22 1 259:2 / / rw,relatime shared:1 - ext4 /dev/root rw\n" + miRunShared = "25 22 0:25 / /run rw,nosuid,nodev shared:5 - tmpfs tmpfs rw,mode=755\n" + miCtrRoot = "1180 1100 0:120 / / rw,relatime - overlay overlay rw\n" + miNetnsPriv = "1203 1180 0:25 /netns /run/netns ro,nosuid,nodev - tmpfs tmpfs rw,mode=755\n" + miNetnsSlave = "1203 1180 0:25 /netns /run/netns ro,nosuid,nodev master:5 - tmpfs tmpfs rw,mode=755\n" + miDockerPriv = "1204 1180 0:25 /docker/netns /run/docker/netns ro,nosuid,nodev - tmpfs tmpfs rw,mode=755\n" + miDockerSlv = "1204 1180 0:25 /docker/netns /run/docker/netns ro,nosuid,nodev master:5 - tmpfs tmpfs rw,mode=755\n" +) + +func TestMountPropagation(t *testing.T) { + for _, tc := range []struct { + name, category, description string + mountinfo, dir string + receives, found bool + }{ + {"rslave", "positive", "container bind mount with rslave (master:N)", miCtrRoot + miNetnsSlave, "/run/netns", true, true}, + {"host-shared", "positive", "on the host, not a mount itself: the shared /run holds it", miRoot + miRunShared, "/run/docker/netns", true, true}, + {"rprivate", "negative", "container bind mount with Docker's default rprivate, the prod v2.0.0 shape", miCtrRoot + miNetnsPriv + miDockerSlv, "/run/netns", false, true}, + {"none", "negative", "no mounts listed", "", "/run/netns", false, false}, + {"unbindable", "negative", "unbindable alone is not propagation", miCtrRoot + strings.Replace(miNetnsPriv, "nodev -", "nodev unbindable -", 1), "/run/netns", false, true}, + {"sibling-prefix", "boundary", "a mount at /run/net is not a parent of /run/netns: the private root holds it", miCtrRoot + strings.Replace(miNetnsSlave, " /run/netns ", " /run/net ", 1), "/run/netns", false, true}, + {"unclean-dir", "boundary", "a dir with a doubled and a trailing slash still finds its rslave mount", miCtrRoot + miDockerSlv, "/run//docker/netns/", true, true}, + {"stacked-slave-on-top", "corner", "two mounts on one point: the later one is on top", miCtrRoot + miNetnsPriv + miNetnsSlave, "/run/netns", true, true}, + {"stacked-private-on-top", "corner", "the reverse stack", miCtrRoot + miNetnsSlave + miNetnsPriv, "/run/netns", false, true}, + {"escaped", "corner", "a mount point with a space (\\040)", miCtrRoot + strings.Replace(miNetnsSlave, " /run/netns ", ` /run/my\040netns `, 1), "/run/my netns", true, true}, + {"shared-and-slave", "corner", "shared:N master:M both receive", miCtrRoot + strings.Replace(miNetnsSlave, "master:5", "shared:9 master:5", 1), "/run/netns", true, true}, + {"superoptions", "corner", "a shared: after the - separator is not an optional field", miCtrRoot + strings.Replace(miNetnsPriv, "rw,mode=755", "shared:1", 1), "/run/netns", false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Logf("%s: %s", tc.category, tc.description) + receives, found := mountPropagation([]byte(tc.mountinfo), tc.dir) + if receives != tc.receives || found != tc.found { + t.Fatalf("got receives=%v found=%v, expected receives=%v found=%v", receives, found, tc.receives, tc.found) + } + }) + } +} + +func TestCheckNetnsPropagation(t *testing.T) { + dirs := []string{"/run/netns/", "/run/docker/netns/"} + for _, tc := range []struct { + name, category, description string + mountinfo *string // nil: no mountinfo file + present []string + expected map[string]float64 // gauge per dir; absent: no series + }{ + {"rslave", "positive", "both dirs rslave", ptr(miCtrRoot + miNetnsSlave + miDockerSlv), dirs, map[string]float64{"/run/netns": 0, "/run/docker/netns": 0}}, + {"rprivate", "negative", "both dirs rprivate, the prod v2.0.0 shape", ptr(miCtrRoot + miNetnsPriv + miDockerPriv), dirs, map[string]float64{"/run/netns": 1, "/run/docker/netns": 1}}, + {"mixed", "negative", "one of each", ptr(miCtrRoot + miNetnsSlave + miDockerPriv), dirs, map[string]float64{"/run/netns": 0, "/run/docker/netns": 1}}, + {"absent-dir", "boundary", "a dir initSyncMaps didn't find gets no series", ptr(miCtrRoot + miNetnsPriv), dirs[:1], map[string]float64{"/run/netns": 1}}, + {"no-mountinfo", "corner", "mountinfo unreadable: no series, no panic", nil, dirs, map[string]float64{}}, + {"unlisted", "corner", "no mount holds the dir: unknown, not private", ptr(""), dirs, map[string]float64{"/run/netns": 0, "/run/docker/netns": 0}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Logf("%s: %s", tc.category, tc.description) + proc := t.TempDir() + if tc.mountinfo != nil { + if err := os.MkdirAll(filepath.Join(proc, "self"), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(proc, "self", "mountinfo"), []byte(*tc.mountinfo), 0o600); err != nil { + t.Fatal(err) + } + } + prevProc, prevDirs := procRoot, netNsCandidateDirs + t.Cleanup(func() { procRoot, netNsCandidateDirs = prevProc, prevDirs }) + procRoot, netNsCandidateDirs = proc, dirs + + x := newMetricsFixture(t, 0) + x.netNsDirs = &sync.Map{} + for _, d := range tc.present { + x.netNsDirs.Store(d, true) + } + x.checkNetnsPropagation() + + if n := testutil.CollectAndCount(x.pGV); n != len(tc.expected) { + t.Fatalf("%d series, expected %d", n, len(tc.expected)) + } + for dir, want := range tc.expected { + if got := testutil.ToFloat64(x.pGV.WithLabelValues("checkNetnsPropagation", "private", dir)); got != want { + t.Fatalf("%s: got %v, expected %v", dir, got, want) + } + } + }) + } +} + +func ptr[T any](v T) *T { return &v } From f9f241b27a28063cec2368d723abfd3a4c7a1240 Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Tue, 3 Nov 2026 18:12:56 -0800 Subject: [PATCH 3/3] =?UTF-8?q?docs:=20CHANGELOG=20and=20TODO-SOON=20?= =?UTF-8?q?=C2=A730=20for=20the=20v2.0.0=20enrichment=20gaps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CHANGELOG Unreleased: the uplink fix and the netns propagation check - TODO-SOON §30: root causes of UPLINK2=lo, empty NIC/LLDP and container columns; open items for link-monitor panic isolation, AI098 veth churn, gRPC Health provisional flag, and Ubuntu guest rows Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 ++++++++++ TODO-SOON.md | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2562eae35..f73018bff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## Unreleased + +### Fixes + +- **Uplink columns: `lo` as UPLINK2, empty NIC and LLDP columns.** Automatic uplink detection took the kernel's `unreachable ::/0 dev lo`, which every host has, as a default route, and used a bond itself as the uplink, which has no NIC attributes and no LLDP neighbour. Only up, non-reject default routes off `lo` count now, and a bond, vlan or bridge holding the route is replaced by the physical NICs beneath it. `-uplinkInterfaces` is unchanged. + +### New + +- **Netns mount propagation check.** In a container, `/run/netns` and `/run/docker/netns` must be mounted `rslave` (`-v /run/netns:/run/netns:ro,rslave`). With Docker's default `rprivate`, namespaces created after xtcp2 starts are invisible: their sockets aren't collected and their container name/image stays empty. xtcp2 now logs a warning at startup and sets `xtcp_gauges{function="checkNetnsPropagation",variable="private",type=""}` to 1 for each such dir. + ## 0.2.0 (2026-10) The first tagged release. These notes compare against `32a4fed4` (2026-08-17), the last revision most deployments built from; that tree also said `0.1.0` in `VERSION`. diff --git a/TODO-SOON.md b/TODO-SOON.md index c2e96eed7..8c6fc62b7 100644 --- a/TODO-SOON.md +++ b/TODO-SOON.md @@ -2404,3 +2404,36 @@ main, so a finding like these shows on the PR that adds it. Only the ratchet runs there; see CONTRIBUTING.md "Linting" for why not `nix flake check`. Making the job a required status check is a branch-protection setting, not yet done. + +## 30. Prod v2.0.0 records: UPLINK2 `lo`, empty NIC/LLDP and container columns — PARTLY FIXED 2026-11-03 + +**What.** The xtcp-status review of the first v2.0.0 data (2026-11-03) found +four daemon-side gaps. Root causes, from the fleet check +(`fleet-snapshots/xtcp2-v2-check`): + +| Symptom | Cause | Status | +|---|---|---| +| `UPLINK2_IFNAME` = `lo` | `DefaultUplinks` took every `::/0` in `/proc/net/ipv6_route`, including the kernel's `unreachable ::/0 dev lo` (RTF_REJECT, not RTF_UP) | FIXED: up, non-reject, non-`lo` routes only | +| NIC and LLDP columns empty | the default route is on `bond0`, which has no `device` entry; LLDP neighbours are keyed by the member ports | FIXED: a virtual uplink expands to the NICs under its `lower_*` links | +| container name/image empty, `CONTAINER_RUNTIME` = `cgroupfs` | ansible-host mounted `/run/netns` and `/run/docker/netns` with Docker's default `rprivate`, so pods started after xtcp2 were invisible (setns EINVAL, stale inode, dockermeta miss); one host saw 10 of 17 netns | deploy fix in runpod/ansible-host#1120 (`ro,rslave`); xtcp2 now warns at startup and sets `xtcp_gauges{function="checkNetnsPropagation",variable="private"}` | +| those pods' sockets missing entirely | same as above | same as above | + +**Still open.** + +- **Link monitor panic isolation.** `runLinkMonitor` turns an error into + `xtcp_gauges{function="linkMonitor",variable="up"} 0` and collection carries + on (fleet data: uploads were unaffected by the baseline bug), but a panic in + the link-monitor goroutines takes the whole daemon down. Recover at the + goroutine roots, count it, and mark the link monitor down. +- **AI098 veth churn.** On a host with heavy pod churn the link-monitor + reconcile fails transiently: a veth removed mid-walk, "RDMA topology + changed", "interrupted dump". Each retry recovers, but the warn rate is + high. Treat a vanished netdev during the walk as a normal retry, not a + warning, and add a churn row to the reconcile tables. +- **gRPC Health has no provisional flag.** `baseline_provisional` (the learn + deadline, PR #284) is in `/metrics` but not in the admin gRPC Health + response. +- **Ubuntu guest rows.** Add to §29's Vagrant table: a bonded uplink with + lldpd (expect UPLINK1/2 = the member NICs with NIC and LLDP columns set), + and the container started before a new netns, once with `ro` (gauge 1, + netns missed) and once with `ro,rslave` (gauge 0, netns collected).