From a255f9e76d9e71bd665c51564395053ed8659692 Mon Sep 17 00:00:00 2001 From: chasylexus <13235797+chasylexus@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:37:27 +0300 Subject: [PATCH] Add tsnet.Server.Underlay for DERP and magicsock sockets tsnet.Server.Dialer only reaches the control plane. DERP dials go through netns.NewDialerAlwaysDirect and magicsock binds its UDP sockets through netns.Listener or the process-global netns.ListenPacketFunc, so a tsnet user cannot move all of a server's own traffic onto one transport, and two servers in one process cannot use different transports. Add netmon.Underlay, carried per Monitor next to Dialer, and use it in derphttp.Client.dialContext and magicsock.Conn.listenPacket. Rebind keeps passing the current port through it, and the per-server value takes precedence over the process-global ListenPacketFunc. netcheck skips the ICMP probes when an Underlay is set, since ICMP cannot go through it. When Underlay is nil every path behaves as before. --- derp/derphttp/derphttp_client.go | 3 + derp/derphttp/underlay_test.go | 91 +++++++++++++ feature/debugportmapper/debugportmapper.go | 2 +- net/netcheck/netcheck.go | 4 +- net/netmon/export.go | 16 +++ net/netmon/netmon.go | 4 +- net/udprelay/server.go | 2 +- tsnet/tsnet.go | 8 +- wgengine/magicsock/magicsock.go | 3 + wgengine/magicsock/underlay_test.go | 144 +++++++++++++++++++++ wgengine/userspace.go | 2 +- 11 files changed, 273 insertions(+), 6 deletions(-) create mode 100644 derp/derphttp/underlay_test.go create mode 100644 wgengine/magicsock/underlay_test.go diff --git a/derp/derphttp/derphttp_client.go b/derp/derphttp/derphttp_client.go index 8f66dcf3ad368..73eb55460a7ef 100644 --- a/derp/derphttp/derphttp_client.go +++ b/derp/derphttp/derphttp_client.go @@ -716,6 +716,9 @@ func (c *Client) _DialRegionTLS(ctx context.Context, reg *tailcfg.DERPRegion) (t } func (c *Client) dialContext(ctx context.Context, proto, addr string) (net.Conn, error) { + if underlay := c.netMon.Underlay(); underlay != nil { + return underlay.DialContext(ctx, proto, addr) + } return netns.NewDialerAlwaysDirect(c.logf, c.netMon).DialContext(ctx, proto, addr) } diff --git a/derp/derphttp/underlay_test.go b/derp/derphttp/underlay_test.go new file mode 100644 index 0000000000000..ced597e5eff56 --- /dev/null +++ b/derp/derphttp/underlay_test.go @@ -0,0 +1,91 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package derphttp + +import ( + "context" + "errors" + "net" + "strconv" + "testing" + + "github.com/sagernet/tailscale/net/netmon" + "github.com/sagernet/tailscale/tailcfg" + "github.com/sagernet/tailscale/types/key" + "github.com/sagernet/tailscale/types/nettype" + "github.com/sagernet/tailscale/util/eventbus" +) + +var errUnderlayDial = errors.New("underlay dial refused") + +// refusingUnderlay records the DERP dial and refuses it, so the test can +// tell the underlay path from a direct dial. +type refusingUnderlay struct { + network, address string +} + +func (u *refusingUnderlay) DialContext(_ context.Context, network, address string) (net.Conn, error) { + u.network, u.address = network, address + return nil, errUnderlayDial +} + +func (u *refusingUnderlay) ListenPacket(context.Context, string, string) (nettype.PacketConn, error) { + return nil, errors.New("unexpected ListenPacket") +} + +func newRegionClientWithUnderlay(t *testing.T, underlay netmon.Underlay) (*Client, *tailcfg.DERPNode) { + t.Helper() + ln, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + node := &tailcfg.DERPNode{ + Name: "1a", + RegionID: 1, + HostName: "derp.example.invalid", + IPv4: "127.0.0.1", + IPv6: "none", + DERPPort: ln.Addr().(*net.TCPAddr).Port, + } + bus := eventbus.New() + t.Cleanup(bus.Close) + netMon, err := netmon.New(bus, t.Logf, nil, underlay) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { netMon.Close() }) + c := NewRegionClient(key.NewNode(), t.Logf, netMon, func() *tailcfg.DERPRegion { + return &tailcfg.DERPRegion{RegionID: 1, Nodes: []*tailcfg.DERPNode{node}} + }) + t.Cleanup(func() { c.Close() }) + return c, node +} + +func TestDialNodeUsesUnderlay(t *testing.T) { + underlay := new(refusingUnderlay) + c, node := newRegionClientWithUnderlay(t, underlay) + + conn, err := c.dialNode(context.Background(), node) + if conn != nil { + conn.Close() + t.Fatal("dialNode connected directly, want underlay error") + } + if !errors.Is(err, errUnderlayDial) { + t.Fatalf("dialNode error = %v, want %v", err, errUnderlayDial) + } + if want := net.JoinHostPort(node.IPv4, strconv.Itoa(node.DERPPort)); underlay.network != "tcp4" || underlay.address != want { + t.Fatalf("underlay dialed %s %s, want tcp4 %s", underlay.network, underlay.address, want) + } +} + +func TestDialNodeNilUnderlayDialsDirectly(t *testing.T) { + c, node := newRegionClientWithUnderlay(t, nil) + + conn, err := c.dialNode(context.Background(), node) + if err != nil { + t.Fatal(err) + } + conn.Close() +} diff --git a/feature/debugportmapper/debugportmapper.go b/feature/debugportmapper/debugportmapper.go index 2830de1e6b7cf..507a6d7e4114b 100644 --- a/feature/debugportmapper/debugportmapper.go +++ b/feature/debugportmapper/debugportmapper.go @@ -132,7 +132,7 @@ func serveDebugPortmap(h *localapi.Handler, w http.ResponseWriter, r *http.Reque bus := eventbus.New() defer bus.Close() - netMon, err := netmon.New(bus, logger.WithPrefix(logf, "monitor: "), nil) + netMon, err := netmon.New(bus, logger.WithPrefix(logf, "monitor: "), nil, nil) if err != nil { logf("error creating monitor: %v", err) return diff --git a/net/netcheck/netcheck.go b/net/netcheck/netcheck.go index 209cb37c0c3bb..13da0763bb4f3 100644 --- a/net/netcheck/netcheck.go +++ b/net/netcheck/netcheck.go @@ -978,7 +978,9 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetRe } } if len(need) > 0 { - if opts == nil || !opts.OnlyTCP443 { + // ICMP cannot go through an Underlay, so skip the probes + // rather than open direct sockets. + if (opts == nil || !opts.OnlyTCP443) && c.NetMon.Underlay() == nil { // Kick off ICMP in parallel to HTTPS checks; we don't // reuse the same WaitGroup for those probes because we // need to close the underlying Pinger after a timeout diff --git a/net/netmon/export.go b/net/netmon/export.go index 865ac0aba27c5..b8adc7b57c541 100644 --- a/net/netmon/export.go +++ b/net/netmon/export.go @@ -1,9 +1,25 @@ package netmon import ( + "context" + "net" + N "github.com/sagernet/sing/common/network" + "github.com/sagernet/tailscale/types/nettype" ) +// Underlay opens the sockets that carry Tailscale's own traffic to the +// network: DERP connections and the magicsock UDP sockets used for peer, +// disco and STUN traffic. A nil Underlay keeps the direct netns sockets. +type Underlay interface { + DialContext(ctx context.Context, network, address string) (net.Conn, error) + ListenPacket(ctx context.Context, network, address string) (nettype.PacketConn, error) +} + func (m *Monitor) Dialer() N.Dialer { return m.dialer } + +func (m *Monitor) Underlay() Underlay { + return m.underlay +} diff --git a/net/netmon/netmon.go b/net/netmon/netmon.go index 42334c772cc26..9188d4097b3fb 100644 --- a/net/netmon/netmon.go +++ b/net/netmon/netmon.go @@ -91,6 +91,7 @@ type Monitor struct { lastWall time.Time jumpDuration time.Duration // wall-clock time elapsed during detected time jump; 0 if no time jump observed since reset dialer N.Dialer + underlay Underlay } // ChangeFunc is a callback function registered with Monitor that's called when the @@ -372,7 +373,7 @@ func filterRoutableIPs(addrs []netip.Prefix) []netip.Prefix { // New instantiates and starts a monitoring instance. // The returned monitor is inactive until it's started by the Start method. // Use RegisterChangeCallback to get notified of network changes. -func New(bus *eventbus.Bus, logf logger.Logf, dialer N.Dialer) (*Monitor, error) { +func New(bus *eventbus.Bus, logf logger.Logf, dialer N.Dialer, underlay Underlay) (*Monitor, error) { logf = logger.WithPrefix(logf, "monitor: ") m := &Monitor{ logf: logf, @@ -381,6 +382,7 @@ func New(bus *eventbus.Bus, logf logger.Logf, dialer N.Dialer) (*Monitor, error) stop: make(chan struct{}), lastWall: wallTime(), dialer: dialer, + underlay: underlay, } m.changed = eventbus.Publish[ChangeDelta](m.b) st, err := m.interfaceStateUncached() diff --git a/net/udprelay/server.go b/net/udprelay/server.go index 818ac4261e641..58e2e5c674796 100644 --- a/net/udprelay/server.go +++ b/net/udprelay/server.go @@ -400,7 +400,7 @@ func NewServer(logf logger.Logf, port uint16, onlyStaticAddrPorts bool, metrics // in a running client. bus := eventbus.New() s.bus = bus - netMon, err := netmon.New(s.bus, logf, nil) + netMon, err := netmon.New(s.bus, logf, nil, nil) if err != nil { return nil, err } diff --git a/tsnet/tsnet.go b/tsnet/tsnet.go index 976081cb238df..5763681ddcd98 100644 --- a/tsnet/tsnet.go +++ b/tsnet/tsnet.go @@ -318,6 +318,12 @@ type Server struct { Dialer N.Dialer + // Underlay, if non-nil, opens the sockets that carry Tailscale's own + // traffic to the network: DERP connections and the magicsock UDP sockets + // used for peer, disco and STUN traffic. Control plane connections keep + // using Dialer. If nil, direct sockets are used as before. + Underlay netmon.Underlay + LookupHook dnscache.LookupHookFunc PeerDNSQueryHandler ipnlocal.PeerDNSQueryHandler OnlyTCP443 bool @@ -847,7 +853,7 @@ func (s *Server) start() (reterr error) { return err } - s.netMon, err = netmon.New(sys.Bus.Get(), tsLogf, s.Dialer) + s.netMon, err = netmon.New(sys.Bus.Get(), tsLogf, s.Dialer, s.Underlay) if err != nil { return err } diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 64be76523f8c9..2adea3015eede 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3742,6 +3742,9 @@ func (c *Conn) listenPacket(network string, port uint16) (nettype.PacketConn, er if c.testOnlyPacketListener != nil { return nettype.MakePacketListenerWithNetIP(c.testOnlyPacketListener).ListenPacket(ctx, network, addr) } + if underlay := c.netMon.Underlay(); underlay != nil { + return underlay.ListenPacket(ctx, network, addr) + } listenPacketFunc := netns.ListenPacketFunc() if listenPacketFunc != nil { return listenPacketFunc(ctx, network, addr) diff --git a/wgengine/magicsock/underlay_test.go b/wgengine/magicsock/underlay_test.go new file mode 100644 index 0000000000000..e0c9569e9c9f9 --- /dev/null +++ b/wgengine/magicsock/underlay_test.go @@ -0,0 +1,144 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package magicsock + +import ( + "context" + "errors" + "net" + "net/netip" + "slices" + "strconv" + "sync" + "testing" + + "github.com/sagernet/tailscale/net/netmon" + "github.com/sagernet/tailscale/types/nettype" + "github.com/sagernet/tailscale/util/eventbus" + "github.com/sagernet/tailscale/util/usermetric" +) + +// loopbackUnderlay records ListenPacket calls and binds them to loopback, +// which is distinguishable from the unspecified address the direct path uses. +type loopbackUnderlay struct { + mu sync.Mutex + calls []string +} + +func (u *loopbackUnderlay) DialContext(context.Context, string, string) (net.Conn, error) { + return nil, errors.New("unexpected DialContext") +} + +func (u *loopbackUnderlay) ListenPacket(ctx context.Context, network, address string) (nettype.PacketConn, error) { + u.mu.Lock() + u.calls = append(u.calls, network+" "+address) + u.mu.Unlock() + _, port, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + host := "127.0.0.1" + if network == "udp6" { + host = "::1" + } + var lc net.ListenConfig + pc, err := lc.ListenPacket(ctx, network, net.JoinHostPort(host, port)) + if err != nil { + return nil, err + } + return pc.(*net.UDPConn), nil +} + +func (u *loopbackUnderlay) snapshot() []string { + u.mu.Lock() + defer u.mu.Unlock() + return slices.Clone(u.calls) +} + +func newUnderlayConn(t *testing.T, underlay netmon.Underlay) *Conn { + t.Helper() + bus := eventbus.New() + t.Cleanup(bus.Close) + netMon, err := netmon.New(bus, t.Logf, nil, underlay) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { netMon.Close() }) + c, err := NewConn(Options{ + Logf: t.Logf, + NetMon: netMon, + EventBus: bus, + Metrics: new(usermetric.Registry), + DisablePortMapper: true, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Close() }) + return c +} + +func localAddr(t *testing.T, c *Conn) netip.AddrPort { + t.Helper() + ap, err := netip.ParseAddrPort(c.pconn4.LocalAddr().String()) + if err != nil { + t.Fatal(err) + } + return ap +} + +func TestListenPacketUnderlay(t *testing.T) { + underlay := new(loopbackUnderlay) + c := newUnderlayConn(t, underlay) + + if got := localAddr(t, c); !got.Addr().IsLoopback() { + t.Fatalf("IPv4 socket bound to %v, want loopback from underlay", got) + } + if got, want := underlay.snapshot(), []string{"udp6 :0", "udp4 :0"}; !slices.Equal(got, want) { + t.Fatalf("underlay calls = %q, want %q", got, want) + } +} + +func TestRebindKeepsPortThroughUnderlay(t *testing.T) { + underlay := new(loopbackUnderlay) + c := newUnderlayConn(t, underlay) + before := localAddr(t, c) + + c.Rebind() + + after := localAddr(t, c) + if after.Port() != before.Port() { + t.Fatalf("port changed across rebind: %v -> %v", before, after) + } + want := "udp4 :" + strconv.Itoa(int(before.Port())) + if got := underlay.snapshot(); !slices.Contains(got, want) { + t.Fatalf("underlay calls = %q, want %q for rebind", got, want) + } +} + +func TestUnderlayIsPerConn(t *testing.T) { + a, b := new(loopbackUnderlay), new(loopbackUnderlay) + ca := newUnderlayConn(t, a) + cb := newUnderlayConn(t, b) + beforeA, beforeB := len(a.snapshot()), len(b.snapshot()) + + ca.Rebind() + + if got := len(a.snapshot()); got == beforeA { + t.Fatal("rebind of first conn did not reach its underlay") + } + if got := len(b.snapshot()); got != beforeB { + t.Fatalf("rebind of first conn reached second underlay: %d calls, want %d", got, beforeB) + } + if localAddr(t, cb).Port() == localAddr(t, ca).Port() { + t.Fatal("conns share a port") + } +} + +func TestNilUnderlayBindsDirectly(t *testing.T) { + c := newUnderlayConn(t, nil) + if got := localAddr(t, c); !got.Addr().IsUnspecified() { + t.Fatalf("IPv4 socket bound to %v, want unspecified direct bind", got) + } +} diff --git a/wgengine/userspace.go b/wgengine/userspace.go index 9b4c01fb80619..c26c29054c485 100644 --- a/wgengine/userspace.go +++ b/wgengine/userspace.go @@ -407,7 +407,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error) if conf.NetMon != nil { e.netMon = conf.NetMon } else { - mon, err := netmon.New(conf.EventBus, logf, nil) + mon, err := netmon.New(conf.EventBus, logf, nil, nil) if err != nil { return nil, err }