Add tsnet.Server.Underlay for DERP and magicsock sockets

derp/derphttp/derphttp_client.go

@@ -716,6 +716,9 @@ func (c *Client) _DialRegionTLS(ctx context.Context, reg *tailcfg.DERPRegion) (t
 }
 
 func (c *Client) dialContext(ctx context.Context, proto, addr string) (net.Conn, error) {
+	if underlay := c.netMon.Underlay(); underlay != nil {
+		return underlay.DialContext(ctx, proto, addr)
+	}
 	return netns.NewDialerAlwaysDirect(c.logf, c.netMon).DialContext(ctx, proto, addr)
 }
 

derp/derphttp/underlay_test.go

@@ -0,0 +1,91 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package derphttp
+
+import (
+	"context"
+	"errors"
+	"net"
+	"strconv"
+	"testing"
+
+	"github.com/sagernet/tailscale/net/netmon"
+	"github.com/sagernet/tailscale/tailcfg"
+	"github.com/sagernet/tailscale/types/key"
+	"github.com/sagernet/tailscale/types/nettype"
+	"github.com/sagernet/tailscale/util/eventbus"
+)
+
+var errUnderlayDial = errors.New("underlay dial refused")
+
+// refusingUnderlay records the DERP dial and refuses it, so the test can
+// tell the underlay path from a direct dial.
+type refusingUnderlay struct {
+	network, address string
+}
+
+func (u *refusingUnderlay) DialContext(_ context.Context, network, address string) (net.Conn, error) {
+	u.network, u.address = network, address
+	return nil, errUnderlayDial
+}
+
+func (u *refusingUnderlay) ListenPacket(context.Context, string, string) (nettype.PacketConn, error) {
+	return nil, errors.New("unexpected ListenPacket")
+}
+
+func newRegionClientWithUnderlay(t *testing.T, underlay netmon.Underlay) (*Client, *tailcfg.DERPNode) {
+	t.Helper()
+	ln, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { ln.Close() })
+	node := &tailcfg.DERPNode{
+		Name:     "1a",
+		RegionID: 1,
+		HostName: "derp.example.invalid",
+		IPv4:     "127.0.0.1",
+		IPv6:     "none",
+		DERPPort: ln.Addr().(*net.TCPAddr).Port,
+	}
+	bus := eventbus.New()
+	t.Cleanup(bus.Close)
+	netMon, err := netmon.New(bus, t.Logf, nil, underlay)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { netMon.Close() })
+	c := NewRegionClient(key.NewNode(), t.Logf, netMon, func() *tailcfg.DERPRegion {
+		return &tailcfg.DERPRegion{RegionID: 1, Nodes: []*tailcfg.DERPNode{node}}
+	})
+	t.Cleanup(func() { c.Close() })
+	return c, node
+}
+
+func TestDialNodeUsesUnderlay(t *testing.T) {
+	underlay := new(refusingUnderlay)
+	c, node := newRegionClientWithUnderlay(t, underlay)
+
+	conn, err := c.dialNode(context.Background(), node)
+	if conn != nil {
+		conn.Close()
+		t.Fatal("dialNode connected directly, want underlay error")
+	}
+	if !errors.Is(err, errUnderlayDial) {
+		t.Fatalf("dialNode error = %v, want %v", err, errUnderlayDial)
+	}
+	if want := net.JoinHostPort(node.IPv4, strconv.Itoa(node.DERPPort)); underlay.network != "tcp4" || underlay.address != want {
+		t.Fatalf("underlay dialed %s %s, want tcp4 %s", underlay.network, underlay.address, want)
+	}
+}
+
+func TestDialNodeNilUnderlayDialsDirectly(t *testing.T) {
+	c, node := newRegionClientWithUnderlay(t, nil)
+
+	conn, err := c.dialNode(context.Background(), node)
+	if err != nil {
+		t.Fatal(err)
+	}
+	conn.Close()
+}

feature/debugportmapper/debugportmapper.go

@@ -132,7 +132,7 @@ func serveDebugPortmap(h *localapi.Handler, w http.ResponseWriter, r *http.Reque
 
 	bus := eventbus.New()
 	defer bus.Close()
-	netMon, err := netmon.New(bus, logger.WithPrefix(logf, "monitor: "), nil)
+	netMon, err := netmon.New(bus, logger.WithPrefix(logf, "monitor: "), nil, nil)
 	if err != nil {
 		logf("error creating monitor: %v", err)
 		return

net/netcheck/netcheck.go

@@ -978,7 +978,9 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetRe
 			}
 		}
 		if len(need) > 0 {
-			if opts == nil || !opts.OnlyTCP443 {
+			// ICMP cannot go through an Underlay, so skip the probes
+			// rather than open direct sockets.
+			if (opts == nil || !opts.OnlyTCP443) && c.NetMon.Underlay() == nil {
 				// Kick off ICMP in parallel to HTTPS checks; we don't
 				// reuse the same WaitGroup for those probes because we
 				// need to close the underlying Pinger after a timeout

net/netmon/export.go

@@ -1,9 +1,25 @@
 package netmon
 
 import (
+	"context"
+	"net"
+
 	N "github.com/sagernet/sing/common/network"
+	"github.com/sagernet/tailscale/types/nettype"
 )
 
+// Underlay opens the sockets that carry Tailscale's own traffic to the
+// network: DERP connections and the magicsock UDP sockets used for peer,
+// disco and STUN traffic. A nil Underlay keeps the direct netns sockets.
+type Underlay interface {
+	DialContext(ctx context.Context, network, address string) (net.Conn, error)
+	ListenPacket(ctx context.Context, network, address string) (nettype.PacketConn, error)
+}
+
 func (m *Monitor) Dialer() N.Dialer {
 	return m.dialer
 }
+
+func (m *Monitor) Underlay() Underlay {
+	return m.underlay
+}

net/netmon/netmon.go

@@ -91,6 +91,7 @@ type Monitor struct {
 	lastWall     time.Time
 	jumpDuration time.Duration // wall-clock time elapsed during detected time jump; 0 if no time jump observed since reset
 	dialer       N.Dialer
+	underlay     Underlay
 }
 
 // ChangeFunc is a callback function registered with Monitor that's called when the
@@ -372,7 +373,7 @@ func filterRoutableIPs(addrs []netip.Prefix) []netip.Prefix {
 // New instantiates and starts a monitoring instance.
 // The returned monitor is inactive until it's started by the Start method.
 // Use RegisterChangeCallback to get notified of network changes.
-func New(bus *eventbus.Bus, logf logger.Logf, dialer N.Dialer) (*Monitor, error) {
+func New(bus *eventbus.Bus, logf logger.Logf, dialer N.Dialer, underlay Underlay) (*Monitor, error) {
 	logf = logger.WithPrefix(logf, "monitor: ")
 	m := &Monitor{
 		logf:     logf,
@@ -381,6 +382,7 @@ func New(bus *eventbus.Bus, logf logger.Logf, dialer N.Dialer) (*Monitor, error)
 		stop:     make(chan struct{}),
 		lastWall: wallTime(),
 		dialer:   dialer,
+		underlay: underlay,
 	}
 	m.changed = eventbus.Publish[ChangeDelta](m.b)
 	st, err := m.interfaceStateUncached()

net/udprelay/server.go

@@ -400,7 +400,7 @@ func NewServer(logf logger.Logf, port uint16, onlyStaticAddrPorts bool, metrics
 	// in a running client.
 	bus := eventbus.New()
 	s.bus = bus
-	netMon, err := netmon.New(s.bus, logf, nil)
+	netMon, err := netmon.New(s.bus, logf, nil, nil)
 	if err != nil {
 		return nil, err
 	}

tsnet/tsnet.go

@@ -318,6 +318,12 @@ type Server struct {
 
 	Dialer N.Dialer
 
+	// Underlay, if non-nil, opens the sockets that carry Tailscale's own
+	// traffic to the network: DERP connections and the magicsock UDP sockets
+	// used for peer, disco and STUN traffic. Control plane connections keep
+	// using Dialer. If nil, direct sockets are used as before.
+	Underlay netmon.Underlay
+
 	LookupHook          dnscache.LookupHookFunc
 	PeerDNSQueryHandler ipnlocal.PeerDNSQueryHandler
 	OnlyTCP443          bool
@@ -847,7 +853,7 @@ func (s *Server) start() (reterr error) {
 		return err
 	}
 
-	s.netMon, err = netmon.New(sys.Bus.Get(), tsLogf, s.Dialer)
+	s.netMon, err = netmon.New(sys.Bus.Get(), tsLogf, s.Dialer, s.Underlay)
 	if err != nil {
 		return err
 	}

wgengine/magicsock/magicsock.go

@@ -3742,6 +3742,9 @@ func (c *Conn) listenPacket(network string, port uint16) (nettype.PacketConn, er
 	if c.testOnlyPacketListener != nil {
 		return nettype.MakePacketListenerWithNetIP(c.testOnlyPacketListener).ListenPacket(ctx, network, addr)
 	}
+	if underlay := c.netMon.Underlay(); underlay != nil {
+		return underlay.ListenPacket(ctx, network, addr)
+	}
 	listenPacketFunc := netns.ListenPacketFunc()
 	if listenPacketFunc != nil {
 		return listenPacketFunc(ctx, network, addr)

wgengine/magicsock/underlay_test.go

@@ -0,0 +1,144 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package magicsock
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/netip"
+	"slices"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/sagernet/tailscale/net/netmon"
+	"github.com/sagernet/tailscale/types/nettype"
+	"github.com/sagernet/tailscale/util/eventbus"
+	"github.com/sagernet/tailscale/util/usermetric"
+)
+
+// loopbackUnderlay records ListenPacket calls and binds them to loopback,
+// which is distinguishable from the unspecified address the direct path uses.
+type loopbackUnderlay struct {
+	mu    sync.Mutex
+	calls []string
+}
+
+func (u *loopbackUnderlay) DialContext(context.Context, string, string) (net.Conn, error) {
+	return nil, errors.New("unexpected DialContext")
+}
+
+func (u *loopbackUnderlay) ListenPacket(ctx context.Context, network, address string) (nettype.PacketConn, error) {
+	u.mu.Lock()
+	u.calls = append(u.calls, network+" "+address)
+	u.mu.Unlock()
+	_, port, err := net.SplitHostPort(address)
+	if err != nil {
+		return nil, err
+	}
+	host := "127.0.0.1"
+	if network == "udp6" {
+		host = "::1"
+	}
+	var lc net.ListenConfig
+	pc, err := lc.ListenPacket(ctx, network, net.JoinHostPort(host, port))
+	if err != nil {
+		return nil, err
+	}
+	return pc.(*net.UDPConn), nil
+}
+
+func (u *loopbackUnderlay) snapshot() []string {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+	return slices.Clone(u.calls)
+}
+
+func newUnderlayConn(t *testing.T, underlay netmon.Underlay) *Conn {
+	t.Helper()
+	bus := eventbus.New()
+	t.Cleanup(bus.Close)
+	netMon, err := netmon.New(bus, t.Logf, nil, underlay)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { netMon.Close() })
+	c, err := NewConn(Options{
+		Logf:              t.Logf,
+		NetMon:            netMon,
+		EventBus:          bus,
+		Metrics:           new(usermetric.Registry),
+		DisablePortMapper: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { c.Close() })
+	return c
+}
+
+func localAddr(t *testing.T, c *Conn) netip.AddrPort {
+	t.Helper()
+	ap, err := netip.ParseAddrPort(c.pconn4.LocalAddr().String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ap
+}
+
+func TestListenPacketUnderlay(t *testing.T) {
+	underlay := new(loopbackUnderlay)
+	c := newUnderlayConn(t, underlay)
+
+	if got := localAddr(t, c); !got.Addr().IsLoopback() {
+		t.Fatalf("IPv4 socket bound to %v, want loopback from underlay", got)
+	}
+	if got, want := underlay.snapshot(), []string{"udp6 :0", "udp4 :0"}; !slices.Equal(got, want) {
+		t.Fatalf("underlay calls = %q, want %q", got, want)
+	}
+}
+
+func TestRebindKeepsPortThroughUnderlay(t *testing.T) {
+	underlay := new(loopbackUnderlay)
+	c := newUnderlayConn(t, underlay)
+	before := localAddr(t, c)
+
+	c.Rebind()
+
+	after := localAddr(t, c)
+	if after.Port() != before.Port() {
+		t.Fatalf("port changed across rebind: %v -> %v", before, after)
+	}
+	want := "udp4 :" + strconv.Itoa(int(before.Port()))
+	if got := underlay.snapshot(); !slices.Contains(got, want) {
+		t.Fatalf("underlay calls = %q, want %q for rebind", got, want)
+	}
+}
+
+func TestUnderlayIsPerConn(t *testing.T) {
+	a, b := new(loopbackUnderlay), new(loopbackUnderlay)
+	ca := newUnderlayConn(t, a)
+	cb := newUnderlayConn(t, b)
+	beforeA, beforeB := len(a.snapshot()), len(b.snapshot())
+
+	ca.Rebind()
+
+	if got := len(a.snapshot()); got == beforeA {
+		t.Fatal("rebind of first conn did not reach its underlay")
+	}
+	if got := len(b.snapshot()); got != beforeB {
+		t.Fatalf("rebind of first conn reached second underlay: %d calls, want %d", got, beforeB)
+	}
+	if localAddr(t, cb).Port() == localAddr(t, ca).Port() {
+		t.Fatal("conns share a port")
+	}
+}
+
+func TestNilUnderlayBindsDirectly(t *testing.T) {
+	c := newUnderlayConn(t, nil)
+	if got := localAddr(t, c); !got.Addr().IsUnspecified() {
+		t.Fatalf("IPv4 socket bound to %v, want unspecified direct bind", got)
+	}
+}

wgengine/userspace.go

@@ -407,7 +407,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 	if conf.NetMon != nil {
 		e.netMon = conf.NetMon
 	} else {
-		mon, err := netmon.New(conf.EventBus, logf, nil)
+		mon, err := netmon.New(conf.EventBus, logf, nil, nil)
 		if err != nil {
 			return nil, err
 		}