From b2b45d58129f8d447506d86e4816da8a235d1365 Mon Sep 17 00:00:00 2001 From: Thijs van Emmerik Date: Sun, 5 Jul 2026 08:06:32 +0000 Subject: [PATCH 1/2] e2e: tolerate multiple tunnel statuses in all-devices unicast QA test --- CHANGELOG.md | 1 + e2e/internal/qa/client_unicast_test.go | 31 +++++++++++++++++++++++ e2e/qa_alldevices_unicast_test.go | 35 +++++++++++++++++++++----- 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89870a39a..86acf56ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file. - Harden ledger writes against a slow/degraded RPC endpoint: bound each RPC request (default 15s, `--ledger-rpc-timeout`), size the connection pool above the submitter concurrency (default 128, `--ledger-rpc-max-conns`), and deadline each submission attempt so it fails fast and retries with a fresh blockhash instead of sending an expired one and failing preflight with `BlockhashNotFound`. (#3973) - E2E - Fix the multicast settlement QA test's seat-allocation ack wait. It read the reused client seat at finalized commitment and could accept the previous run's already-acked state, then withdraw while the current request was still pending. It now waits to observe the request pending before treating a cleared flag as the ack. (#3972) + - Make the all-devices unicast QA test tolerate a host reporting multiple tunnel statuses. It now selects the IBRL status via `GetUserStatuses`/`FindIBRLStatus` instead of erroring on a lingering Multicast tunnel and dropping the host, and logs a warning when a host reports more than one status. (#3976) ## [v0.29.0](https://github.com/malbeclabs/doublezero/compare/client/v0.28.0...client/v0.29.0) - 2026-07-02 diff --git a/e2e/internal/qa/client_unicast_test.go b/e2e/internal/qa/client_unicast_test.go index 5e3ab4197..fe6277770 100644 --- a/e2e/internal/qa/client_unicast_test.go +++ b/e2e/internal/qa/client_unicast_test.go @@ -3,6 +3,7 @@ package qa import ( "testing" + pb "github.com/malbeclabs/doublezero/e2e/proto/qa/gen/pb-go" "github.com/stretchr/testify/require" ) @@ -291,3 +292,33 @@ this shouldn't match }) } } + +func TestFindIBRLStatus(t *testing.T) { + t.Parallel() + + ibrl := &pb.Status{UserType: "IBRL", SessionStatus: UserStatusUp} + ibrlAllocated := &pb.Status{UserType: "IBRLWithAllocatedIP", SessionStatus: UserStatusUp} + multicast := &pb.Status{UserType: "Multicast", SessionStatus: UserStatusUp} + + tests := []struct { + name string + statuses []*pb.Status + want *pb.Status + }{ + {"nil list returns nil", nil, nil}, + {"single multicast returns sole status", []*pb.Status{multicast}, multicast}, + {"ibrl plus lingering multicast returns ibrl", []*pb.Status{multicast, ibrl}, ibrl}, + {"ibrl-allocated plus multicast returns ibrl", []*pb.Status{multicast, ibrlAllocated}, ibrlAllocated}, + {"multiple with no ibrl returns nil", []*pb.Status{multicast, multicast}, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := FindIBRLStatus(tt.statuses) + if tt.want == nil { + require.Nil(t, got) + return + } + require.Same(t, tt.want, got) + }) + } +} diff --git a/e2e/qa_alldevices_unicast_test.go b/e2e/qa_alldevices_unicast_test.go index 177a69578..75e34952a 100644 --- a/e2e/qa_alldevices_unicast_test.go +++ b/e2e/qa_alldevices_unicast_test.go @@ -164,11 +164,22 @@ func TestQA_AllDevices_UnicastConnectivity(t *testing.T) { getStatus := func(hostname string) (string, error) { for _, c := range clients { if c.Host == hostname { - status, err := c.GetUserStatus(ctx) + statuses, err := c.GetUserStatuses(ctx) if err != nil { return "", err } - return status.SessionStatus, nil + if len(statuses) > 1 { + log.Warn("Host reported multiple tunnel statuses; selecting IBRL", "client", c.Host, "count", len(statuses)) + } + // A host may report multiple tunnel statuses (an IBRL and + // a Multicast tunnel). Select the IBRL status, or the sole + // status on single-tunnel hosts. A nil result means no IBRL + // tunnel, reported as empty so the client reconnects. + ibrl := qa.FindIBRLStatus(statuses) + if ibrl == nil { + return "", nil + } + return ibrl.SessionStatus, nil } } return "", fmt.Errorf("client %s not found", hostname) @@ -389,14 +400,26 @@ func connectClientsAndWaitForRoutes( if _, ok := batch[c.Host]; !ok { continue } - status, err := c.GetUserStatus(ctx) + userStatuses, err := c.GetUserStatuses(ctx) if err != nil { log.Error("Failed to get user status", "client", c.Host, "error", err) continue } - statuses[c.Host] = status.SessionStatus - if !qa.IsStatusUp(status.SessionStatus) { - log.Warn("Client not up", "client", c.Host, "status", status.SessionStatus) + if len(userStatuses) > 1 { + log.Warn("Host reported multiple tunnel statuses; selecting IBRL", "client", c.Host, "count", len(userStatuses)) + } + // A host may report multiple tunnel statuses (an IBRL and a + // Multicast tunnel). Select the IBRL status, or the sole status + // on single-tunnel hosts. A nil result means no IBRL tunnel, so + // the host is excluded from routing. + ibrl := qa.FindIBRLStatus(userStatuses) + if ibrl == nil { + log.Warn("Client has no IBRL status", "client", c.Host) + continue + } + statuses[c.Host] = ibrl.SessionStatus + if !qa.IsStatusUp(ibrl.SessionStatus) { + log.Warn("Client not up", "client", c.Host, "status", ibrl.SessionStatus) } } connectedClients := qa.FilterStatusUpClients(allClients, batch, statuses) From 970e6d11762f318ee513dbd4bf6a784e00e096e1 Mon Sep 17 00:00:00 2001 From: Thijs van Emmerik Date: Mon, 6 Jul 2026 13:52:40 +0000 Subject: [PATCH 2/2] e2e: reword multi-tunnel status warning to avoid contradictory log --- e2e/qa_alldevices_unicast_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/qa_alldevices_unicast_test.go b/e2e/qa_alldevices_unicast_test.go index 75e34952a..8c932bffb 100644 --- a/e2e/qa_alldevices_unicast_test.go +++ b/e2e/qa_alldevices_unicast_test.go @@ -169,7 +169,7 @@ func TestQA_AllDevices_UnicastConnectivity(t *testing.T) { return "", err } if len(statuses) > 1 { - log.Warn("Host reported multiple tunnel statuses; selecting IBRL", "client", c.Host, "count", len(statuses)) + log.Warn("Host reported multiple tunnel statuses", "client", c.Host, "count", len(statuses)) } // A host may report multiple tunnel statuses (an IBRL and // a Multicast tunnel). Select the IBRL status, or the sole @@ -406,7 +406,7 @@ func connectClientsAndWaitForRoutes( continue } if len(userStatuses) > 1 { - log.Warn("Host reported multiple tunnel statuses; selecting IBRL", "client", c.Host, "count", len(userStatuses)) + log.Warn("Host reported multiple tunnel statuses", "client", c.Host, "count", len(userStatuses)) } // A host may report multiple tunnel statuses (an IBRL and a // Multicast tunnel). Select the IBRL status, or the sole status