kubernetes: jitter transient dial retries

driver/kubernetes/driver.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	stderrors "errors"
 	"fmt"
+	"math/rand"
 	"net"
 	"strings"
 	"syscall"
@@ -389,9 +390,20 @@ func isTransientConnectionError(err error) bool {
 	return false
 }
 
-// calculateBackoff calculates the delay for the given attempt with exponential backoff.
+// calculateBackoff returns a randomized exponential backoff delay for attempt,
+// never exceeding maxDelay.
 func calculateBackoff(attempt int, baseDelay, maxDelay time.Duration) time.Duration {
-	return min(time.Duration(1<<uint(attempt))*baseDelay, maxDelay)
+	delay := min(time.Duration(1<<uint(attempt))*baseDelay, maxDelay)
+	jitter := delay
+	if remaining := maxDelay - delay; remaining < jitter {
+		jitter = remaining
+	}
+	if jitter <= 0 {
+		return delay
+	}
+	// Add jitter on top of the exponential delay so retries never happen sooner
+	// than the schedule alone would allow.
+	return delay + time.Duration(rand.Int63n(int64(jitter)+1)) // #nosec G404 -- no strong randomness required for retry jitter
 }
 
 func (d *Driver) Client(ctx context.Context, opts ...client.ClientOpt) (*client.Client, error) {

driver/kubernetes/driver_test.go

@@ -0,0 +1,50 @@
+package kubernetes
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCalculateBackoffNeverShorterThanSchedule(t *testing.T) {
+	const (
+		baseDelay = 500 * time.Millisecond
+		maxDelay  = 10 * time.Second
+	)
+	for attempt := range 6 {
+		schedule := min(time.Duration(1<<uint(attempt))*baseDelay, maxDelay)
+		ceiling := min(2*schedule, maxDelay)
+
+		for range 500 {
+			got := calculateBackoff(attempt, baseDelay, maxDelay)
+			require.GreaterOrEqual(t, got, schedule, "attempt %d must never wait less than the exponential schedule", attempt)
+			require.LessOrEqual(t, got, ceiling, "attempt %d must not exceed twice the schedule, nor maxDelay", attempt)
+		}
+	}
+}
+
+func TestCalculateBackoffRespectsMaxDelay(t *testing.T) {
+	const (
+		baseDelay = 500 * time.Millisecond
+		maxDelay  = 10 * time.Second
+	)
+	for range 500 {
+		require.LessOrEqual(t, calculateBackoff(20, baseDelay, maxDelay), maxDelay, "a large attempt count must stay capped at maxDelay")
+	}
+}
+
+func TestCalculateBackoffVaries(t *testing.T) {
+	const (
+		baseDelay = 500 * time.Millisecond
+		maxDelay  = 10 * time.Second
+	)
+	seen := make(map[time.Duration]struct{})
+	for range 500 {
+		seen[calculateBackoff(3, baseDelay, maxDelay)] = struct{}{}
+	}
+	// A deterministic implementation returns a single value. The range at
+	// attempt 3 is four seconds wide, so one distinct value across 500 draws
+	// would not be chance.
+	require.Greater(t, len(seen), 1, "backoff must vary so concurrent builders do not retry in lockstep")
+}