diff --git a/core/plugins/inferencerouter/models_test.go b/core/plugins/inferencerouter/models_test.go index 9fa697ba0..9fad324de 100644 --- a/core/plugins/inferencerouter/models_test.go +++ b/core/plugins/inferencerouter/models_test.go @@ -322,6 +322,8 @@ type fakeStore struct { mu sync.Mutex vals map[string]string ttls map[string]time.Duration + // onSet, when set, sees every write after it lands. + onSet func(k, v string) } func (f *fakeStore) Get(_ context.Context, k string) (string, error) { @@ -336,6 +338,9 @@ func (f *fakeStore) Set(_ context.Context, k, v string, ttl time.Duration) error f.vals, f.ttls = map[string]string{}, map[string]time.Duration{} } f.vals[k], f.ttls[k] = v, ttl + if f.onSet != nil { + f.onSet(k, v) + } return nil } func (f *fakeStore) Incr(context.Context, string, int64) (int64, error) { return 0, nil } @@ -403,6 +408,13 @@ func TestInit_FetchesEachServersModelsAndKeepsThem(t *testing.T) { st := &fakeStore{} _ = st.Set(context.Background(), catalogPrefix+"glm", `["rits/zai-org/glm-5-3"]`, 0) + fetched := make(chan struct{}) + var once sync.Once + st.onSet = func(k, v string) { + if k == catalogPrefix+"glm" && strings.Contains(v, "glm-6") { + once.Do(func() { close(fetched) }) + } + } r := New() r.SetStore(st) r.client = srv.Client() @@ -417,17 +429,7 @@ func TestInit_FetchesEachServersModelsAndKeepsThem(t *testing.T) { // Wait for the durable copy: the fetch writes it after the in-memory one, so // seeing it in the store means both are done. - deadline := time.Now().Add(5 * time.Second) - for { - v, _ := st.Get(context.Background(), catalogPrefix+"glm") - if strings.Contains(v, "glm-6") { - break - } - if time.Now().After(deadline) { - t.Fatalf("stored list = %s, want the fetched one kept", v) - } - time.Sleep(10 * time.Millisecond) - } + <-fetched if got := routerconfig.Resolve("glm", r.catalogs.get("glm")); got != "rits/zai-org/glm-6" { t.Errorf("list = %v, want the fetched one", r.catalogs.get("glm")) } diff --git a/core/plugins/lineage/plugin.go b/core/plugins/lineage/plugin.go index e47435f97..d508fb20b 100644 --- a/core/plugins/lineage/plugin.go +++ b/core/plugins/lineage/plugin.go @@ -165,6 +165,8 @@ type LineageTelemetry struct { // bgCancel stops the self_id_file poller Init starts when the file is not // readable yet. Same shape as jwt-validation's audience_file poller. bgCancel atomic.Pointer[context.CancelFunc] + // bgDone is closed when that poller returns. + bgDone chan struct{} // exportFailures counts batches the collector refused or never received. // Export is asynchronous and the dial is lazy, so this — with the WARN // exportObserver logs — is how an unreachable collector or a TLS chain @@ -385,7 +387,12 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { // bounds synchronous work, and the file may take longer than that. bgCtx, cancel := context.WithCancel(context.Background()) p.bgCancel.Store(&cancel) - go p.awaitIdentity(bgCtx, pending, identityPollInterval) + done := make(chan struct{}) + p.bgDone = done + go func() { + defer close(done) + p.awaitIdentity(bgCtx, pending, identityPollInterval) + }() slog.Info("lineage-telemetry: initialized, not ready until self_id_file resolves", "endpoint", endpoint, "self_id_file", pending, "namespace", p.cfg.Namespace) return nil } diff --git a/core/plugins/lineage/plugin_test.go b/core/plugins/lineage/plugin_test.go index 2dba98613..f57206e6f 100644 --- a/core/plugins/lineage/plugin_test.go +++ b/core/plugins/lineage/plugin_test.go @@ -1536,9 +1536,7 @@ func TestInit_RefusesToStartWithoutIdentity(t *testing.T) { p.cfg = tc.cfg err := p.Init(context.Background()) if p.tp != nil { - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - _ = p.tp.Shutdown(ctx) - cancel() + _ = p.tp.Shutdown(context.Background()) } if tc.wantErr && err == nil { t.Fatal("Init succeeded without a resolvable identity") @@ -1588,9 +1586,7 @@ func TestInit_RefusesWithoutNamespace(t *testing.T) { if err != nil { t.Fatalf("Init refused a valid namespace %q: %v", tc.ns, err) } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - _ = p.Shutdown(ctx) // the provider AND the gRPC client Init stored - cancel() + _ = p.Shutdown(context.Background()) // the provider AND the gRPC client Init stored if got := strings.TrimSpace(tc.ns); p.cfg.Namespace != got { t.Errorf("stored namespace = %q, want %q", p.cfg.Namespace, got) } @@ -1661,9 +1657,7 @@ func TestInit_NamespaceFile(t *testing.T) { p.cfg = tc.cfg err := p.Init(context.Background()) if err == nil { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - _ = p.Shutdown(ctx) - cancel() + _ = p.Shutdown(context.Background()) } if tc.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tc.wantErr) { @@ -1693,17 +1687,11 @@ func TestNamespace_ThroughDecodeAndInit(t *testing.T) { t.Fatalf("Init: %v", err) } t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - _ = p.Shutdown(ctx) // the in-memory provider swapped in below, and the gRPC client + _ = p.Shutdown(context.Background()) // the in-memory provider swapped in below, and the gRPC client }) // Swap the real exporter for an in-memory one so the spans can be read — // after shutting the real provider down, so nothing Init built is orphaned. - { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - _ = p.tp.Shutdown(ctx) - cancel() - } + _ = p.tp.Shutdown(context.Background()) exp := tracetest.NewInMemoryExporter() p.tp = sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) p.tracer = p.tp.Tracer("test") @@ -1824,15 +1812,13 @@ func TestShutdown_StopsIdentityPoll(t *testing.T) { if err := os.WriteFile(path, []byte("weather-service\n"), 0o600); err != nil { t.Fatal(err) } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - if err := p.Shutdown(ctx); err != nil { + if err := p.Shutdown(context.Background()); err != nil { t.Fatalf("Shutdown: %v", err) } - cancel() plugins = append(plugins, p) } - time.Sleep(200 * identityPollInterval) for i, p := range plugins { + <-p.bgDone if p.Ready() { t.Fatalf("plugin %d: the identity poller outlived Shutdown and made the plugin ready again", i) } @@ -1851,23 +1837,17 @@ func initPolling(t *testing.T, path string) *LineageTelemetry { if err := p.Init(context.Background()); err != nil { t.Fatalf("Init with an unreadable self_id_file must not fail the process: %v", err) } - t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - _ = p.Shutdown(ctx) - }) + t.Cleanup(func() { _ = p.Shutdown(context.Background()) }) return p } -// waitReady blocks until the poller flips readiness, or fails the test. +// waitReady waits for the poller to return, and fails the test unless it +// left the plugin ready. func waitReady(t *testing.T, p *LineageTelemetry) { t.Helper() - deadline := time.Now().Add(3 * time.Second) - for !p.Ready() { - if time.Now().After(deadline) { - t.Fatal("plugin never became ready after the identity file appeared") - } - time.Sleep(identityPollInterval) + <-p.bgDone + if !p.Ready() { + t.Fatal("plugin not ready after the identity file appeared") } } @@ -1902,9 +1882,7 @@ func TestInit_ReadsSelfIDFile(t *testing.T) { if err := p.Init(context.Background()); err != nil { t.Fatalf("Init: %v", err) } - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - _ = p.tp.Shutdown(ctx) - cancel() + _ = p.tp.Shutdown(context.Background()) if p.selfID != "weather-service" { t.Errorf("selfID = %q, want trimmed file content", p.selfID) } @@ -2089,8 +2067,7 @@ func TestShutdown_ClosesConn(t *testing.T) { if p.conn == nil { t.Fatal("Init did not store the gRPC conn for Shutdown to close") } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() + ctx := context.Background() if err := p.Shutdown(ctx); err != nil { t.Fatalf("Shutdown returned an error: %v", err) } @@ -2122,8 +2099,7 @@ func TestShutdown_ClearsReady(t *testing.T) { if !p.Ready() { t.Fatal("Ready() is false after a successful Init") } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() + ctx := context.Background() if err := p.Shutdown(ctx); err != nil { t.Fatalf("Shutdown returned an error: %v", err) } @@ -2356,9 +2332,7 @@ func TestInit_CAFile(t *testing.T) { if !p.Ready() { t.Error("plugin not ready after a successful Init") } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - _ = p.Shutdown(ctx) + _ = p.Shutdown(context.Background()) }) } } diff --git a/core/plugins/litellm_budgettrack/forwardproxy_integration_test.go b/core/plugins/litellm_budgettrack/forwardproxy_integration_test.go index 6732fe00c..305801bac 100644 --- a/core/plugins/litellm_budgettrack/forwardproxy_integration_test.go +++ b/core/plugins/litellm_budgettrack/forwardproxy_integration_test.go @@ -75,7 +75,11 @@ func TestForwardProxyStreamedSSEUpdatesLedger(t *testing.T) { if err != nil { t.Fatalf("NewServer: %v", err) } - proxy := httptest.NewServer(srv.Handler()) + handled := make(chan struct{}) + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv.Handler().ServeHTTP(w, r) + close(handled) + })) t.Cleanup(proxy.Close) pu, _ := url.Parse(proxy.URL) @@ -98,19 +102,16 @@ func TestForwardProxyStreamedSSEUpdatesLedger(t *testing.T) { } // The terminal last=true dispatch runs in the handler's defer after the - // stream is forwarded, so poll briefly for the ledger to settle. + // stream is forwarded, so wait for the handler to return. + <-handled want := 100*1e-6 + 40*5e-6 // 0.0003 - for i := 0; i < 200; i++ { - p.mu.Lock() - got, calls := p.ledger.TotalSpend, p.ledger.TotalCalls - p.mu.Unlock() - if calls > 0 { - if got < want-1e-12 || got > want+1e-12 { - t.Fatalf("ledger TotalSpend = %v, want %v", got, want) - } - return - } - time.Sleep(10 * time.Millisecond) + p.mu.Lock() + got, calls := p.ledger.TotalSpend, p.ledger.TotalCalls + p.mu.Unlock() + if calls == 0 { + t.Fatal("ledger never updated from a streamed SSE response — the forward proxy did not dispatch frames to the plugin") + } + if got < want-1e-12 || got > want+1e-12 { + t.Fatalf("ledger TotalSpend = %v, want %v", got, want) } - t.Fatal("ledger never updated from a streamed SSE response — the forward proxy did not dispatch frames to the plugin") } diff --git a/core/plugins/script/catalog/catalog_test.go b/core/plugins/script/catalog/catalog_test.go index 68015241d..2c6a247de 100644 --- a/core/plugins/script/catalog/catalog_test.go +++ b/core/plugins/script/catalog/catalog_test.go @@ -8,7 +8,6 @@ import ( "os/exec" "path/filepath" "testing" - "time" "github.com/rossoctl/cortex/core/plugins/script/internal/worker" ) @@ -112,28 +111,20 @@ func TestCatalog_EveryScriptHandshakes(t *testing.T) { t.Fatal(err) } defer cmd.Process.Kill() - line := make(chan []byte, 1) - go func() { - s := bufio.NewScanner(out) - if s.Scan() { - line <- append([]byte(nil), s.Bytes()...) - } - close(line) - }() - select { - case b := <-line: - var hs struct { - Cortex int `json:"cortex"` - Hooks []string `json:"hooks"` - RewriteResponse bool `json:"rewrite_response"` - } - if err := json.Unmarshal(b, &hs); err != nil || hs.Cortex != worker.Protocol || len(hs.Hooks) == 0 { - t.Fatalf("handshake %q: %v", b, err) - } - rewrites = rewrites || hs.RewriteResponse - case <-time.After(10 * time.Second): - t.Fatal("no handshake within 10s") + sc := bufio.NewScanner(out) + if !sc.Scan() { + t.Fatalf("the script exited without a handshake: %v", sc.Err()) } + b := sc.Bytes() + var hs struct { + Cortex int `json:"cortex"` + Hooks []string `json:"hooks"` + RewriteResponse bool `json:"rewrite_response"` + } + if err := json.Unmarshal(b, &hs); err != nil || hs.Cortex != worker.Protocol || len(hs.Hooks) == 0 { + t.Fatalf("handshake %q: %v", b, err) + } + rewrites = rewrites || hs.RewriteResponse }) } // install orders slots by it, so it must say what the scripts declare. diff --git a/core/plugins/script/internal/worker/worker.go b/core/plugins/script/internal/worker/worker.go index b25d98e56..93de5f890 100644 --- a/core/plugins/script/internal/worker/worker.go +++ b/core/plugins/script/internal/worker/worker.go @@ -133,6 +133,9 @@ type child struct { // call made meanwhile is not held up by it. start *startup now func() time.Time + // withTimeout bounds a call's wait for sem and its exchange with the child. A + // field so a test can choose when a call's time runs out. + withTimeout func(context.Context, time.Duration) (context.Context, context.CancelFunc) } // startup is one background start of the child. @@ -150,7 +153,7 @@ func New(cfg Config) *Worker { if cfg.CallTimeout <= 0 { cfg.CallTimeout = DefaultCallTimeout } - c := &child{cfg: cfg, name: filepath.Base(cfg.Path), sem: make(chan struct{}, 1), now: time.Now} + c := &child{cfg: cfg, name: filepath.Base(cfg.Path), sem: make(chan struct{}, 1), now: time.Now, withTimeout: context.WithTimeout} return c.with(cfg.CallTimeout) } @@ -162,6 +165,9 @@ func (w *child) with(d time.Duration) *Worker { return &Worker{child: w, callTimeout: d} } +// CallTimeout is the bound on each of w's calls. +func (w *Worker) CallTimeout() time.Duration { return w.callTimeout } + func (w *child) lock(ctx context.Context) error { select { case w.sem <- struct{}{}: @@ -218,7 +224,7 @@ func (w *child) Handshake() Handshake { // marshal to a JSON object. more holds the parts the child gets only when it // fetches them; each is encoded at that time. func (w *Worker) Call(ctx context.Context, msg, more map[string]any) (Reply, error) { - queued, cancel := context.WithTimeout(ctx, w.callTimeout) + queued, cancel := w.withTimeout(ctx, w.callTimeout) defer cancel() if err := w.lock(queued); err != nil { return Reply{}, err @@ -260,7 +266,7 @@ func (w *Worker) Call(ctx context.Context, msg, more map[string]any) (Reply, err ch := make(chan result, 1) go func() { defer w.unlock() - sctx, cancel := context.WithTimeout(context.Background(), w.callTimeout) + sctx, cancel := w.withTimeout(context.Background(), w.callTimeout) defer cancel() reply, err := w.proc.call(sctx, append(line, '\n'), id, answer) if err != nil { diff --git a/core/plugins/script/internal/worker/worker_test.go b/core/plugins/script/internal/worker/worker_test.go index 63313efe4..92603e580 100644 --- a/core/plugins/script/internal/worker/worker_test.go +++ b/core/plugins/script/internal/worker/worker_test.go @@ -12,6 +12,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "slices" "strconv" "strings" @@ -43,15 +44,18 @@ func fakeWorker(mode string) { case "no-handshake": fmt.Fprintln(out, "hello, I am not a handshake") out.Flush() - time.Sleep(time.Hour) + block() case "wrong-protocol": send(map[string]any{"cortex": 99}) - time.Sleep(time.Hour) + block() case "exit-at-start": os.Exit(3) } - if d, err := time.ParseDuration(os.Getenv("FAKE_START_DELAY")); err == nil { - time.Sleep(d) + if path := os.Getenv("FAKE_STARTED"); path != "" { + signal(path) + } + if path := os.Getenv("FAKE_START_GATE"); path != "" { + await(path) } if mode == "stderr-flood-start" { os.Stderr.WriteString(strings.Repeat("x", 2<<20) + "\n") @@ -59,13 +63,19 @@ func fakeWorker(mode string) { fmt.Fprintln(out, "starting up") // stray output before the handshake is skipped send(Handshake{Cortex: Protocol, Hooks: []string{"request"}, Enforce: mode == "enforce"}) if mode == "deaf" { - time.Sleep(time.Hour) // handshakes, then never reads stdin + block() // handshakes, then never reads stdin } if mode == "spawn-child" || mode == "spawn-child-then-crash" { - // A grandchild in the same process group; the test checks a timeout kills it. + // A grandchild in the same process group, holding the test's FIFO open for + // as long as it lives; the test checks a timeout or an exit kills it. + hold, _ := os.OpenFile(os.Getenv("FAKE_HOLD"), os.O_WRONLY, 0) child := exec.Command("sleep", "3600") - _ = child.Start() - os.WriteFile(os.Getenv("FAKE_PIDFILE"), []byte(strconv.Itoa(child.Process.Pid)), 0o600) + child.Stdout = hold + if err := child.Start(); err != nil { + os.Exit(1) // the FIFO closes unsignalled, so the test reports no grandchild + } + hold.Write([]byte{1}) + hold.Close() } in := bufio.NewScanner(os.Stdin) in.Buffer(make([]byte, 1<<20), 64<<20) @@ -76,13 +86,13 @@ func fakeWorker(mode string) { count++ switch mode { case "hang", "spawn-child": - time.Sleep(time.Hour) + block() case "stderr-flood-call": os.Stderr.WriteString(strings.Repeat("x", 2<<20) + "\n") case "stderr-flood-unterminated": os.Stderr.WriteString(strings.Repeat("x", 2<<20)) case "slow": - time.Sleep(300 * time.Millisecond) + await(os.Getenv("FAKE_GATE")) case "spawn-child-then-crash": os.Exit(2) case "crash-after-one": @@ -95,14 +105,14 @@ func fakeWorker(mode string) { // Asks for every held-back part and for one the call does not hold, then // replies with what it got. if mode == "fetch-late" { - time.Sleep(300 * time.Millisecond) + await(os.Getenv("FAKE_GATE")) } fetched := map[string]any{} names, _ := msg["more"].([]any) for _, n := range append(names, "absent") { send(map[string]any{"id": msg["id"], "fetch": n}) if mode == "fetch-deaf" { - time.Sleep(time.Hour) + block() } in.Scan() var answer map[string]any @@ -121,6 +131,172 @@ func fakeWorker(mode string) { } } +// block never returns: it reads a pipe nothing writes to. +func block() { + r, w, _ := os.Pipe() + r.Read(make([]byte, 1)) + runtime.KeepAlive(w) +} + +// await reads one byte from the FIFO at path: the fake script's step waits for the +// test to release it. +func await(path string) { + f, err := os.Open(path) + if err != nil { + return + } + defer f.Close() + f.Read(make([]byte, 1)) +} + +// signal writes one byte to the FIFO at path, telling the test a step was reached. +func signal(path string) { + f, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + return + } + defer f.Close() + f.Write([]byte{1}) +} + +// fifo makes a FIFO the test holds open read-write, so neither side's open blocks +// and the fake script never reads EOF from it while the test runs. +func fifo(t *testing.T) (string, *os.File) { + t.Helper() + path := filepath.Join(t.TempDir(), "fifo") + if err := syscall.Mkfifo(path, 0o600); err != nil { + t.Fatal(err) + } + f, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { f.Close() }) + return path, f +} + +// gate returns the FAKE_GATE setting and a release that lets the fake script take +// n more of the steps it holds. +func gate(t *testing.T) (string, func(n int)) { + t.Helper() + path, f := fifo(t) + return "FAKE_GATE=" + path, func(n int) { + if _, err := f.Write(bytes.Repeat([]byte{1}, n)); err != nil { + t.Fatal(err) + } + } +} + +// grandchild opens the FIFO at path that the fake script's grandchild holds, once +// the grandchild runs. Reading it to EOF waits for the grandchild to die. +func grandchild(t *testing.T, path string) *os.File { + t.Helper() + f, err := os.Open(path) // returns once the script opens it to write + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { f.Close() }) + if _, err := io.ReadFull(f, make([]byte, 1)); err != nil { + t.Fatalf("the grandchild never started: %v", err) + } + return f +} + +// holdFIFO makes the FIFO a fake script's grandchild holds, and its FAKE_HOLD setting. +func holdFIFO(t *testing.T) (path, env string) { + t.Helper() + path = filepath.Join(t.TempDir(), "hold") + if err := syscall.Mkfifo(path, 0o600); err != nil { + t.Fatal(err) + } + return path, "FAKE_HOLD=" + path +} + +// expiring is a context whose cancellation by context.DeadlineExceeded reads as its +// deadline passing. +type expiring struct{ context.Context } + +func (e expiring) Err() error { + if err := e.Context.Err(); err != nil && context.Cause(e.Context) == context.DeadlineExceeded { + return context.DeadlineExceeded + } + return e.Context.Err() +} + +// newExpiring returns a context whose deadline passes when expire is called. +func newExpiring(parent context.Context) (ctx context.Context, expire, cancel func()) { + c, cancelCause := context.WithCancelCause(parent) + return expiring{c}, func() { cancelCause(context.DeadlineExceeded) }, func() { cancelCause(context.Canceled) } +} + +// deadline is one call timeout a worker set. +type deadline struct { + d time.Duration + expire func() +} + +// clock stands in for a worker's call timeouts. Each one passes only when the test +// expires it, and is sent on made as it is set: a call's wait for the script first, +// then, once it holds the script, its exchange. +type clock struct { + made chan deadline + // passed makes every timeout pass as it is set. + passed atomic.Bool + mu sync.Mutex + set []func() +} + +// useClock gives w's calls c's timeouts. Call it after registering w's Stop: every +// timeout passes at cleanup, before the stop, so a call a failed test leaves +// holding the script lets the stop through. +func useClock(t *testing.T, w *Worker) *clock { + c := &clock{made: make(chan deadline, 256)} + w.withTimeout = c.withTimeout + t.Cleanup(func() { + c.mu.Lock() + defer c.mu.Unlock() + for _, expire := range c.set { + expire() + } + }) + return c +} + +func (c *clock) withTimeout(parent context.Context, d time.Duration) (context.Context, context.CancelFunc) { + ctx, expire, cancel := newExpiring(parent) + if c.passed.Load() { + expire() + } + c.mu.Lock() + c.set = append(c.set, expire) + c.mu.Unlock() + c.made <- deadline{d, expire} + return ctx, cancel +} + +// exchange returns the timeout of the exchange the call that done reports on +// begins once it holds the script, and fails if the call ends first. +func exchange(t *testing.T, c *clock, done <-chan error) deadline { + t.Helper() + select { + case d := <-c.made: + return d + case err := <-done: + t.Fatalf("the call ended before it held the script: %v", err) + return deadline{} + } +} + +// callAsync runs one call and returns its error when it ends. +func callAsync(ctx context.Context, w *Worker, more map[string]any) <-chan error { + done := make(chan error, 1) + go func() { + _, err := w.Call(ctx, map[string]any{}, more) + done <- err + }() + return done +} + func fake(t *testing.T, mode string, extraEnv ...string) Config { t.Helper() exe, err := os.Executable() @@ -128,10 +304,11 @@ func fake(t *testing.T, mode string, extraEnv ...string) Config { t.Fatal(err) } return Config{ - Path: exe, - Env: append(append(os.Environ(), "FAKE_WORKER="+mode), extraEnv...), - StartTimeout: 5 * time.Second, - CallTimeout: 2 * time.Second, + Path: exe, + // A race-built fake exits at once rather than after the race runtime's exit pause. + Env: append(append(os.Environ(), "FAKE_WORKER="+mode, "GORACE=atexit_sleep_ms=0"), extraEnv...), + StartTimeout: time.Hour, + CallTimeout: time.Hour, } } @@ -182,32 +359,41 @@ func TestWorker_CallStartsTheProcessInTheBackground(t *testing.T) { if _, err := w.Call(context.Background(), map[string]any{}, nil); !errors.Is(err, ErrStarting) { t.Fatalf("first Call = %v, want ErrStarting", err) } - deadline := time.Now().Add(5 * time.Second) - for { - if _, err := w.Call(context.Background(), map[string]any{}, nil); err == nil { - return - } - if time.Now().After(deadline) { - t.Fatal("the background start never finished") - } - time.Sleep(20 * time.Millisecond) + // The call began a start; once it is done, calls are answered. + w.sem <- struct{}{} + s, running := w.start, w.proc != nil + w.unlock() + if s == nil && !running { + t.Fatal("the call started nothing") + } + if s != nil { + <-s.done + } + if _, err := w.Call(context.Background(), map[string]any{}, nil); err != nil { + t.Fatalf("Call after the background start = %v", err) } } func TestWorker_HandshakeFailures(t *testing.T) { - for mode, want := range map[string]string{ - "no-handshake": "no handshake", - "wrong-protocol": "protocol 99", - "exit-at-start": "no handshake", + for mode, tc := range map[string]struct { + want string + start time.Duration + }{ + // It never sends a handshake, so its start ends when the start timeout does. + "no-handshake": {"no handshake", time.Nanosecond}, + "wrong-protocol": {"protocol 99", 0}, + "exit-at-start": {"no handshake", 0}, } { t.Run(mode, func(t *testing.T) { cfg := fake(t, mode) - cfg.StartTimeout = 300 * time.Millisecond + if tc.start > 0 { + cfg.StartTimeout = tc.start + } w := New(cfg) t.Cleanup(w.Stop) _, err := w.Start(context.Background()) - if err == nil || !strings.Contains(err.Error(), want) { - t.Fatalf("Start error = %v, want one mentioning %q", err, want) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Start error = %v, want one mentioning %q", err, tc.want) } }) } @@ -228,21 +414,18 @@ func TestWorker_IgnoresARepliesWithAnotherID(t *testing.T) { } func TestWorker_TimeoutKillsAndRestarts(t *testing.T) { - cfg := fake(t, "hang") - cfg.CallTimeout = 200 * time.Millisecond - w := New(cfg) + w := New(fake(t, "hang")) t.Cleanup(w.Stop) + c := useClock(t, w) if _, err := w.Start(context.Background()); err != nil { t.Fatal(err) } - start := time.Now() - _, err := w.Call(context.Background(), map[string]any{}, nil) - if !errors.Is(err, context.DeadlineExceeded) { + done := callAsync(context.Background(), w, nil) + <-c.made // the wait for the script + exchange(t, c, done).expire() // the exchange, which the script never answers + if err := <-done; !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("Call error = %v", err) } - if d := time.Since(start); d > 2*time.Second { - t.Errorf("timed-out call took %v", d) - } // Backing off: refused immediately rather than started again. if _, err := w.Call(context.Background(), map[string]any{}, nil); !errors.Is(err, ErrBackoff) { t.Errorf("second Call error = %v, want ErrBackoff", err) @@ -254,9 +437,10 @@ func TestWorker_TimeoutKillsAndRestarts(t *testing.T) { func TestWorker_CallsDuringAStartFailAtOnce(t *testing.T) { for _, by := range []string{"call", "Start"} { t.Run("started by "+by, func(t *testing.T) { - cfg := fake(t, "plain", "FAKE_START_DELAY=1500ms") - cfg.CallTimeout = 500 * time.Millisecond - w := New(cfg) + // The script signals once it runs, then handshakes only when released. + startedPath, startedFIFO := fifo(t) + startGate, startGateFIFO := fifo(t) + w := New(fake(t, "plain", "FAKE_STARTED="+startedPath, "FAKE_START_GATE="+startGate)) t.Cleanup(w.Stop) started := make(chan error, 1) if by == "call" { @@ -266,21 +450,20 @@ func TestWorker_CallsDuringAStartFailAtOnce(t *testing.T) { } else { go func() { _, err := w.Start(context.Background()); started <- err }() } - time.Sleep(100 * time.Millisecond) + if _, err := startedFIFO.Read(make([]byte, 1)); err != nil { + t.Fatal(err) + } for i := range 3 { - t0 := time.Now() _, err := w.Call(context.Background(), map[string]any{}, nil) if !errors.Is(err, ErrStarting) { t.Fatalf("call %d during the start = %v, want ErrStarting", i, err) } - if d := time.Since(t0); d > 100*time.Millisecond { - t.Fatalf("call %d during the start took %v", i, d) - } } - t0 := time.Now() - w.Handshake() - if d := time.Since(t0); d > 100*time.Millisecond { - t.Errorf("Handshake during the start took %v", d) + if hs := w.Handshake(); hs.Cortex != 0 { + t.Errorf("Handshake during the first start = %+v, want none yet", hs) + } + if _, err := startGateFIFO.Write([]byte{1}); err != nil { + t.Fatal(err) } // Start joins the start in progress rather than running a second child. hs, err := w.Start(context.Background()) @@ -300,61 +483,83 @@ func TestWorker_CallsDuringAStartFailAtOnce(t *testing.T) { } // Calls queued behind a slow script may give up waiting, but a script that answers -// each call within the call timeout is not killed for it: its state survives. +// each call within the call timeout is not killed for it: its state survives. A +// call's time in the queue does not count against its exchange. func TestWorker_AQueueThatOutlastsACallerKeepsTheScript(t *testing.T) { - cfg := fake(t, "slow") - cfg.CallTimeout = time.Second - w := New(cfg) + gateEnv, release := gate(t) + w := New(fake(t, "slow", gateEnv)) t.Cleanup(w.Stop) + c := useClock(t, w) if _, err := w.Start(context.Background()); err != nil { t.Fatal(err) } - var wg sync.WaitGroup - for range 6 { // 6 x 300ms queues past a 1s timeout; no one call comes near it - wg.Add(1) - go func() { defer wg.Done(); w.Call(context.Background(), map[string]any{}, nil) }() - } - wg.Wait() - got := callKept(t, w) - if got["count"].(float64) < 2 { - t.Errorf("reply %v: the script was restarted", got) + first := callAsync(context.Background(), w, nil) + <-c.made + exchange(t, c, first) // first holds the script, which has not answered it + second := callAsync(context.Background(), w, nil) + secondQueued := <-c.made + gaveUp := callAsync(context.Background(), w, nil) + (<-c.made).expire() + if err := <-gaveUp; err == nil { + t.Fatal("a call whose time ran out in the queue succeeded") + } + release(1) + if err := <-first; err != nil { + t.Fatalf("first call = %v", err) + } + exchange(t, c, second) // second holds the script now + secondQueued.expire() + release(1) + if err := <-second; err != nil { + t.Fatalf("second call = %v: its wait in the queue cut its exchange short", err) + } + release(1) + if got := callKept(t, w); got["count"] != float64(3) { + t.Errorf("reply %v, want the third the script answered", got) } } // Under a queue longer than the call timeout, a message the script is sent is one its -// caller still waits for: the script answers no call nobody reads, so calls keep -// succeeding at the rate it can answer them. +// caller still waits for: the script answers no call nobody reads. func TestWorker_AnOverloadedScriptAnswersOnlyCallersStillWaiting(t *testing.T) { - cfg := fake(t, "slow") // 300ms a call - cfg.CallTimeout = time.Second - w := New(cfg) + gateEnv, release := gate(t) + w := New(fake(t, "slow", gateEnv)) t.Cleanup(w.Stop) + c := useClock(t, w) if _, err := w.Start(context.Background()); err != nil { t.Fatal(err) } - var ok atomic.Int64 - var wg sync.WaitGroup - end := time.Now().Add(3 * time.Second) + first := callAsync(context.Background(), w, nil) + <-c.made + exchange(t, c, first) // first holds the script, which has not answered it + // Callers queued behind it run out of time while they wait. + var queued []<-chan error for range 8 { - wg.Add(1) - go func() { - defer wg.Done() - for time.Now().Before(end) { - if _, err := w.Call(context.Background(), map[string]any{}, nil); err == nil { - ok.Add(1) - } - } - }() + queued = append(queued, callAsync(context.Background(), w, nil)) } - wg.Wait() - got := callKept(t, w) - answered := int64(got["count"].(float64)) - 1 // the script's count includes callKept's call - if answered != ok.Load() { - t.Errorf("the script answered %d calls and %d callers got a reply: %d answers went unread", - answered, ok.Load(), answered-ok.Load()) + for range queued { + (<-c.made).expire() } - if ok.Load() == 0 { - t.Error("no call succeeded") + for i, q := range queued { + if err := <-q; err == nil { + t.Fatalf("queued call %d succeeded after its time ran out", i) + } + } + release(1) + if err := <-first; err != nil { + t.Fatalf("first call = %v", err) + } + // Callers whose time has run out by the time the script is free send nothing. + c.passed.Store(true) + for i := range 20 { + if _, err := w.Call(context.Background(), map[string]any{}, nil); err == nil { + t.Fatalf("call %d succeeded after its time ran out", i) + } + } + c.passed.Store(false) + release(1) + if got := callKept(t, w); got["count"] != float64(2) { + t.Errorf("reply %v: the script was sent calls whose callers had stopped waiting", got) } } @@ -420,25 +625,17 @@ func (l lockedWriter) Write(p []byte) (int, error) { // A script that exits takes the processes it started with it. func TestWorker_AnExitKillsTheWholeProcessGroup(t *testing.T) { - pidfile := filepath.Join(t.TempDir(), "pid") - w := New(fake(t, "spawn-child-then-crash", "FAKE_PIDFILE="+pidfile)) + hold, holdEnv := holdFIFO(t) + w := New(fake(t, "spawn-child-then-crash", holdEnv)) t.Cleanup(w.Stop) if _, err := w.Start(context.Background()); err != nil { t.Fatal(err) } + g := grandchild(t, hold) w.Call(context.Background(), map[string]any{}, nil) // the script exits on it - b, err := os.ReadFile(pidfile) - if err != nil { + if _, err := io.ReadAll(g); err != nil { t.Fatal(err) } - pid, _ := strconv.Atoi(string(b)) - deadline := time.Now().Add(2 * time.Second) - for syscall.Kill(pid, 0) == nil { - if time.Now().After(deadline) { - t.Fatalf("grandchild %d survived the script's exit", pid) - } - time.Sleep(20 * time.Millisecond) - } } // A child that exited and was reaped before the worker noticed is not signalled @@ -467,39 +664,58 @@ func TestWorker_AReapedChildIsNotSignalledLater(t *testing.T) { // A caller that goes away mid-call stops waiting without killing the script. func TestWorker_ACallerThatGivesUpKeepsTheScript(t *testing.T) { - for name, mk := range map[string]func() (context.Context, context.CancelFunc){ - "cancelled mid-call": func() (context.Context, context.CancelFunc) { + // Each returns the caller's context, what makes it give up, and its cancel. + for name, tc := range map[string]struct { + mk func() (ctx context.Context, giveUp, cancel func()) + before bool + }{ + "cancelled mid-call": {mk: func() (context.Context, func(), func()) { ctx, c := context.WithCancel(context.Background()) - time.AfterFunc(50*time.Millisecond, c) - return ctx, c - }, - "its own deadline": func() (context.Context, context.CancelFunc) { - return context.WithTimeout(context.Background(), 50*time.Millisecond) - }, - "cancelled before it": func() (context.Context, context.CancelFunc) { + return ctx, c, c + }}, + "its own deadline": {mk: func() (context.Context, func(), func()) { + return newExpiring(context.Background()) + }}, + "cancelled before it": {before: true, mk: func() (context.Context, func(), func()) { ctx, c := context.WithCancel(context.Background()) c() - return ctx, c - }, + return ctx, c, c + }}, } { t.Run(name, func(t *testing.T) { - w := New(fake(t, "slow")) + gateEnv, release := gate(t) + w := New(fake(t, "slow", gateEnv)) t.Cleanup(w.Stop) + c := useClock(t, w) if _, err := w.Start(context.Background()); err != nil { t.Fatal(err) } for range 5 { - ctx, cancel := mk() - t0 := time.Now() - _, err := w.Call(ctx, map[string]any{}, nil) + ctx, giveUp, cancel := tc.mk() + done := callAsync(ctx, w, nil) + <-c.made // the wait for the script + if !tc.before { + exchange(t, c, done) // the call holds the script, which has not answered it + giveUp() + } + err := <-done cancel() if err == nil { t.Fatal("a call whose caller gave up succeeded") } - if d := time.Since(t0); d > 250*time.Millisecond { - t.Fatalf("the call outlived its caller by %v", d) + if !tc.before { + release(1) // the script answers the call nobody waits for + continue + } + // The call has let go of the script. + select { + case w.sem <- struct{}{}: + w.unlock() + default: + t.Fatal("a call whose caller had already gone holds the script") } } + release(1) callKept(t, w) }) } @@ -521,29 +737,23 @@ func callKept(t *testing.T, w *Worker) map[string]any { } func TestWorker_TimeoutKillsTheWholeProcessGroup(t *testing.T) { - pidfile := filepath.Join(t.TempDir(), "pid") - cfg := fake(t, "spawn-child", "FAKE_PIDFILE="+pidfile) - cfg.CallTimeout = 200 * time.Millisecond - w := New(cfg) + hold, holdEnv := holdFIFO(t) + w := New(fake(t, "spawn-child", holdEnv)) t.Cleanup(w.Stop) + c := useClock(t, w) if _, err := w.Start(context.Background()); err != nil { t.Fatal(err) } - if _, err := w.Call(context.Background(), map[string]any{}, nil); err == nil { + g := grandchild(t, hold) + done := callAsync(context.Background(), w, nil) + <-c.made + exchange(t, c, done).expire() + if err := <-done; err == nil { t.Fatal("call to a hanging script succeeded") } - b, err := os.ReadFile(pidfile) - if err != nil { + if _, err := io.ReadAll(g); err != nil { t.Fatal(err) } - pid, _ := strconv.Atoi(string(b)) - deadline := time.Now().Add(2 * time.Second) - for syscall.Kill(pid, 0) == nil { - if time.Now().After(deadline) { - t.Fatalf("grandchild %d survived the timeout", pid) - } - time.Sleep(20 * time.Millisecond) - } } func TestWorker_CrashThenRestartAfterBackoff(t *testing.T) { @@ -568,21 +778,9 @@ func TestWorker_CrashThenRestartAfterBackoff(t *testing.T) { t.Fatalf("call after backoff = %v, want ErrStarting", err) } // A new process: its count starts over, which is the state loss a crash costs. - deadline := time.Now().Add(5 * time.Second) - for { - r, err := w.Call(context.Background(), map[string]any{"echo": "again"}, nil) - if err == nil { - var got map[string]any - json.Unmarshal(r.Line, &got) - if got["count"] != float64(1) { - t.Errorf("after restart, reply %v", got) - } - break - } - if time.Now().After(deadline) { - t.Fatalf("never restarted: %v", err) - } - time.Sleep(20 * time.Millisecond) + // Start waits for the restart that call began. + if got, _ := call(t, w, "again"); got["count"] != float64(1) { + t.Errorf("after restart, reply %v", got) } } @@ -624,7 +822,7 @@ func TestLoginPath_ProfileThatPrints(t *testing.T) { loginPathMu.Lock() loginPathVal, loginPathTried = "", time.Time{} old := loginPathTimeout - loginPathTimeout = 30 * time.Second + loginPathTimeout = time.Hour loginPathMu.Unlock() t.Cleanup(func() { loginPathVal, loginPathTried, loginPathTimeout = "", time.Now(), old }) if got := loginPath(); got != "/from/login" { @@ -716,19 +914,25 @@ func TestLoginPath_BackgroundChildHoldingStdout(t *testing.T) { } } -// waitStopped waits for w to refuse calls; a released worker stops in the background. -func waitStopped(t *testing.T, w *Worker, what string) { +// exitOf returns a channel closed when w's running child exits. +func exitOf(t *testing.T, w *Worker) <-chan struct{} { t.Helper() - deadline := time.Now().Add(3 * time.Second) - for { - _, err := w.Call(context.Background(), map[string]any{}, nil) - if err != nil && strings.Contains(err.Error(), "stopped") { - return - } - if time.Now().After(deadline) { - t.Fatalf("%s still answers", what) - } - time.Sleep(20 * time.Millisecond) + w.sem <- struct{}{} + defer w.unlock() + if w.proc == nil { + t.Fatal("no child is running") + } + return w.proc.done +} + +// waitStopped waits for w to refuse calls; a released worker stops in the +// background. exited is w's child's exit, which a stop brings about while it holds +// the worker, so the next call sees the stop. +func waitStopped(t *testing.T, w *Worker, exited <-chan struct{}, what string) { + t.Helper() + <-exited + if _, err := w.Call(context.Background(), map[string]any{}, nil); !errors.Is(err, errStopped) { + t.Fatalf("%s still answers: %v", what, err) } } @@ -750,18 +954,34 @@ func TestPool_ReusesAnUnchangedScript(t *testing.T) { if got, _ := call(t, a, "y"); got["count"] != float64(2) { t.Errorf("state lost across builds: %v", got) } + exited := exitOf(t, a) p.Release(two) - waitStopped(t, a, "a worker no build holds") + waitStopped(t, a, exited, "a worker no build holds") +} + +// callBoundBy makes one call on w and fails unless both its timeouts, the wait for +// the script and the exchange, are want. +func callBoundBy(t *testing.T, c *clock, w *Worker, want time.Duration, what string) { + t.Helper() + if _, err := w.Call(context.Background(), map[string]any{}, nil); err != nil { + t.Fatalf("%s: call = %v", what, err) + } + for range 2 { + if d := (<-c.made).d; d != want { + t.Errorf("%s: the call was bounded by %v, want %v", what, d, want) + } + } } // A reused worker takes the call timeout of the build that acquired it, so an edited // timeout applies to the running script without restarting it. func TestPool_AReusedWorkerTakesTheNewCallTimeout(t *testing.T) { p := NewPool() - cfg := fake(t, "hang") + cfg := fake(t, "plain") cfg.CallTimeout = 10 * time.Second a := p.Acquire(p.NewHolder(), cfg, "h1") t.Cleanup(a.Stop) + c := useClock(t, a) if _, err := a.Start(context.Background()); err != nil { t.Fatal(err) } @@ -770,23 +990,18 @@ func TestPool_AReusedWorkerTakesTheNewCallTimeout(t *testing.T) { if !sameChild(a, b) { t.Fatal("a changed timeout restarted the script") } - t0 := time.Now() - if _, err := b.Call(context.Background(), map[string]any{}, nil); err == nil { - t.Fatal("call to a hanging script succeeded") - } - if d := time.Since(t0); d > 2*time.Second { - t.Errorf("call took %v: the new 200ms timeout was not applied", d) - } + callBoundBy(t, c, b, 200*time.Millisecond, "the new build") } // A build's timeout governs that build's calls only: one configured and then refused, // or one still draining beside a newer one, leaves the other's calls as they were. func TestPool_ABuildsCallTimeoutStaysWithThatBuild(t *testing.T) { p := NewPool() - cfg := fake(t, "slow") // answers in 300ms + cfg := fake(t, "plain") running := p.NewHolder() a := p.Acquire(running, cfg, "h1") t.Cleanup(a.Stop) + c := useClock(t, a) if _, err := a.Start(context.Background()); err != nil { t.Fatal(err) } @@ -794,15 +1009,11 @@ func TestPool_ABuildsCallTimeoutStaysWithThatBuild(t *testing.T) { cfg.CallTimeout = 50 * time.Millisecond refused := p.NewHolder() p.Acquire(refused, cfg, "h1") // configured, never started - if _, err := a.Call(context.Background(), map[string]any{}, nil); err != nil { - t.Fatalf("the running build's call = %v: it took the refused build's timeout", err) - } + callBoundBy(t, c, a, time.Hour, "the running build beside a refused one") next := p.NewHolder() p.Acquire(next, cfg, "h1") p.Activate(next) // the old build drains beside it - if _, err := a.Call(context.Background(), map[string]any{}, nil); err != nil { - t.Fatalf("the draining build's call = %v: it took the newer build's timeout", err) - } + callBoundBy(t, c, a, time.Hour, "the draining build") } // The environment is fixed for a process's life, so a changed one gets a new worker. @@ -838,8 +1049,9 @@ func TestPool_AnEditedScriptKeepsItsOldWorkerUntilTheOldBuildIsReleased(t *testi call(t, old, "still serving") // the running pipeline still works p.Activate(two) call(t, old, "draining") + exited := exitOf(t, old) p.Release(one) - waitStopped(t, old, "the old version") + waitStopped(t, old, exited, "the old version") call(t, edited, "new") } @@ -856,11 +1068,12 @@ func TestPool_AnAbandonedBuildIsSweptWhenALaterOneStarts(t *testing.T) { refused := p.NewHolder() orphan := p.Acquire(refused, cfg, "h2") // never activated, never released call(t, orphan, "x") + exited := exitOf(t, orphan) next := p.NewHolder() p.Acquire(next, cfg, "h1") p.Activate(next) - waitStopped(t, orphan, "a refused build's worker") + waitStopped(t, orphan, exited, "a refused build's worker") call(t, kept, "the running build's worker") } @@ -868,10 +1081,9 @@ func TestPool_AnAbandonedBuildIsSweptWhenALaterOneStarts(t *testing.T) { // stops reading fills the pipe; a body larger than the pipe buffer — which every // inference request is — must still time out rather than block the request. func TestWorker_TimeoutCoversTheWrite(t *testing.T) { - cfg := fake(t, "deaf") - cfg.CallTimeout = 200 * time.Millisecond - w := New(cfg) + w := New(fake(t, "deaf")) t.Cleanup(w.Stop) + c := useClock(t, w) if _, err := w.Start(context.Background()); err != nil { t.Fatal(err) } @@ -880,28 +1092,16 @@ func TestWorker_TimeoutCoversTheWrite(t *testing.T) { _, err := w.Call(context.Background(), map[string]any{"body": strings.Repeat("x", 2<<20)}, nil) done <- err }() - select { - case err := <-done: - if err == nil { - t.Fatal("a call to a script that does not read succeeded") - } - case <-time.After(3 * time.Second): - t.Fatal("the call blocked past its 200ms deadline") + <-c.made + exchange(t, c, done).expire() // the exchange, whose write the script never reads + if err := <-done; err == nil { + t.Fatal("a call to a script that does not read succeeded") } - start := time.Now() + // Neither the next call nor Stop waits behind the stuck write. if _, err := w.Call(context.Background(), map[string]any{}, nil); !errors.Is(err, ErrBackoff) { t.Errorf("next call = %v, want ErrBackoff", err) } - if d := time.Since(start); d > time.Second { - t.Errorf("the next call waited %v behind the stuck one", d) - } - stopped := make(chan struct{}) - go func() { w.Stop(); close(stopped) }() - select { - case <-stopped: - case <-time.After(3 * time.Second): - t.Fatal("Stop blocked behind the stuck call") - } + w.Stop() } // A held-back part reaches the child only when it asks, named in "more" and @@ -949,17 +1149,23 @@ func (c encodeCount) MarshalJSON() ([]byte, error) { // change once the caller stops waiting. So a fetch that comes after Call has // returned gets null, and the part is never encoded. func TestWorker_FetchAfterTheCallerGaveUpGetsNull(t *testing.T) { - w := New(fake(t, "fetch-late")) // fetches 300ms after each message + gateEnv, release := gate(t) + w := New(fake(t, "fetch-late", gateEnv)) // fetches only when released t.Cleanup(w.Stop) + c := useClock(t, w) if _, err := w.Start(context.Background()); err != nil { t.Fatal(err) } var encoded atomic.Int32 - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() - if _, err := w.Call(ctx, map[string]any{}, map[string]any{"body": encodeCount{&encoded}}); err == nil { + ctx, cancel := context.WithCancel(context.Background()) + done := callAsync(ctx, w, map[string]any{"body": encodeCount{&encoded}}) + <-c.made + exchange(t, c, done) // the call holds the script, which has not fetched yet + cancel() + if err := <-done; err == nil { t.Fatal("a call whose caller gave up succeeded") } + release(2) // the late fetch, then the next call // This call waits for the child to reply to the first, so the late fetch is done. if _, err := w.Call(context.Background(), map[string]any{}, nil); err != nil { t.Fatal(err) @@ -972,24 +1178,30 @@ func TestWorker_FetchAfterTheCallerGaveUpGetsNull(t *testing.T) { // The call deadline covers the answer to a fetch: a script that asks for a part // larger than the pipe buffer and then stops reading must not block the request. func TestWorker_TimeoutCoversAFetch(t *testing.T) { - cfg := fake(t, "fetch-deaf") - cfg.CallTimeout = 200 * time.Millisecond - w := New(cfg) + w := New(fake(t, "fetch-deaf")) t.Cleanup(w.Stop) + c := useClock(t, w) if _, err := w.Start(context.Background()); err != nil { t.Fatal(err) } - done := make(chan error, 1) - go func() { - _, err := w.Call(context.Background(), map[string]any{}, map[string]any{"body": strings.Repeat("x", 2<<20)}) - done <- err - }() - select { - case err := <-done: - if err == nil { - t.Fatal("a call to a script that stopped reading succeeded") - } - case <-time.After(3 * time.Second): - t.Fatal("the fetch answer blocked past the 200ms deadline") + part := askedFor{asked: make(chan struct{}), value: strings.Repeat("x", 2<<20)} + done := callAsync(context.Background(), w, map[string]any{"body": part}) + <-c.made + ex := exchange(t, c, done) + <-part.asked // the script asked for the part and reads nothing more + ex.expire() + if err := <-done; err == nil { + t.Fatal("a call to a script that stopped reading succeeded") } } + +// askedFor is a held-back part that reports when it is fetched. +type askedFor struct { + asked chan struct{} + value string +} + +func (a askedFor) MarshalJSON() ([]byte, error) { + close(a.asked) + return json.Marshal(a.value) +} diff --git a/core/plugins/script/plugin.go b/core/plugins/script/plugin.go index daadc9c71..2527173e0 100644 --- a/core/plugins/script/plugin.go +++ b/core/plugins/script/plugin.go @@ -49,6 +49,10 @@ func init() { // worker — and its state — to the next build. var pool = worker.NewPool() +// startTimeout bounds each script's handshake; zero is the worker's default. A var +// so a test can give up on a script at once. +var startTimeout time.Duration + type config struct { Dir string `json:"dir" description:"Folder of scripts, relative to ~/.cortex/plugins. Empty: that folder itself." default:""` Timeout string `json:"timeout" description:"How long one script may take to answer one hook." default:"2s"` @@ -186,7 +190,7 @@ func (s *Script) Configure(raw json.RawMessage) error { results := make([]*loaded, len(files)) var wg sync.WaitGroup for i, f := range files { - w := pool.Acquire(s.holder, worker.Config{Path: f.path, Env: env, CallTimeout: s.timeout}, f.hash) + w := pool.Acquire(s.holder, worker.Config{Path: f.path, Env: env, StartTimeout: startTimeout, CallTimeout: s.timeout}, f.hash) wg.Add(1) go func() { defer wg.Done() @@ -363,6 +367,9 @@ const finalReserve = time.Second // after the slot. var errReserveSpent = errors.New("skipped to leave the response deadline's reserve for the plugins after the slot") +// budgetTimeout bounds a response-hook call to its share; tests replace it. +var budgetTimeout = context.WithTimeout + // responseBudget narrows ctx to the part of its deadline one response-hook call // may spend. ok is false when the reserve is all that remains. func responseBudget(ctx context.Context) (context.Context, context.CancelFunc, bool) { @@ -374,7 +381,7 @@ func responseBudget(ctx context.Context) (context.Context, context.CancelFunc, b if budget <= 0 { return nil, nil, false } - narrowed, cancel := context.WithTimeout(ctx, budget) + narrowed, cancel := budgetTimeout(ctx, budget) return narrowed, cancel, true } diff --git a/core/plugins/script/plugin_test.go b/core/plugins/script/plugin_test.go index 4404c58b3..e349e1eec 100644 --- a/core/plugins/script/plugin_test.go +++ b/core/plugins/script/plugin_test.go @@ -5,12 +5,14 @@ import ( "context" "encoding/json" "fmt" + "io" "maps" "net/http" "os" "path/filepath" "slices" "strings" + "syscall" "testing" "time" @@ -26,6 +28,7 @@ func TestMain(m *testing.M) { fakeScript(mode) os.Exit(0) } + startTimeout = time.Hour // a script's start ends with its handshake or its exit os.Exit(m.Run()) } @@ -534,13 +537,15 @@ func TestConfigure_ARebuildClearsAScriptThatDidNotStart(t *testing.T) { // counts as not started. func TestOnRequest_FailClosedRefusesAScriptThatNeverAnswers(t *testing.T) { dir := home(t) - if err := os.WriteFile(filepath.Join(dir, "guard"), []byte("#!/bin/sh\nexec sleep 30\n"), 0o700); err != nil { + if err := os.WriteFile(filepath.Join(dir, "guard"), []byte("#!/bin/sh\nwhile read -r _; do :; done\n"), 0o700); err != nil { t.Fatal(err) } - start := time.Now() + prev := startTimeout + startTimeout = time.Nanosecond + t.Cleanup(func() { startTimeout = prev }) s := configured(t, true, `{"fail_closed": true}`) - if d := time.Since(start); d > worker.DefaultStartTimeout+2*time.Second { - t.Errorf("Configure took %v", d) + if got := metric(s, "scripts not started"); got != 1 { + t.Errorf("scripts not started = %v, want 1", got) } if a := s.OnRequest(context.Background(), request("x")); a.Type != pipeline.Reject { t.Fatal("a script that never answered let requests through") @@ -585,24 +590,42 @@ func TestConfigure_RebuildKeepsScriptState(t *testing.T) { } } -// The slot's timeout bounds a call to a script that never answers. +// The slot's timeout is the worker's call timeout, and a call it cuts short is a script error. func TestOnRequest_Timeout(t *testing.T) { dir := home(t) - exe, _ := os.Executable() - // Answers the handshake, then never replies. - body := fmt.Sprintf("#!/bin/sh\nprintf '{\"cortex\":1,\"hooks\":[\"request\"],\"enforce\":true}\\n'\nexec sleep 3600\n# %s\n", exe) - os.WriteFile(filepath.Join(dir, "hang"), []byte(body), 0o700) - s := configured(t, true, `{"timeout": "150ms"}`) - start := time.Now() + hang(t, dir, "request") + s := configured(t, true, `{"timeout": "1ns"}`) + if got := s.loaded[0].w.CallTimeout(); got != time.Nanosecond { + t.Fatalf("the script's calls are bounded by %v, want the slot's 1ns", got) + } pctx := request("x") if a := s.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue { t.Fatal("a hung script blocked the request") } - if d := time.Since(start); d > 2*time.Second { - t.Errorf("took %v", d) + if invs := invocations(pctx); len(invs) != 1 || invs[0].Plugin != "hang" || invs[0].Reason != "script_error" { + t.Errorf("invocations = %+v, want one script_error row for hang", invs) } } +// hang writes a script into dir that answers the handshake for hook, then reads +// until its stdin closes and never replies. +func hang(t *testing.T, dir, hook string) { + t.Helper() + body := fmt.Sprintf("#!/bin/sh\nprintf '{\"cortex\":%d,\"hooks\":[\"%s\"],\"enforce\":true}\\n'\nwhile read -r _; do :; done\n", worker.Protocol, hook) + if err := os.WriteFile(filepath.Join(dir, "hang"), []byte(body), 0o700); err != nil { + t.Fatal(err) + } +} + +// deadlineOnly carries a deadline and is never done, so only what the code under +// test derives from the deadline can expire. +type deadlineOnly struct { + context.Context + d time.Time +} + +func (c deadlineOnly) Deadline() (time.Time, bool) { return c.d, true } + // A rebuild while a crashed script waits out its backoff keeps the script loaded. func TestConfigure_KeepsAScriptThatIsBackingOff(t *testing.T) { dir := home(t) @@ -642,23 +665,39 @@ func TestInit_SweepsARefusedBuildsWorkers(t *testing.T) { script(t, dir, "count", "count") running := configured(t, true, "") exe, _ := os.Executable() - os.WriteFile(filepath.Join(dir, "count"), []byte(fmt.Sprintf("#!/bin/sh\n# v2\nFAKE_SCRIPT=count exec %q\n", exe)), 0o700) - refused := built(t, true, "") + // The refused build's script holds a FIFO open for as long as it runs, so reading + // the FIFO to EOF waits for it to exit. + hold := filepath.Join(t.TempDir(), "hold") + if err := syscall.Mkfifo(hold, 0o600); err != nil { + t.Fatal(err) + } + os.WriteFile(filepath.Join(dir, "count"), []byte(fmt.Sprintf("#!/bin/sh\n# v2\nexec 3>%q\nFAKE_SCRIPT=count exec %q\n", hold, exe)), 0o700) + held := make(chan *os.File, 1) + go func() { + f, err := os.Open(hold) // returns once the script opens it to write + if err != nil { + t.Error(err) + } + held <- f + }() + refused := built(t, true, "") // starts the script orphan := refused.loaded[0].w + f := <-held + if f == nil { + t.FailNow() + } + defer f.Close() os.WriteFile(filepath.Join(dir, "count"), []byte(fmt.Sprintf("#!/bin/sh\n# v3\nFAKE_SCRIPT=count exec %q\n", exe)), 0o700) next := built(t, true, "") next.Init(context.Background()) running.Shutdown(context.Background()) - deadline := time.Now().Add(3 * time.Second) - for { - _, err := orphan.Call(context.Background(), map[string]any{}, nil) - if err != nil && strings.Contains(err.Error(), "stopped") { - break - } - if time.Now().After(deadline) { - t.Fatal("a refused build's worker was never stopped") - } - time.Sleep(20 * time.Millisecond) + // The orphan's script exits once its stop closes its stdin, while that stop holds + // the worker, so the next call sees the stop. + if _, err := io.ReadAll(f); err != nil { + t.Fatal(err) + } + if _, err := orphan.Call(context.Background(), map[string]any{}, nil); err == nil || !strings.Contains(err.Error(), "stopped") { + t.Fatalf("a refused build's worker answers: %v", err) } if a := next.OnRequest(context.Background(), request("x")); a.Type != pipeline.Continue { t.Fatal("the build that started does not work") @@ -1028,21 +1067,23 @@ func (l *lastFrame) OnResponseFrame(_ context.Context, _ *pipeline.Context, _ [] // agent a 499 for a response that arrived complete and drops the settle. func TestResponse_SlowScriptsLeaveTheDeadlineForTheParsersAfterTheSlot(t *testing.T) { dir := home(t) - // Answers the handshake, then never replies. - body := "#!/bin/sh\nprintf '{\"cortex\":1,\"hooks\":[\"response\"],\"enforce\":true}\\n'\nexec sleep 3600\n" - if err := os.WriteFile(filepath.Join(dir, "hang"), []byte(body), 0o700); err != nil { - t.Fatal(err) - } - // A timeout larger than the dispatch deadline, so only the reserve can stop it. - s := configured(t, true, `{"timeout": "30s"}`) + hang(t, dir, "response") + // A timeout no test outlasts, so the share is the only bound on the call. + s := configured(t, true, `{"timeout": "1h"}`) + // The share is spent as soon as it is given, so the call ends before it reaches the script. + var share time.Duration + budgetTimeout = func(parent context.Context, d time.Duration) (context.Context, context.CancelFunc) { + share = d + return context.WithTimeout(parent, 0) + } + t.Cleanup(func() { budgetTimeout = context.WithTimeout }) rec := &lastFrame{} p, err := pipeline.New([]pipeline.Plugin{rec, s}) if err != nil { t.Fatal(err) } pctx := response(`{"answer":"hi"}`) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() + ctx := deadlineOnly{context.Background(), time.Now().Add(finalReserve + time.Hour)} a := p.RunResponseFrame(ctx, pctx, pctx.ResponseBody, true) if a.Type != pipeline.Continue { t.Fatalf("action %+v", a) @@ -1050,8 +1091,15 @@ func TestResponse_SlowScriptsLeaveTheDeadlineForTheParsersAfterTheSlot(t *testin if !rec.got { t.Fatal("the parser after the slot never saw its last frame") } - if d, ok := ctx.Deadline(); !ok || time.Until(d) <= 0 { - t.Fatal("the scripts spent the whole dispatch deadline") + inv := invocations(pctx)[0] + if inv.Plugin != "hang" || inv.Reason != "script_error" { + t.Errorf("the expired call is recorded as %+v", inv) + } + if err := inv.Details["error"]; !strings.Contains(err, context.DeadlineExceeded.Error()) { + t.Errorf("the call ended with %q, want its share spent", err) + } + if share <= 0 || share > time.Hour { + t.Errorf("the script's share is %v, want the deadline less the reserve", share) } } @@ -1068,8 +1116,7 @@ func TestResponse_AnExhaustedDeadlineSkipsTheScripts(t *testing.T) { t.Fatal(err) } pctx := response(`{"answer":"hi"}`) - ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) - defer cancel() + ctx := deadlineOnly{context.Background(), time.Now().Add(finalReserve)} a := p.RunResponseFrame(ctx, pctx, pctx.ResponseBody, true) if a.Type != pipeline.Continue { t.Fatalf("action %+v", a) @@ -1077,11 +1124,27 @@ func TestResponse_AnExhaustedDeadlineSkipsTheScripts(t *testing.T) { if !rec.got { t.Fatal("the parser after the slot never saw its last frame") } - if inv := invocations(pctx)[0]; inv.Reason != "script_error" || inv.Shadow { + if inv := invocations(pctx)[0]; inv.Reason != "script_error" || inv.Shadow || inv.Details["error"] != errReserveSpent.Error() { t.Errorf("the skipped script is recorded as %+v", inv) } } +// The scripts' share of a response deadline ends finalReserve before it. +func TestResponseBudget_EndsTheReserveBeforeTheDeadline(t *testing.T) { + before := time.Now() + d := before.Add(finalReserve + time.Hour) + narrowed, cancel, ok := responseBudget(deadlineOnly{context.Background(), d}) + after := time.Now() + if !ok { + t.Fatal("an hour past the reserve was refused") + } + defer cancel() + got, _ := narrowed.Deadline() + if want := d.Add(-finalReserve); got.Before(want) || got.After(want.Add(after.Sub(before))) { + t.Errorf("the scripts' deadline is %v from the dispatch's, want %v", d.Sub(got), finalReserve) + } +} + // Set-Cookie is a credential too, and never reaches a script. func TestResponse_HeadersHideCookies(t *testing.T) { if _, ok := flatten(response("").ResponseHeaders)["set-cookie"]; ok { diff --git a/core/plugins/sessionbudget/e2e_test.go b/core/plugins/sessionbudget/e2e_test.go index 0e8e9756c..a80261c21 100644 --- a/core/plugins/sessionbudget/e2e_test.go +++ b/core/plugins/sessionbudget/e2e_test.go @@ -9,6 +9,7 @@ import ( "net/url" "sync" "testing" + "testing/synctest" "time" "github.com/rossoctl/cortex/core/listener/forwardproxy" @@ -46,7 +47,7 @@ func newE2EPluginPause(t *testing.T, maxTokens int64, store *memStore, webhookUR MaxTokens: maxTokens, OnExceed: "pause", PauseWebhook: webhookURL, - PauseTimeout: "2s", + PauseTimeout: "1h", PauseTimeoutAction: "deny", PauseGracePeriod: "0s", RefreshInterval: "30ms", @@ -270,11 +271,7 @@ func TestE2E_PodRestart(t *testing.T) { // (deny/observe intentionally skip cold-cache to keep Redis off the hot path). func TestE2E_HydrateSingleflight(t *testing.T) { inner := newMemStore() - cs := &controllableStore{inner: inner, hashGetDelay: 50 * time.Millisecond} - webhook := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(`{"action":"deny"}`)) - })) - defer webhook.Close() + cs := &controllableStore{inner: inner} // Build without starting refreshLoop so background HashGet calls don't // pollute the singleflight counter we're asserting on. p := New() @@ -282,8 +279,8 @@ func TestE2E_HydrateSingleflight(t *testing.T) { RedisURL: "mem://test", MaxTokens: 200, OnExceed: "pause", - PauseWebhook: webhook.URL, - PauseTimeout: "2s", + PauseWebhook: "http://127.0.0.1:1/unused", + PauseTimeout: "1h", PauseTimeoutAction: "deny", PauseGracePeriod: "0s", RefreshInterval: "30ms", @@ -293,52 +290,56 @@ func TestE2E_HydrateSingleflight(t *testing.T) { t.Fatalf("Configure: %v", err) } p.store = cs - p.httpClient = &http.Client{Timeout: 0} + // Under the limit, so the requests hydrate and continue without a webhook. ctx := context.Background() - inner.HashIncr(ctx, "session-budget:s", "tokens", 210) + inner.HashIncr(ctx, "session-budget:s", "tokens", 100) inner.HashIncr(ctx, "session-budget:s", "calls", 8) inner.HashSetNX(ctx, "session-budget:s", "started_at", "1700000000") - // This test asserts singleflight dedup on hydrate — it counts HashGet - // calls, not per-request outcomes. Pause-mode concurrent breaches - // piggyback on pendingApproval (one leader gets Reject, others continue - // via pause_pending_approval), so we don't assert per-request Reject. - const N = 20 - var wg sync.WaitGroup - wg.Add(N) - for i := 0; i < N; i++ { - go func() { - defer wg.Done() - _ = request(p, "s") - }() - } - wg.Wait() - - // Without singleflight all N calls would hit HashGet. With it, only the - // first flight does — later arrivals see the populated cache and skip hydrate. - // Allow a small slack for goroutines that raced past the cache-check before - // the first flight populated it. - got := cs.hashGetCalls() - if got > 3 { - t.Errorf("HashGet called %d times for %d concurrent cold-cache requests; expected ≤3 (singleflight dedup)", got, N) - } + synctest.Test(t, func(t *testing.T) { + gate := make(chan struct{}) + cs.setHashGetGate(gate) + const N = 20 + var wg sync.WaitGroup + wg.Add(N) + for i := 0; i < N; i++ { + go func() { + defer wg.Done() + if a := request(p, "s"); a.Type != pipeline.Continue { + t.Errorf("under-budget request: expected Continue, got %v", a.Type) + } + }() + } + // Every request is now waiting on the one lookup the gate holds. + synctest.Wait() + if got := cs.hashGetCalls(); got != 1 { + t.Errorf("HashGet called %d times for %d concurrent cold-cache requests, want 1", got, N) + } + close(gate) + wg.Wait() + }) } // controllableStore delegates to inner memStore but can be toggled to fail, -// counts HashGet calls, and optionally injects latency into HashGet. +// counts HashGet calls, and can hold HashGet until a gate closes. type controllableStore struct { - inner *memStore - failing bool - hashGetDelay time.Duration - mu sync.Mutex - hashGets int + inner *memStore + failing bool + hashGetGate chan struct{} + mu sync.Mutex + hashGets int } func (c *controllableStore) setFailing(v bool) { c.mu.Lock(); c.failing = v; c.mu.Unlock() } func (c *controllableStore) isFailing() bool { c.mu.Lock(); defer c.mu.Unlock(); return c.failing } func (c *controllableStore) hashGetCalls() int { c.mu.Lock(); defer c.mu.Unlock(); return c.hashGets } func (c *controllableStore) err() error { return context.DeadlineExceeded } +func (c *controllableStore) setHashGetGate(g chan struct{}) { + c.mu.Lock() + c.hashGetGate = g + c.mu.Unlock() +} func (c *controllableStore) Get(ctx context.Context, key string) (string, error) { if c.isFailing() { @@ -367,11 +368,11 @@ func (c *controllableStore) HashIncr(ctx context.Context, key, field string, del func (c *controllableStore) HashGet(ctx context.Context, key string) (map[string]string, error) { c.mu.Lock() c.hashGets++ - delay := c.hashGetDelay + gate := c.hashGetGate failing := c.failing c.mu.Unlock() - if delay > 0 { - time.Sleep(delay) + if gate != nil { + <-gate } if failing { return nil, c.err() @@ -423,7 +424,7 @@ func TestE2E_PauseMode(t *testing.T) { MaxCalls: 3, OnExceed: "pause", PauseWebhook: srv.URL, - PauseTimeout: "5s", + PauseTimeout: "1h", PauseTimeoutAction: "deny", RefreshInterval: "30ms", RedisUnavailable: "fail_open", diff --git a/core/plugins/sessionbudget/plugin_test.go b/core/plugins/sessionbudget/plugin_test.go index 3e570f7c6..b427842cf 100644 --- a/core/plugins/sessionbudget/plugin_test.go +++ b/core/plugins/sessionbudget/plugin_test.go @@ -666,7 +666,7 @@ func newPausePlugin(t *testing.T, maxCalls int64, webhookURL, timeoutAction stri "max_calls": %d, "on_exceed": "pause", "pause_webhook": %q, - "pause_timeout": "200ms", + "pause_timeout": "1h", "pause_timeout_action": %q, "refresh_interval": "100ms" }`, maxCalls, webhookURL, timeoutAction) @@ -722,6 +722,9 @@ func TestOnRequest_PauseWebhookFailureFallback(t *testing.T) { defer func() { close(done); srv.Close() }() p := newPausePlugin(t, 3, srv.URL, tt.action) + if tt.handler == nil { + p.pauseTimeout = time.Nanosecond // the hanging webhook times out at once + } p.mu.Lock() p.cache["sess-1"] = &counters{tokens: 0, calls: 3, startedAt: time.Now()} p.mu.Unlock() @@ -761,7 +764,7 @@ func TestRefreshCache_PreservesLastApprovedAt(t *testing.T) { "max_calls": 3, "on_exceed": "pause", "pause_webhook": %q, - "pause_timeout": "2s", + "pause_timeout": "1h", "pause_timeout_action": "deny", "pause_grace_period": "10m", "refresh_interval": "100ms" @@ -820,7 +823,7 @@ func TestOnRequest_PauseGraceWindow(t *testing.T) { "max_calls": 3, "on_exceed": "pause", "pause_webhook": %q, - "pause_timeout": "2s", + "pause_timeout": "1h", "pause_timeout_action": "deny", "pause_grace_period": "10m", "refresh_interval": "100ms" @@ -869,7 +872,7 @@ func TestOnRequest_PauseGraceExpired(t *testing.T) { "max_calls": 3, "on_exceed": "pause", "pause_webhook": %q, - "pause_timeout": "2s", + "pause_timeout": "1h", "pause_timeout_action": "deny", "pause_grace_period": "10m", "refresh_interval": "100ms" diff --git a/core/plugins/tokenbroker/client/client_network_test.go b/core/plugins/tokenbroker/client/client_network_test.go index 40671b511..9b28a2f06 100644 --- a/core/plugins/tokenbroker/client/client_network_test.go +++ b/core/plugins/tokenbroker/client/client_network_test.go @@ -17,16 +17,16 @@ import ( // Context and Timeout Tests // ============================================================================= +// The client's timeout surfaces as a timeout error. At 1ns it expires before the +// request reaches the broker. func TestClient_AcquireToken_ClientTimeout(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(1 * time.Second) - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(map[string]string{"token": "late-token"}) + <-r.Context().Done() })) defer srv.Close() client := &Client{ - httpClient: &http.Client{Timeout: 100 * time.Millisecond}, + httpClient: &http.Client{Timeout: time.Nanosecond}, } _, err := client.AcquireToken(context.Background(), srv.URL, "user-token", "https://api.github.com", "", "") @@ -70,28 +70,51 @@ func TestClient_AcquireToken_ContextCancellationMidRequest(t *testing.T) { } func TestClient_AcquireToken_LongPolling(t *testing.T) { - const brokerDelay = 200 * time.Millisecond + reached := make(chan struct{}) + release := make(chan struct{}) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(brokerDelay) + close(reached) + <-release w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(map[string]string{"token": "delayed-token"}) })) defer srv.Close() + unblock := sync.OnceFunc(func() { close(release) }) + defer unblock() client := NewClient() - start := time.Now() - token, err := client.AcquireToken(context.Background(), srv.URL, "user-token", "https://api.github.com", "", "") - duration := time.Since(start) + if got := client.httpClient.Timeout; got <= 300*time.Second { + t.Fatalf("client timeout = %v, want longer than the broker's 300s long poll", got) + } + type result struct { + token string + err error + } + done := make(chan result, 1) + go func() { + token, err := client.AcquireToken(context.Background(), srv.URL, "user-token", "https://api.github.com", "", "") + done <- result{token, err} + }() - if err != nil { - t.Fatalf("unexpected error: %v", err) + // The client waits on the broker for as long as the broker holds the request. + select { + case <-reached: + case r := <-done: + t.Fatalf("AcquireToken returned before the broker had the request: %q, %v", r.token, r.err) + } + select { + case r := <-done: + t.Fatalf("AcquireToken returned before the broker answered: %q, %v", r.token, r.err) + default: } - if token != "delayed-token" { - t.Errorf("token = %q, want delayed-token", token) + unblock() + r := <-done + if r.err != nil { + t.Fatalf("unexpected error: %v", r.err) } - if duration < brokerDelay { - t.Errorf("request completed too quickly: %v (expected >= %v)", duration, brokerDelay) + if r.token != "delayed-token" { + t.Errorf("token = %q, want delayed-token", r.token) } } diff --git a/core/plugins/tokenexchangeaiac/plugin.go b/core/plugins/tokenexchangeaiac/plugin.go index fcb0fd386..c0140b5ea 100644 --- a/core/plugins/tokenexchangeaiac/plugin.go +++ b/core/plugins/tokenexchangeaiac/plugin.go @@ -157,6 +157,8 @@ type TokenExchangeAIAC struct { unknown *unknownCache creds atomic.Pointer[credentials] bgCancel atomic.Pointer[context.CancelFunc] + // bgDone is closed when the credential wait Init starts returns. + bgDone chan struct{} // now is the clock for both caches; tests replace it. now func() time.Time @@ -276,7 +278,12 @@ func (p *TokenExchangeAIAC) Init(_ context.Context) error { } ctx, cancel := context.WithCancel(context.Background()) p.bgCancel.Store(&cancel) - go p.pollCredentials(ctx) + done := make(chan struct{}) + p.bgDone = done + go func() { + defer close(done) + p.pollCredentials(ctx) + }() return nil } diff --git a/core/plugins/tokenexchangeaiac/plugin_test.go b/core/plugins/tokenexchangeaiac/plugin_test.go index ea66a6712..0f64161d8 100644 --- a/core/plugins/tokenexchangeaiac/plugin_test.go +++ b/core/plugins/tokenexchangeaiac/plugin_test.go @@ -783,12 +783,9 @@ func TestInitPicksUpCredentialFilesThatArriveLater(t *testing.T) { if err := os.WriteFile(dir+"/secret", []byte("s3cret\n"), 0o600); err != nil { t.Fatal(err) } - deadline := time.Now().Add(10 * time.Second) - for !p.Ready() { - if time.Now().After(deadline) { - t.Fatal("credentials written to disk were never loaded") - } - time.Sleep(50 * time.Millisecond) + <-p.bgDone + if !p.Ready() { + t.Fatal("credentials written to disk were not loaded") } pctx := outbound("github-tool", "Bearer user-token") if a := p.OnRequest(context.Background(), pctx); a.Type != pipeline.Continue {