From 93094402cf1016793138630ee3503037a5dddad5 Mon Sep 17 00:00:00 2001 From: Evaline Ju <69598118+evaline-ju@users.noreply.github.com> Date: Thu, 19 Nov 2026 23:26:42 -0700 Subject: [PATCH] :white_check_mark: Wait on events in agentop pkg tests Signed-off-by: Evaline Ju <69598118+evaline-ju@users.noreply.github.com> --- cmd/agentop/apiclient/client_test.go | 60 +++++++------ cmd/agentop/apiclient/search.go | 2 +- cmd/agentop/apiclient/search_test.go | 20 +++++ cmd/agentop/cmd_doctor_checks_test.go | 6 +- cmd/agentop/cmd_exec_test.go | 98 ++++++++++++---------- cmd/agentop/cmd_import_test.go | 6 +- cmd/agentop/cmd_observe_test.go | 23 ++--- cmd/agentop/cmd_script_test.go | 4 - cmd/agentop/cmd_service_bootout_test.go | 23 ++++- cmd/agentop/cmd_service_goos_test.go | 6 +- cmd/agentop/cmd_service_platform.go | 6 +- cmd/agentop/cmd_service_restricted_test.go | 9 +- cmd/agentop/cmd_service_test.go | 51 ++++++++--- cmd/agentop/cmd_setup.go | 7 +- cmd/agentop/cmd_setup_test.go | 22 +++-- cmd/agentop/cmd_uninstall_test.go | 37 ++++---- cmd/agentop/codex_daemon_test.go | 8 +- cmd/agentop/local_endpoint.go | 11 ++- cmd/agentop/local_endpoint_test.go | 49 +++++++---- cmd/agentop/server_probe_test.go | 6 +- cmd/agentop/setup_e2e_test.go | 40 +++++---- cmd/agentop/setup_prior_test.go | 44 +++++----- cmd/agentop/setup_probe_test.go | 5 -- cmd/agentop/setup_step_service_test.go | 49 ++++++----- cmd/agentop/update_provenance_test.go | 17 ++-- 25 files changed, 353 insertions(+), 256 deletions(-) diff --git a/cmd/agentop/apiclient/client_test.go b/cmd/agentop/apiclient/client_test.go index ccd109b2a..ffff84f73 100644 --- a/cmd/agentop/apiclient/client_test.go +++ b/cmd/agentop/apiclient/client_test.go @@ -667,6 +667,41 @@ func TestGetJSON_ServerThinkTimeInsideTheCallersBudgetSucceeds(t *testing.T) { // lasts until the client gives up. The error has to come from the body read: a default that // ends at the headers fails a call only before they arrive. func TestGetBody_ADeadlinelessCallerIsBoundedMidBody(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(stallMidBody(`{"id":"s1","events":[`))) + defer ts.Close() + + c := New(ts.URL) + given := expireStreamDeadlineOnHeaders(t, c) + // No deadline, exactly as the TUI's root context has none. + _, err := c.GetSessionPage(context.Background(), "s1", 0, 10) + if err == nil { + t.Fatal("GetSessionPage returned no error against a server that stalled mid-body") + } + if !strings.Contains(err.Error(), ": decode: ") { + t.Errorf("failed with %v, want a failed body read: the call ended before the body", err) + } + if *given != streamDefaultTimeout { + t.Errorf("the call was bounded by %v, want streamDefaultTimeout (%v)", *given, streamDefaultTimeout) + } +} + +// stallMidBody sends the headers and the start of a body, then nothing until the client gives up. +func stallMidBody(start string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + // Headers are out, so the header bound is satisfied and no longer applies. + w.(http.Flusher).Flush() + _, _ = w.Write([]byte(start)) + w.(http.Flusher).Flush() + <-r.Context().Done() + } +} + +// expireStreamDeadlineOnHeaders makes streamDeadline end c's call as soon as its response +// headers arrive, until the test ends. The returned duration is the bound the call was given. +func expireStreamDeadlineOnHeaders(t *testing.T, c *Client) *time.Duration { + t.Helper() var given time.Duration var expire context.CancelFunc streamDeadline = func(parent context.Context, d time.Duration) (context.Context, context.CancelFunc) { @@ -676,35 +711,12 @@ func TestGetBody_ADeadlinelessCallerIsBoundedMidBody(t *testing.T) { return ctx, cancel } t.Cleanup(func() { streamDeadline = context.WithTimeout }) - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - // Headers are out, so the header bound is satisfied and no longer applies. - w.(http.Flusher).Flush() - _, _ = w.Write([]byte(`{"id":"s1","events":[`)) - w.(http.Flusher).Flush() - <-r.Context().Done() // and then nothing, until the client gives up - })) - defer ts.Close() - - c := New(ts.URL) c.http.Transport = expireAfterHeaders{c.http.Transport, func() { if expire != nil { expire() } }} - // No deadline, exactly as the TUI's root context has none. - _, err := c.GetSessionPage(context.Background(), "s1", 0, 10) - if err == nil { - t.Fatal("GetSessionPage returned no error against a server that stalled mid-body") - } - if !strings.Contains(err.Error(), ": decode: ") { - t.Errorf("failed with %v, want a failed body read: the call ended before the body", err) - } - if given != streamDefaultTimeout { - t.Errorf("the call was bounded by %v, want streamDefaultTimeout (%v)", given, streamDefaultTimeout) - } + return &given } // expireAfterHeaders runs expire once a response's headers have arrived. diff --git a/cmd/agentop/apiclient/search.go b/cmd/agentop/apiclient/search.go index d363bdcd6..cfb933ee2 100644 --- a/cmd/agentop/apiclient/search.go +++ b/cmd/agentop/apiclient/search.go @@ -31,7 +31,7 @@ const maxSearchBody = 16 << 20 func (c *Client) SearchSession(ctx context.Context, id, q string) (SearchResult, error) { if _, ok := ctx.Deadline(); !ok { var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, streamDefaultTimeout) + ctx, cancel = streamDeadline(ctx, streamDefaultTimeout) defer cancel() } path := fmt.Sprintf("/v1/sessions/%s/search?q=%s", url.PathEscape(id), url.QueryEscape(q)) diff --git a/cmd/agentop/apiclient/search_test.go b/cmd/agentop/apiclient/search_test.go index 19ce1b0f3..8bc0b0748 100644 --- a/cmd/agentop/apiclient/search_test.go +++ b/cmd/agentop/apiclient/search_test.go @@ -5,6 +5,7 @@ import ( "errors" "net/http" "net/http/httptest" + "net/url" "slices" "testing" ) @@ -52,3 +53,22 @@ func TestSearchSession_TellsTheTwo404sApart(t *testing.T) { } } } + +func TestSearchSession_ADeadlinelessCallerIsBoundedMidBody(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(stallMidBody(`{"seqs":[`))) + defer ts.Close() + + c := New(ts.URL) + given := expireStreamDeadlineOnHeaders(t, c) + _, err := c.SearchSession(context.Background(), "s", "q") + if err == nil { + t.Fatal("SearchSession returned no error against a server that stalled mid-body") + } + var urlErr *url.Error + if errors.As(err, &urlErr) || !errors.Is(err, context.Canceled) { + t.Errorf("failed with %v, want a canceled body read: the call ended before the body", err) + } + if *given != streamDefaultTimeout { + t.Errorf("the call was bounded by %v, want streamDefaultTimeout (%v)", *given, streamDefaultTimeout) + } +} diff --git a/cmd/agentop/cmd_doctor_checks_test.go b/cmd/agentop/cmd_doctor_checks_test.go index 6ad2c1558..dbe8fea03 100644 --- a/cmd/agentop/cmd_doctor_checks_test.go +++ b/cmd/agentop/cmd_doctor_checks_test.go @@ -8,7 +8,6 @@ import ( "slices" "strconv" "strings" - "syscall" "testing" "time" @@ -224,10 +223,7 @@ func TestDoctorChecksABackgroundInstallWhoseProxyIsGone(t *testing.T) { if pid = readPIDFile(pidFile); pid == 0 { t.Fatal("setup --no-service left no proxy.pid") } - _ = syscall.Kill(pid, syscall.SIGKILL) - for i := 0; i < 50 && alive(pid); i++ { - time.Sleep(100 * time.Millisecond) - } + killAndReap(pid) if alive(pid) { t.Fatalf("fixture: pid %d is still alive", pid) } diff --git a/cmd/agentop/cmd_exec_test.go b/cmd/agentop/cmd_exec_test.go index aecb4cb27..e75401e20 100644 --- a/cmd/agentop/cmd_exec_test.go +++ b/cmd/agentop/cmd_exec_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "encoding/json" + "fmt" "github.com/rossoctl/cortex/core/config" "github.com/rossoctl/cortex/core/redact" "github.com/rossoctl/cortex/core/tlsbridge" @@ -15,9 +16,9 @@ import ( "slices" "sort" "strings" + "sync" "syscall" "testing" - "time" ) // execCfg writes a Cortex config and returns its path plus the CA path it names. @@ -579,30 +580,20 @@ func TestRunExec_RelaysSIGTERM(t *testing.T) { } cfgPath, _ := execCfg(t) dir := t.TempDir() - started := filepath.Join(dir, "started") // The child records that it caught SIGTERM. Without relaying it never runs the // trap, and the file stays absent. caught := filepath.Join(dir, "caught") - script := "trap 'touch " + caught + "; exit 0' TERM; touch " + started + + script := "trap 'touch " + caught + "; exit 0' TERM; echo started" + "; i=0; while [ $i -lt 100 ]; do sleep 0.1; i=$((i+1)); done" - var stdout, stderr bytes.Buffer + stdout, stderr := newStartedWriter(), &bytes.Buffer{} done := make(chan int, 1) go func() { - done <- runExec([]string{"--cortex-stats-url", execStats(t, cfgPath), "--", "/bin/sh", "-c", script}, &stdout, &stderr) + done <- runExec([]string{"--cortex-stats-url", execStats(t, cfgPath), "--", "/bin/sh", "-c", script}, stdout, stderr) }() - // Wait for the child to install its trap. - deadline := time.Now().Add(10 * time.Second) - for { - if _, err := os.Stat(started); err == nil { - break - } - if time.Now().After(deadline) { - t.Fatal("child never started") - } - time.Sleep(20 * time.Millisecond) - } + // The child prints once its trap is installed. + <-stdout.started // Signal THIS process, the way `timeout` signals agentop. runExec's relay must // pass it to the child. @@ -610,16 +601,39 @@ func TestRunExec_RelaysSIGTERM(t *testing.T) { t.Fatal(err) } - select { - case <-done: - case <-time.After(20 * time.Second): - t.Fatal("runExec did not return after SIGTERM; the child was not signalled") - } + <-done if _, err := os.Stat(caught); err != nil { t.Error("the child never received SIGTERM, so it would have been orphaned") } } +// startedWriter is a child's stdout that closes started once the child has +// printed its first line. +type startedWriter struct { + mu sync.Mutex + b bytes.Buffer + once sync.Once + started chan struct{} +} + +func newStartedWriter() *startedWriter { return &startedWriter{started: make(chan struct{})} } + +func (w *startedWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + n, err := w.b.Write(p) + if strings.Contains(w.b.String(), "\n") { + w.once.Do(func() { close(w.started) }) + } + return n, err +} + +func (w *startedWriter) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.b.String() +} + // TestRunExec_PreservesArgv0AsTyped. exec.Command sets Args[0] to the resolved // absolute path; a shell passes the name as typed. Multi-call binaries dispatch // on argv[0], and tools echo it in their usage and errors. @@ -1027,29 +1041,23 @@ func TestRunExec_SurvivesSIGINT(t *testing.T) { t.Skip("no POSIX signals") } cfgPath, _ := execCfg(t) - dir := t.TempDir() - started := filepath.Join(dir, "started") - // The child exits on its own shortly after; the point is that runExec is still - // there to report its status rather than having been killed by the SIGINT. - script := "trap 'true' INT; touch " + started + - "; i=0; while [ $i -lt 40 ]; do sleep 0.05; i=$((i+1)); done; exit 3" + // The child exits 3 on a SIGTERM the test sends it after the SIGINT; the point + // is that runExec is still there to report that status rather than having been + // killed by the SIGINT. + script := "trap 'true' INT; trap 'exit 3' TERM; echo \"started $$\"" + + "; i=0; while [ $i -lt 100 ]; do sleep 0.1; i=$((i+1)); done" - var stdout, stderr bytes.Buffer + stdout, stderr := newStartedWriter(), &bytes.Buffer{} done := make(chan int, 1) go func() { done <- runExec([]string{"--cortex-stats-url", execStats(t, cfgPath), "--", - "/bin/sh", "-c", script}, &stdout, &stderr) + "/bin/sh", "-c", script}, stdout, stderr) }() - deadline := time.Now().Add(10 * time.Second) - for { - if _, err := os.Stat(started); err == nil { - break - } - if time.Now().After(deadline) { - t.Fatal("child never started") - } - time.Sleep(20 * time.Millisecond) + <-stdout.started + var child int + if _, err := fmt.Sscanf(stdout.String(), "started %d", &child); err != nil { + t.Fatalf("no child pid in %q: %v", stdout.String(), err) } // Signal THIS process, as a tty would. If runExec did not catch SIGINT, the @@ -1057,15 +1065,13 @@ func TestRunExec_SurvivesSIGINT(t *testing.T) { if err := syscall.Kill(syscall.Getpid(), syscall.SIGINT); err != nil { t.Fatal(err) } + if err := syscall.Kill(child, syscall.SIGTERM); err != nil { + t.Fatal(err) + } - select { - case code := <-done: - // The child's own exit status, proving agentop stayed alive to Wait on it. - if code != 3 { - t.Errorf("exit = %d, want 3 (the child's); agentop did not survive the SIGINT", code) - } - case <-time.After(20 * time.Second): - t.Fatal("runExec never returned after SIGINT") + // The child's own exit status, proving agentop stayed alive to Wait on it. + if code := <-done; code != 3 { + t.Errorf("exit = %d, want 3 (the child's); agentop did not survive the SIGINT", code) } } diff --git a/cmd/agentop/cmd_import_test.go b/cmd/agentop/cmd_import_test.go index f838bd3ad..b97cf51bb 100644 --- a/cmd/agentop/cmd_import_test.go +++ b/cmd/agentop/cmd_import_test.go @@ -110,11 +110,13 @@ func TestRunImport_SinceForms(t *testing.T) { } got = importer.Request{} + before := time.Now() if code := runImport([]string{"--endpoint", srv.URL, "--since", "30d"}, &out, &errOut); code != 0 { t.Fatalf("exit = %d; stderr = %s", code, errOut.String()) } - if d := time.Since(got.Since) - 720*time.Hour; d < 0 || d > time.Second { - t.Errorf("since 30d = %v, want now - 720h within a second (off by %v)", got.Since, d) + after := time.Now() + if got.Since.Before(before.Add(-720*time.Hour)) || got.Since.After(after.Add(-720*time.Hour)) { + t.Errorf("since 30d = %v, want 720h before a time in [%v, %v]", got.Since, before, after) } for _, bad := range []string{"bad", "0d", "-3d", "2026-13-01"} { diff --git a/cmd/agentop/cmd_observe_test.go b/cmd/agentop/cmd_observe_test.go index 58d081f32..2d9e0f0b4 100644 --- a/cmd/agentop/cmd_observe_test.go +++ b/cmd/agentop/cmd_observe_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "context" "flag" "fmt" "io" @@ -10,8 +9,8 @@ import ( "os/exec" "path/filepath" "strings" + "syscall" "testing" - "time" "github.com/rossoctl/cortex/cmd/agentop/tui" ) @@ -508,15 +507,6 @@ const ( // an env var, is what it means, and cannot appear in a flag or path anyone would type. const argSep = "\x1f" -// childTimeout bounds a re-exec'd child. -// -// Needed because not every argv exits on its own. `observe --help` does, through -// flag.ExitOnError, which is the only reason CombinedOutput did not hang before — but an argv -// that reaches tui.Run leaves the child blocking on TTY acquisition, and an unbounded -// CombinedOutput would then hang the whole package to its test timeout with no clue why. -// Generous enough that a slow machine does not flake, short enough to fail fast. -const childTimeout = 30 * time.Second - // runAgentopChild re-execs this test binary as agentop with the given argv, returning its combined // output. // @@ -528,17 +518,16 @@ const childTimeout = 30 * time.Second // inherits through os.Environ() below. An extraEnv parameter existed and every caller passed nil. func runAgentopChild(t *testing.T, args ...string) (string, error) { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), childTimeout) - defer cancel() - cmd := exec.CommandContext(ctx, os.Args[0]) + cmd := exec.Command(os.Args[0]) cmd.Env = append(os.Environ(), childSentinel+"=1", childArgs+"="+strings.Join(args, argSep), ) + // Not every argv exits on its own: one that reaches tui.Run opens the terminal. In + // a session of its own the child has no controlling terminal, so that open fails + // and the child exits. + cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} out, err := cmd.CombinedOutput() - if ctx.Err() != nil { - t.Fatalf("child did not exit within %s; argv=%q output:\n%s", childTimeout, args, out) - } return string(out), err } diff --git a/cmd/agentop/cmd_script_test.go b/cmd/agentop/cmd_script_test.go index 1412c2f5a..e754e5c71 100644 --- a/cmd/agentop/cmd_script_test.go +++ b/cmd/agentop/cmd_script_test.go @@ -570,14 +570,10 @@ func TestScript_AnUpdateReportsARefusalBeforeTheFirstPoll(t *testing.T) { refused := func() bool { got, _ := os.ReadFile(script); return !bytes.Equal(got, old) } f.refusedNow.Store(&refused) - start := time.Now() code, _, errOut := runScriptCmd(t, "install", "moca-sandbox", "--config", cfg) if code != 1 || !strings.Contains(errOut, "refused by the test") { t.Fatalf("exit %d: %s", code, errOut) } - if took := time.Since(start); took > 10*time.Second { - t.Errorf("took %v; the refusal must not wait out the poll deadline", took) - } if got, _ := os.ReadFile(script); !bytes.Equal(got, old) { t.Errorf("the earlier script was not put back:\n%s", got) } diff --git a/cmd/agentop/cmd_service_bootout_test.go b/cmd/agentop/cmd_service_bootout_test.go index c80de901e..8ba746185 100644 --- a/cmd/agentop/cmd_service_bootout_test.go +++ b/cmd/agentop/cmd_service_bootout_test.go @@ -159,10 +159,27 @@ func TestWaitGone_Progress(t *testing.T) { } }) + t.Run("a teardown inside the grace stays silent", func(t *testing.T) { + saved := waitGoneAnnounceAfter + waitGoneAnnounceAfter = time.Hour + t.Cleanup(func() { waitGoneAnnounceAfter = saved }) + var out bytes.Buffer + var asks int + if !waitGone(time.Hour, &out, func() bool { asks++; return asks > 1 }) { + t.Fatal("gave up on a label that did go away") + } + if out.Len() != 0 { + t.Errorf("announced a wait shorter than its grace: %q", out.String()) + } + }) + t.Run("a slow teardown announces, then confirms", func(t *testing.T) { + saved := waitGoneAnnounceAfter + waitGoneAnnounceAfter = -1 + t.Cleanup(func() { waitGoneAnnounceAfter = saved }) var out bytes.Buffer - start := time.Now() - ok := waitGone(10*time.Second, &out, func() bool { return time.Since(start) > 1500*time.Millisecond }) + var asks int + ok := waitGone(time.Hour, &out, func() bool { asks++; return asks > 1 }) if !ok { t.Fatal("gave up on a label that did go away") } @@ -177,7 +194,7 @@ func TestWaitGone_Progress(t *testing.T) { t.Run("a label that never goes away times out", func(t *testing.T) { var out bytes.Buffer - if waitGone(300*time.Millisecond, &out, func() bool { return false }) { + if waitGone(0, &out, func() bool { return false }) { t.Error("claimed a still-present label was gone") } }) diff --git a/cmd/agentop/cmd_service_goos_test.go b/cmd/agentop/cmd_service_goos_test.go index ab7c132ce..94ecaa537 100644 --- a/cmd/agentop/cmd_service_goos_test.go +++ b/cmd/agentop/cmd_service_goos_test.go @@ -159,10 +159,10 @@ func TestServiceIsCurrent_BothPlatforms(t *testing.T) { t.Run(tc.goos+": a job the supervisor does not report running is not current", func(t *testing.T) { // supervisorRunning's darwin branch POLLS for serviceReadyTimeout before it // will call a job not-running — transient post-kickstart states must not be - // read as a verdict. Nothing here is transient, so shorten the wait rather - // than spend the real 15s on a stub that answers the same way every time. + // read as a verdict. Nothing here is transient, so a window that has ended + // takes the stub's first answer. saved := serviceReadyTimeout - serviceReadyTimeout = 200 * time.Millisecond + serviceReadyTimeout = time.Nanosecond t.Cleanup(func() { serviceReadyTimeout = saved }) p := currentServiceFixture(t, tc.goos) diff --git a/cmd/agentop/cmd_service_platform.go b/cmd/agentop/cmd_service_platform.go index 4540a0199..2b6e89951 100644 --- a/cmd/agentop/cmd_service_platform.go +++ b/cmd/agentop/cmd_service_platform.go @@ -1078,6 +1078,10 @@ func serviceManagerUsable(goos string) (bool, string) { return systemdUsable() } +// waitGoneAnnounceAfter is how long waitGone waits before it says it is waiting. A +// var so a test can set when the line shows. +var waitGoneAnnounceAfter = time.Second + // waitGone polls gone() until it reports true, or d elapses. // // Split from waitBootedOutf so the progress behaviour is testable without launchd. The @@ -1094,7 +1098,7 @@ func waitGone(d time.Duration, progress io.Writer, gone func() bool) bool { } return true } - if !announced && progress != nil && time.Since(start) > time.Second { + if !announced && progress != nil && time.Since(start) > waitGoneAnnounceAfter { fmt.Fprintf(progress, " Waiting for the previous Cortex to stop (up to %s)...\n", d) announced = true } diff --git a/cmd/agentop/cmd_service_restricted_test.go b/cmd/agentop/cmd_service_restricted_test.go index 6537986cb..c4af65ed9 100644 --- a/cmd/agentop/cmd_service_restricted_test.go +++ b/cmd/agentop/cmd_service_restricted_test.go @@ -11,7 +11,6 @@ import ( "strconv" "strings" "testing" - "time" ) // fakeLaunchctl puts a launchctl on PATH that behaves like a restricted sandbox: it @@ -211,12 +210,8 @@ func TestReportSessionInterruption(t *testing.T) { t.Fatalf("cannot dial the listener we just opened: %v", err) } defer client.Close() //nolint:errcheck - select { - case c := <-accepted: - defer c.Close() //nolint:errcheck - case <-time.After(2 * time.Second): - t.Fatal("our own listener did not accept in 2s") - } + c := <-accepted + defer c.Close() //nolint:errcheck var out strings.Builder reportSessionInterruption(servicePaths{forwardAddr: ln.Addr().String()}, &out) diff --git a/cmd/agentop/cmd_service_test.go b/cmd/agentop/cmd_service_test.go index 16ae9dab9..775eeeef9 100644 --- a/cmd/agentop/cmd_service_test.go +++ b/cmd/agentop/cmd_service_test.go @@ -3,12 +3,16 @@ package main import ( "bytes" "encoding/xml" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" "runtime" "strconv" "strings" + "sync/atomic" + "syscall" "testing" "time" ) @@ -252,7 +256,7 @@ func startSleeperAt(t *testing.T, path string) int { if err := os.WriteFile(path, b, 0o755); err != nil { //nolint:gosec // an executable fixture t.Fatal(err) } - ready := filepath.Join(t.TempDir(), "ready") + ready, waitReady := sleeperReady(t) cmd := exec.Command(path) cmd.Env = append(os.Environ(), sleeperEnv+"=1", sleeperReadyEnv+"="+ready) if err := cmd.Start(); err != nil { @@ -262,15 +266,24 @@ func startSleeperAt(t *testing.T, path string) int { _ = cmd.Process.Kill() _ = cmd.Wait() }) - for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(10 * time.Millisecond) { - if _, err := os.Stat(ready); err == nil { - break - } - if time.Now().After(deadline) { - t.Fatalf("the fixture process at %s never started", path) + waitReady() + return cmd.Process.Pid +} + +// sleeperReady is the path a sleeper marks itself ready at, a FIFO, and the wait +// for that mark: the sleeper's write to it and this read meet. +func sleeperReady(t *testing.T) (string, func()) { + t.Helper() + ready := filepath.Join(t.TempDir(), "ready") + if err := syscall.Mkfifo(ready, 0o600); err != nil { + t.Fatal(err) + } + return ready, func() { + t.Helper() + if _, err := os.ReadFile(ready); err != nil { + t.Fatal(err) } } - return cmd.Process.Pid } // TestServiceStatus_ReportsTheV070ProxyInThePidfile: v0.7.0's installer ran @@ -496,13 +509,27 @@ func TestServiceStatus_NotAnsweringNamesTheAgentsThatFail(t *testing.T) { // TestWaitHealthy_TimesOut: install claims success only after the health endpoint // answers, because a supervisor reports "loaded" for a proxy that is crash-looping. +// Within its window it asks again, past a dropped connection and a 503, until the +// answer is 2xx. func TestWaitHealthy_TimesOut(t *testing.T) { - start := time.Now() - if waitHealthy("http://127.0.0.1:1/healthz", 1200*time.Millisecond) { + if waitHealthy("http://127.0.0.1:1/healthz", time.Nanosecond) { t.Error("reported healthy against a closed port") } - if elapsed := time.Since(start); elapsed < 900*time.Millisecond { - t.Errorf("returned after %s; it should have used the timeout", elapsed) + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + switch calls.Add(1) { + case 1: + c, _, err := http.NewResponseController(w).Hijack() + if err == nil { + _ = c.Close() + } + case 2: + w.WriteHeader(http.StatusServiceUnavailable) + } + })) + t.Cleanup(srv.Close) + if ok := waitHealthy(srv.URL+"/healthz", time.Hour); !ok || calls.Load() != 3 { + t.Errorf("waitHealthy = %v after %d asks, want true after 3", ok, calls.Load()) } } diff --git a/cmd/agentop/cmd_setup.go b/cmd/agentop/cmd_setup.go index 63903db1c..045ff44d3 100644 --- a/cmd/agentop/cmd_setup.go +++ b/cmd/agentop/cmd_setup.go @@ -69,6 +69,9 @@ var setupConfirm = func(w io.Writer) bool { return setupConfirmFrom(tty, w) } +// setupNow is the clock ready in is measured on. A var so a test can set the time. +var setupNow = time.Now + // setupAnimate reports whether the screen animates on w: on a terminal. A var for // the reason uninstallAnimate gives. var setupAnimate = isTerminal @@ -294,14 +297,14 @@ func runSetup(args []string, stdout, stderr io.Writer) int { // ready in is the time the changes took, from here: not the time spent reading // the list of them at the prompt. - started := time.Now() + started := setupNow() if !applySteps(env, ui, planned, sigs) { return 1 } for _, f := range env.onSuccess { f() } - printSetupEnding(env, ui, time.Since(started)) + printSetupEnding(env, ui, setupNow().Sub(started)) return 0 } diff --git a/cmd/agentop/cmd_setup_test.go b/cmd/agentop/cmd_setup_test.go index 3255f3372..bf85d2e6e 100644 --- a/cmd/agentop/cmd_setup_test.go +++ b/cmd/agentop/cmd_setup_test.go @@ -8,7 +8,6 @@ import ( "strings" "syscall" "testing" - "time" "github.com/rossoctl/cortex/cmd/agentop/checklist" ) @@ -160,24 +159,23 @@ func TestSetupSignalsCatchesInterruptAndTerminate(t *testing.T) { t.Fatal(err) } <-guard - select { - case got := <-sigs: - if got != s { - t.Errorf("sent %v, setupSignals' channel got %v", s, got) - } - case <-time.After(2 * time.Second): - t.Errorf("setupSignals' channel got no %v", s) + if got := <-sigs; got != s { + t.Errorf("sent %v, setupSignals' channel got %v", s, got) } } stop() - if err := syscall.Kill(os.Getpid(), syscall.SIGTERM); err != nil { - t.Fatal(err) + // os/signal hands each signal to every subscriber before it takes the next, so + // once guard has the second, any send of the first to sigs has happened. + for _, s := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} { + if err := syscall.Kill(os.Getpid(), s); err != nil { + t.Fatal(err) + } + <-guard } - <-guard select { case got := <-sigs: t.Errorf("a %v arrived after the stop func ran", got) - case <-time.After(200 * time.Millisecond): + default: } } diff --git a/cmd/agentop/cmd_uninstall_test.go b/cmd/agentop/cmd_uninstall_test.go index 548900c06..771471a19 100644 --- a/cmd/agentop/cmd_uninstall_test.go +++ b/cmd/agentop/cmd_uninstall_test.go @@ -11,6 +11,7 @@ import ( "os/exec" "path/filepath" "reflect" + "regexp" "runtime" "slices" "strconv" @@ -1172,18 +1173,20 @@ func TestUninstallRowsRunUnderASpinnerOnATerminal(t *testing.T) { var w lockedWriter ui := checklist.New(&w, true) applyRemovals(&setupEnv{home: "/h"}, ui, []removal{{label: "stopped", run: func(*checklist.Running) (string, []string, error) { - time.Sleep(250 * time.Millisecond) return "background Cortex stopped", nil, nil }}}, nil) ui.Close() out := w.String() - // The row's time ends at column 78: 42 columns of row, then the 4 of "0.Ns". - row := "\r\x1b[K ✓ stopped background Cortex stopped" + strings.Repeat(" ", 78-42-4) + "0." - for _, want := range []string{"\x1b[?25l", " stopped \r", row, "\x1b[?25h"} { + for _, want := range []string{"\x1b[?25l", " stopped \r", "\x1b[?25h"} { if !strings.Contains(out, want) { t.Errorf("animated output lacks %q: %q", want, out) } } + // The row's time ends at column 78: 42 columns of row, then the 4 of "N.Ns". + row := regexp.QuoteMeta("\r\x1b[K ✓ stopped background Cortex stopped"+strings.Repeat(" ", 78-42-4)) + `\d\.\ds` + if !regexp.MustCompile(row).MatchString(out) { + t.Errorf("animated output lacks the row with its time: %q", out) + } } // fakeUninstallSignals stands in for the signals uninstall catches, and records @@ -1264,19 +1267,12 @@ func TestUninstallASignalAtThePromptChangesNothing(t *testing.T) { } sigs <- os.Interrupt // A yes after the Ctrl-C, too late: uninstall has stopped waiting by then. - select { - case <-release: - case <-time.After(10 * time.Second): - } + <-release return true } t.Cleanup(func() { uninstallConfirm = saved }) before := homeFiles(t, sc.home) - start := time.Now() code, out := sc.uninstall(t) - if took := time.Since(start); took > 5*time.Second { - t.Errorf("uninstall took %s, waiting for the answer after the signal", took) - } wantLines(t, code, exitDeclined, out, " Continue? [y/N] \n Not changed.\n") if strings.Contains(out, "✓") { t.Errorf("a removal ran after the signal:\n%s", out) @@ -1296,9 +1292,20 @@ const uninstallSigintChild = "AGENTOP_TEST_UNINSTALL_SIGINT_CHILD" func TestUninstallARealSigintStillShowsTheCursor(t *testing.T) { if os.Getenv(uninstallSigintChild) == "1" { sc := newUninstallScene(t) - saved, savedAnimate := uninstallRemovalsHook, uninstallAnimate - t.Cleanup(func() { uninstallRemovalsHook, uninstallAnimate = saved, savedAnimate }) + saved, savedAnimate, savedSignals := uninstallRemovalsHook, uninstallAnimate, uninstallSignals + t.Cleanup(func() { uninstallRemovalsHook, uninstallAnimate, uninstallSignals = saved, savedAnimate, savedSignals }) uninstallAnimate = func(io.Writer) bool { return true } + // The real subscription, relayed so the removal knows when its signal is in. + arrived := make(chan struct{}) + uninstallSignals = func() (<-chan os.Signal, func()) { + c, stop := savedSignals() + relayed := make(chan os.Signal, 1) + go func() { + relayed <- <-c + close(arrived) + }() + return relayed, stop + } uninstallRemovalsHook = func([]removal) []removal { step := func(label string, fn func()) removal { return removal{label: label, item: checklist.Item{Verb: "remove", What: "the " + label}, @@ -1306,7 +1313,7 @@ func TestUninstallARealSigintStillShowsTheCursor(t *testing.T) { } return []removal{step("first", func() {}), step("second", func() { _ = syscall.Kill(os.Getpid(), syscall.SIGINT) - time.Sleep(300 * time.Millisecond) // for it to arrive while this removal runs + <-arrived }), step("third", func() {})} } code, out := sc.uninstall(t, "--yes") diff --git a/cmd/agentop/codex_daemon_test.go b/cmd/agentop/codex_daemon_test.go index b593cbdae..7b1ca5c61 100644 --- a/cmd/agentop/codex_daemon_test.go +++ b/cmd/agentop/codex_daemon_test.go @@ -13,7 +13,6 @@ import ( "strings" "syscall" "testing" - "time" "github.com/rossoctl/cortex/core/peerproc" ) @@ -734,7 +733,6 @@ func TestCodexStopUpdater_RealProcess(t *testing.T) { } else { writeCodexPIDStarted(t, home, codexUpdaterPIDRel, pid, psStarted(t, pid)) } - start := time.Now() if err := codexStopUpdater(home); err != nil { t.Fatalf("stopping a live updater: %v", err) } @@ -743,9 +741,6 @@ func TestCodexStopUpdater_RealProcess(t *testing.T) { default: t.Fatal("codexStopUpdater returned with the updater still running") } - if d := time.Since(start); d > 5*time.Second { - t.Errorf("stop took %s; SIGTERM should end sleep at once", d) - } if err := codexStopUpdater(home); err != nil { t.Errorf("stale pid file: %v", err) } @@ -799,10 +794,11 @@ func TestCodexStopUpdater_LeavesAReusedPIDAlone(t *testing.T) { if err := codexStopUpdater(home); err != nil { t.Errorf("a pid that is not the updater: %v", err) } + // A stop that signals waits for the exit before it returns. select { case <-exited: t.Fatal("signalled a process the pid file does not describe") - case <-time.After(300 * time.Millisecond): + default: } }) } diff --git a/cmd/agentop/local_endpoint.go b/cmd/agentop/local_endpoint.go index 1e35f0f92..fca28acbf 100644 --- a/cmd/agentop/local_endpoint.go +++ b/cmd/agentop/local_endpoint.go @@ -31,6 +31,11 @@ const localProbeTimeout = 400 * time.Millisecond // localProbeTimeout. A var so a test of a socket that never answers need not wait it out. var localStartupWait = 6 * time.Second +// localStartingNotice is how long waitForLocalSessionAPI waits for an answer before +// it says the Cortex is starting: past the dial's own bound, so a probe still +// waiting is a connected one. A var so a test can set when the notice shows. +var localStartingNotice = localProbeTimeout + 100*time.Millisecond + // localSessionEndpoint returns the session API URL of the Cortex installed on // this machine, or "" if there isn't one. // @@ -196,9 +201,9 @@ func waitForLocalSessionAPI(endpoint string, w io.Writer) bool { select { case up := <-done: return up - // Past the dial's own bound, so a probe still waiting is a connected one: a dial - // that was going to fail has failed by then, and must not be called "starting". - case <-time.After(localProbeTimeout + 100*time.Millisecond): + // A dial that was going to fail has failed by localStartingNotice, and must not + // be called "starting". + case <-time.After(localStartingNotice): fmt.Fprintf(w, "agentop: the Cortex at %s is starting; waiting for it…\n", endpoint) return <-done } diff --git a/cmd/agentop/local_endpoint_test.go b/cmd/agentop/local_endpoint_test.go index 588fe4938..0cde0ad25 100644 --- a/cmd/agentop/local_endpoint_test.go +++ b/cmd/agentop/local_endpoint_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" ) @@ -197,6 +198,9 @@ func TestLocalFallbackNamesABrokenConfig(t *testing.T) { // Cortex that is starting: worth waiting for, where a 400ms answer deadline sent agentop // to the cluster picker on every launch that met a restart. func TestLocalSessionAPIUp_WaitsForAProxyStillStarting(t *testing.T) { + saved := localStartupWait + localStartupWait = time.Hour + t.Cleanup(func() { localStartupWait = saved }) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -215,7 +219,7 @@ func TestLocalSessionAPIUp_WaitsForAProxyStillStarting(t *testing.T) { // The wait is bounded: something that accepts and never answers is given up on. func TestLocalSessionAPIUp_GivesUpOnOneThatNeverAnswers(t *testing.T) { saved := localStartupWait - localStartupWait = time.Second + localStartupWait = time.Nanosecond t.Cleanup(func() { localStartupWait = saved }) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -223,18 +227,17 @@ func TestLocalSessionAPIUp_GivesUpOnOneThatNeverAnswers(t *testing.T) { } defer ln.Close() - start := time.Now() if localSessionAPIUp("http://" + ln.Addr().String()) { t.Error("a socket nothing serves was reported up") } - if took := time.Since(start); took > 2*time.Second { - t.Errorf("took %s, want it bounded by localStartupWait", took) - } } // Nothing listening is answered at the connect, so the startup wait never applies to it: // a machine whose Cortex is stopped still reaches the picker at once. func TestLocalSessionAPIUp_RefusedIsImmediate(t *testing.T) { + saved := localStartupWait + localStartupWait = time.Hour + t.Cleanup(func() { localStartupWait = saved }) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -242,17 +245,30 @@ func TestLocalSessionAPIUp_RefusedIsImmediate(t *testing.T) { dead := "http://" + ln.Addr().String() _ = ln.Close() - start := time.Now() if localSessionAPIUp(dead) { t.Error("a closed port was reported up") } - if took := time.Since(start); took > localProbeTimeout+500*time.Millisecond { - t.Errorf("took %s on a refused connection, want about one connect", took) - } +} + +// writeHook is a writer that runs fn at its first write. +type writeHook struct { + strings.Builder + once sync.Once + fn func() +} + +func (w *writeHook) Write(p []byte) (int, error) { + n, err := w.Builder.Write(p) + w.once.Do(w.fn) + return n, err } // The wait is said out loud once it is noticeable, before the TUI takes the screen. +// The proxy here starts serving once the notice is out. func TestWaitForLocalSessionAPI_SaysWhyItIsWaiting(t *testing.T) { + notice, wait := localStartingNotice, localStartupWait + localStartupWait = time.Hour + t.Cleanup(func() { localStartingNotice, localStartupWait = notice, wait }) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -261,23 +277,24 @@ func TestWaitForLocalSessionAPI_SaysWhyItIsWaiting(t *testing.T) { _, _ = w.Write([]byte(`{"sessions":[]}`)) })} defer srv.Close() - time.AfterFunc(time.Second, func() { _ = srv.Serve(ln) }) endpoint := "http://" + ln.Addr().String() - var out strings.Builder - if !waitForLocalSessionAPI(endpoint, &out) { + localStartingNotice = 0 + out := &writeHook{fn: func() { go func() { _ = srv.Serve(ln) }() }} + if !waitForLocalSessionAPI(endpoint, out) { t.Fatal("reported down") } if !strings.Contains(out.String(), endpoint) || !strings.Contains(out.String(), "starting") { t.Errorf("no waiting notice naming %s: %q", endpoint, out.String()) } - out.Reset() - if !waitForLocalSessionAPI(endpoint, &out) { + localStartingNotice = time.Hour + var prompt strings.Builder + if !waitForLocalSessionAPI(endpoint, &prompt) { t.Fatal("reported down once serving") } - if out.Len() != 0 { - t.Errorf("a prompt answer still printed %q", out.String()) + if prompt.Len() != 0 { + t.Errorf("a prompt answer still printed %q", prompt.String()) } } diff --git a/cmd/agentop/server_probe_test.go b/cmd/agentop/server_probe_test.go index f268c9568..d241a10ff 100644 --- a/cmd/agentop/server_probe_test.go +++ b/cmd/agentop/server_probe_test.go @@ -430,16 +430,18 @@ func TestProbe_HungPathLeavesTimeForItsProtocol(t *testing.T) { } } +// The probe's context ends at the first request after /v1/messages served. func TestProbe_EndedContextKeepsWhatWasFound(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() srv, _ := fakeProbeServer(t, func(w http.ResponseWriter, p probeReq, r *http.Request) { if p.path == "/v1/messages" { io.WriteString(w, anthropicOK) return } + cancel() hang(r) }) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() api, _, err := probeAPI(ctx, srv.Client(), routerconfig.Server{URL: srv.URL, Key: "k"}, "m", "", false) want := map[string]string{"anthropic-messages": "/v1/messages"} if err != nil || !maps.Equal(api, want) { diff --git a/cmd/agentop/setup_e2e_test.go b/cmd/agentop/setup_e2e_test.go index d15a3a115..40e76b3e6 100644 --- a/cmd/agentop/setup_e2e_test.go +++ b/cmd/agentop/setup_e2e_test.go @@ -13,8 +13,8 @@ import ( "path/filepath" "regexp" "runtime" - "strconv" "strings" + "sync" "sync/atomic" "testing" "time" @@ -517,10 +517,7 @@ func TestSetupASignalBeforeApplyChangesNothing(t *testing.T) { } sigs <- os.Interrupt // A yes after the Ctrl-C, too late: setup has stopped waiting by then. - select { - case <-release: - case <-time.After(2 * time.Second): - } + <-release return true } t.Cleanup(func() { setupConfirm = saved }) @@ -729,25 +726,34 @@ func TestSetupDrawsNoLogoUnlessARunWorked(t *testing.T) { } // ready in counts the time the changes took, from the answer at the prompt: not the -// time spent reading the list of them before it. So ready in is at most the run's wall -// time less the 2.5s at the prompt. A fixed bound flaked: under shard load the apply -// alone can take more than 2.5s. +// time spent reading the list of them before it. The clock moves 1s at each read +// and an hour at the prompt, so ready in is 1.0s, and 3601.0s with the prompt in it. func TestSetupReadyInLeavesOutThePrompt(t *testing.T) { - const atThePrompt = 2500 * time.Millisecond sc := newSetupScene(t, ok200) animated(t) - saved := setupConfirm - setupConfirm = func(io.Writer) bool { time.Sleep(atThePrompt); return true } - t.Cleanup(func() { setupConfirm = saved }) - start := time.Now() + var mu sync.Mutex + clock := time.Unix(1000, 0) + savedNow, savedConfirm := setupNow, setupConfirm + setupNow = func() time.Time { + mu.Lock() + defer mu.Unlock() + now := clock + clock = clock.Add(time.Second) + return now + } + setupConfirm = func(io.Writer) bool { + mu.Lock() + defer mu.Unlock() + clock = clock.Add(time.Hour) + return true + } + t.Cleanup(func() { setupNow, setupConfirm = savedNow, savedConfirm }) code, out := sc.run(t, "--from", sc.stage) - wall := time.Since(start) m := regexp.MustCompile(`\n v9\.9\.9 ready in (\d+\.\d)s\n`).FindStringSubmatch(onScreen(out)) if code != 0 || m == nil { t.Fatalf("exit %d, no ready in:\n%s", code, onScreen(out)) } - // 0.05s is the rounding of ready in to one decimal. - if took, _ := strconv.ParseFloat(m[1], 64); took > (wall-atThePrompt).Seconds()+0.05 { - t.Errorf("ready in %ss counts the %v at the prompt (run took %v)", m[1], atThePrompt, wall) + if m[1] != "1.0" { + t.Errorf("ready in %ss, want 1.0s: the hour at the prompt is counted", m[1]) } } diff --git a/cmd/agentop/setup_prior_test.go b/cmd/agentop/setup_prior_test.go index 52c2bc9b5..7285665ac 100644 --- a/cmd/agentop/setup_prior_test.go +++ b/cmd/agentop/setup_prior_test.go @@ -12,7 +12,6 @@ import ( "strings" "syscall" "testing" - "time" "github.com/rossoctl/cortex/cmd/agentop/checklist" ) @@ -63,7 +62,7 @@ func detachedSleeperAt(t *testing.T, path string) int { if err := os.WriteFile(path, b, 0o755); err != nil { //nolint:gosec // an executable fixture t.Fatal(err) } - ready := filepath.Join(t.TempDir(), "ready") + ready, waitReady := sleeperReady(t) cmd := exec.Command("/bin/sh", "-c", `"$0" >/dev/null 2>&1 /dev/null 2>&1 '`+leaving+`'; exit 0; fi ;; + bootstrap) rm -f '`+leaving+`' '`+looked+`' ;; + print) + case "$2" in + */*/*) if [ -f '`+leaving+`' ]; then + if [ -f '`+looked+`' ]; then rm -f '`+loaded+`' '`+leaving+`' '`+looked+`'; else : > '`+looked+`'; fi + fi ;; + esac ;; esac exec '`+fake+`' "$@" `) @@ -632,13 +634,13 @@ func TestSetupRollbackWaitsOutLaunchdTeardown(t *testing.T) { if err := controlService(runtimeGOOS(), "stop", sp, io.Discard); err != nil { t.Fatal(err) } - for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(100 * time.Millisecond) { - if _, err := os.Stat(sc.loaded); err != nil { - break - } - if time.Now().After(deadline) { - t.Fatal("fixture: the stopped job never left") - } + // The stopped job leaves at the second look. + target := "gui/" + strconv.Itoa(os.Getuid()) + "/" + launchdLabel + for range 2 { + _, _ = labelGone(target) + } + if _, err := os.Stat(sc.loaded); err == nil { + t.Fatal("fixture: the stopped job never left") } prev, err := os.ReadFile(bin) if err != nil { diff --git a/cmd/agentop/setup_probe_test.go b/cmd/agentop/setup_probe_test.go index 6720c6ebf..67a6f1ff0 100644 --- a/cmd/agentop/setup_probe_test.go +++ b/cmd/agentop/setup_probe_test.go @@ -11,7 +11,6 @@ import ( "strconv" "strings" "testing" - "time" "golang.org/x/sys/unix" ) @@ -352,13 +351,9 @@ func TestStartUnsupervised(t *testing.T) { if err := os.WriteFile(quits, []byte("#!/bin/sh\necho boom >&2\nexit 1\n"), 0o755); err != nil { //nolint:gosec t.Fatal(err) } - began := time.Now() if _, _, err := startUnsupervised(quits, dir2, bad.URL+"/healthz"); err == nil { t.Error("a proxy that exits at once was reported started") } - if time.Since(began) > 5*time.Second { - t.Error("an immediate exit was not noticed promptly") - } if _, err := os.Stat(filepath.Join(dir2, "proxy.pid")); err == nil { t.Error("the pidfile of an exited proxy was left behind") } diff --git a/cmd/agentop/setup_step_service_test.go b/cmd/agentop/setup_step_service_test.go index 923c20121..54cedb36f 100644 --- a/cmd/agentop/setup_step_service_test.go +++ b/cmd/agentop/setup_step_service_test.go @@ -213,21 +213,26 @@ func TestServiceStepUnsupervisedRestoreNamesItsStartByHand(t *testing.T) { } } -// stopProcessOnCleanup kills *pid, if it is still alive when the test ends: -// startUnsupervised's own goroutine reaps it, so this waits for it to go. +// stopProcessOnCleanup kills *pid, if it is still alive when the test ends. func stopProcessOnCleanup(t *testing.T, pid *int) { t.Helper() t.Cleanup(func() { - if !alive(*pid) { - return - } - _ = syscall.Kill(*pid, syscall.SIGKILL) - for i := 0; i < 50 && alive(*pid); i++ { - time.Sleep(100 * time.Millisecond) + if alive(*pid) { + killAndReap(*pid) } }) } +// killAndReap kills pid and, when it is a child of this process, returns once it +// is reaped: by this wait, or by startUnsupervised's goroutine, which leaves this +// wait an ECHILD. +func killAndReap(pid int) { + _ = syscall.Kill(pid, syscall.SIGKILL) + if p, err := os.FindProcess(pid); err == nil { + _, _ = p.Wait() + } +} + func TestServiceStepRunsUnsupervisedInASandbox(t *testing.T) { fakeNoSupervisor(t) env := serviceEnv(t, ok200) @@ -939,7 +944,13 @@ func TestServiceStepBackgroundRestoreRefusesBesideALoadedUnhealthyJob(t *testing // $!, where installed. sh -c expands the ~ in each path, the redirections' too. func TestStartByHandRunsFromAShell(t *testing.T) { env := newTestSetupEnv(t) - writeExe(t, filepath.Join(env.binDir, "cortex"), "#!/bin/sh\necho \"started $*\"\nexec sleep 30\n") + // The fake cortex logs its argv, then marks itself started on a FIFO this test + // reads. + started := filepath.Join(t.TempDir(), "started") + if err := syscall.Mkfifo(started, 0o600); err != nil { + t.Fatal(err) + } + writeExe(t, filepath.Join(env.binDir, "cortex"), "#!/bin/sh\necho \"started $*\"\n: > '"+started+"'\nexec sleep 30\n") if err := os.MkdirAll(env.cortexDir, 0o700); err != nil { t.Fatal(err) } @@ -955,15 +966,13 @@ func TestStartByHandRunsFromAShell(t *testing.T) { if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("%s: %v %s", sh, err, out) } - var pid int - for deadline := time.Now().Add(5 * time.Second); pid == 0 && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { - pid = readPIDFile(pidFile) - } + // The line writes proxy.pid before its shell exits. + pid := readPIDFile(pidFile) stopProcessOnCleanup(t, &pid) - log, _ := os.ReadFile(filepath.Join(env.cortexDir, "proxy.log")) - for deadline := time.Now().Add(5 * time.Second); !strings.Contains(string(log), "started --local --supervise") && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { - log, _ = os.ReadFile(filepath.Join(env.cortexDir, "proxy.log")) + if _, err := os.ReadFile(started); err != nil { + t.Fatal(err) } + log, _ := os.ReadFile(filepath.Join(env.cortexDir, "proxy.log")) if !alive(pid) || !strings.Contains(string(log), "started --local --supervise") { t.Errorf("%s: proxy.pid %d alive=%v, proxy.log %q", sh, pid, alive(pid), log) } @@ -975,17 +984,13 @@ func TestStartByHandRunsFromAShell(t *testing.T) { // systemd's disable --now has returned by then, so linux waits for nothing. func TestWaitUnloadedNamesItsTimeout(t *testing.T) { saved := serviceBootoutTimeout - serviceBootoutTimeout = 500 * time.Millisecond + serviceBootoutTimeout = time.Nanosecond t.Cleanup(func() { serviceBootoutTimeout = saved }) installStub(t, "launchctl", "#!/bin/sh\necho 'state = running'\n") // the job is still there - began := time.Now() err := waitUnloaded("darwin") - if want := "the launchd user agent is still shutting down after 500ms"; err == nil || err.Error() != want { + if want := "the launchd user agent is still shutting down after 1ns"; err == nil || err.Error() != want { t.Errorf("waitUnloaded = %v, want %q", err, want) } - if took := time.Since(began); took > 5*time.Second { - t.Errorf("waited %s, not serviceBootoutTimeout", took) - } if err := waitUnloaded("linux"); err != nil { t.Errorf("linux waited: %v", err) } diff --git a/cmd/agentop/update_provenance_test.go b/cmd/agentop/update_provenance_test.go index 447f86595..01afcceea 100644 --- a/cmd/agentop/update_provenance_test.go +++ b/cmd/agentop/update_provenance_test.go @@ -230,17 +230,14 @@ func TestProvenanceTrustedMaterialTimesOut(t *testing.T) { t.Setenv("HOME", t.TempDir()) // an empty TUF cache, so the client must fetch oldMirror, oldTimeout := tufMirror, tufFetchTimeout t.Cleanup(func() { tufMirror, tufFetchTimeout = oldMirror, oldTimeout }) - tufMirror, tufFetchTimeout = srv.URL, 200*time.Millisecond + tufMirror, tufFetchTimeout = srv.URL, time.Nanosecond - done := make(chan error, 1) - go func() { _, err := provenanceTrustedMaterial(); done <- err }() - select { - case err := <-done: - if err == nil { - t.Fatal("trusted root loaded from a server that never answers") - } - case <-time.After(10 * time.Second): - t.Fatal("trusted root fetch did not time out") + _, err := provenanceTrustedMaterial() + if err == nil { + t.Fatal("trusted root loaded from a server that never answers") + } + if !strings.Contains(err.Error(), "Client.Timeout exceeded") { + t.Errorf("err = %v, want the fetch's timeout", err) } }