From 2611ab0f7ccc1558187f3ebe8b06a42eccc34474 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Thu, 8 Oct 2026 18:10:58 +0200 Subject: [PATCH] test(eval): create fake runtimes before parallel execution Write fake container runtime scripts before t.Parallel and symlink them from each subtest's temp dir, so forked children don't inherit a writable executable fd and hit ETXTBSY on Linux. Assisted-By: cagent --- pkg/evaluation/eval_judge_test.go | 4 +++- pkg/evaluation/eval_test.go | 20 ++++++++++---------- pkg/evaluation/flavors_test.go | 4 +++- pkg/evaluation/judge_container_test.go | 18 ++++++++++-------- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/pkg/evaluation/eval_judge_test.go b/pkg/evaluation/eval_judge_test.go index 7a1c25951..0e0e9407d 100644 --- a/pkg/evaluation/eval_judge_test.go +++ b/pkg/evaluation/eval_judge_test.go @@ -24,6 +24,8 @@ func TestEvaluateWithEvaluatorJudge(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("the fake container runtime is a POSIX shell script") } + sharedRuntime := filepath.Join(t.TempDir(), "runtime") + writeFakeContainerRuntime(t, sharedRuntime, `{"type":"agent_choice","content":"hello"}`) t.Parallel() for _, tt := range []struct { @@ -60,7 +62,7 @@ func TestEvaluateWithEvaluatorJudge(t *testing.T) { require.NoError(t, os.Mkdir(evalsDir, 0o700)) require.NoError(t, os.WriteFile(filepath.Join(evalsDir, "session.json"), []byte(`{"evals":{`+tt.criteria+`}}`), 0o600)) fakeRuntime := filepath.Join(dir, "runtime") - writeFakeContainerRuntime(t, fakeRuntime, filepath.Join(dir, "args"), `{"type":"agent_choice","content":"hello"}`) + require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime)) run, err := Evaluate(t.Context(), &bytes.Buffer{}, &bytes.Buffer{}, false, "test", &config.RuntimeConfig{EnvProviderOverride: environment.NewMapEnvProvider(map[string]string{"TYPESAFE_API_KEY": "secret"})}, Config{AgentFilename: agentPath, EvalsDir: evalsDir, JudgeType: JudgeTypeEvaluator, JudgeModel: "relevance", Concurrency: 1, ContainerRuntime: fakeRuntime}) diff --git a/pkg/evaluation/eval_test.go b/pkg/evaluation/eval_test.go index f232b90b9..23f6d356e 100644 --- a/pkg/evaluation/eval_test.go +++ b/pkg/evaluation/eval_test.go @@ -77,6 +77,8 @@ func TestEvaluateSkipsUnusedJudge(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("the fake container runtime executable is a POSIX shell script") } + sharedRuntime := filepath.Join(t.TempDir(), "runtime") + writeFakeContainerRuntime(t, sharedRuntime, `{"type":"agent_choice","content":"ok"}`) t.Parallel() for _, tt := range []struct { @@ -113,7 +115,7 @@ func TestEvaluateSkipsUnusedJudge(t *testing.T) { } fakeRuntime := filepath.Join(tmpDir, "fake-runtime") - writeFakeContainerRuntime(t, fakeRuntime, filepath.Join(tmpDir, "args"), `{"type":"agent_choice","content":"ok"}`) + require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime)) runConfig := &config.RuntimeConfig{ EnvProviderForTests: environment.NewNoEnvProvider(), @@ -1226,12 +1228,10 @@ func TestContainerRuntimeOrDefault(t *testing.T) { assert.Equal(t, "podman", custom.containerRuntimeOrDefault()) } -// writeFakeContainerRuntime writes a POSIX shell script standing in for a -// Docker-compatible container runtime CLI: it records its arguments to -// argsFile and prints output on stdout. No daemon is involved. -func writeFakeContainerRuntime(t *testing.T, path, argsFile, output string) { +// Write before t.Parallel so forked children cannot inherit a writable executable. +func writeFakeContainerRuntime(t *testing.T, path, output string) { t.Helper() - script := "#!/bin/sh\necho \"$@\" > \"" + argsFile + "\"\necho '" + output + "'\n" + script := "#!/bin/sh\necho \"$@\" > \"${0%/*}/args\"\necho '" + output + "'\n" require.NoError(t, os.WriteFile(path, []byte(script), 0o755)) } @@ -1242,12 +1242,12 @@ func TestRunDockerAgentInContainerUsesConfiguredRuntime(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("the fake container runtime executable is a POSIX shell script") } - t.Parallel() tmpDir := t.TempDir() argsFile := filepath.Join(tmpDir, "args") fakeRuntime := filepath.Join(tmpDir, "fake-podman") - writeFakeContainerRuntime(t, fakeRuntime, argsFile, `{"type":"agent_choice","content":"ok"}`) + writeFakeContainerRuntime(t, fakeRuntime, `{"type":"agent_choice","content":"ok"}`) + t.Parallel() runner := newRunner( config.NewFileSource(filepath.Join(tmpDir, "agent.yaml")), @@ -1275,12 +1275,12 @@ func TestBuildEvalImageUsesConfiguredRuntime(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("the fake container runtime executable is a POSIX shell script") } - t.Parallel() tmpDir := t.TempDir() argsFile := filepath.Join(tmpDir, "args") fakeRuntime := filepath.Join(tmpDir, "fake-podman") - writeFakeContainerRuntime(t, fakeRuntime, argsFile, "sha256:fake-image-id") + writeFakeContainerRuntime(t, fakeRuntime, "sha256:fake-image-id") + t.Parallel() evalsDir := filepath.Join(tmpDir, "evals") require.NoError(t, os.Mkdir(evalsDir, 0o755)) diff --git a/pkg/evaluation/flavors_test.go b/pkg/evaluation/flavors_test.go index d56b3cc2f..28d213646 100644 --- a/pkg/evaluation/flavors_test.go +++ b/pkg/evaluation/flavors_test.go @@ -18,6 +18,8 @@ func TestEvalContainerForwardsOrderedFlavors(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("the fake container runtime executable is a POSIX shell script") } + sharedRuntime := filepath.Join(t.TempDir(), "runtime") + require.NoError(t, os.WriteFile(sharedRuntime, []byte("#!/bin/sh\nprintf '%s\\n' \"$@\" > \"${0%/*}/args\"\necho '{\"type\":\"agent_choice\",\"content\":\"ok\"}'\n"), 0o755)) t.Parallel() for _, setup := range []string{"", "echo setup"} { @@ -26,7 +28,7 @@ func TestEvalContainerForwardsOrderedFlavors(t *testing.T) { dir := t.TempDir() argsFile := filepath.Join(dir, "args") fake := filepath.Join(dir, "runtime") - require.NoError(t, os.WriteFile(fake, []byte("#!/bin/sh\nprintf '%s\\n' \"$@\" > \""+argsFile+"\"\necho '{\"type\":\"agent_choice\",\"content\":\"ok\"}'\n"), 0o755)) + require.NoError(t, os.Symlink(sharedRuntime, fake)) runConfig := &config.RuntimeConfig{ Config: config.Config{Flavors: []string{"baseline", "fixtures,with spaces"}}, EnvProviderForTests: environment.NewNoEnvProvider(), diff --git a/pkg/evaluation/judge_container_test.go b/pkg/evaluation/judge_container_test.go index bd3a8aa0b..1b67731c5 100644 --- a/pkg/evaluation/judge_container_test.go +++ b/pkg/evaluation/judge_container_test.go @@ -108,14 +108,8 @@ func TestContainerMountsJudgeProviderWithoutCredentials(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("the fake container runtime is a POSIX shell script") } - t.Parallel() - - for _, setup := range []string{"", "echo setup"} { - t.Run(setup, func(t *testing.T) { - t.Parallel() - dir := t.TempDir() - fakeRuntime := filepath.Join(dir, "runtime") - require.NoError(t, os.WriteFile(fakeRuntime, []byte(`#!/bin/sh + sharedRuntime := filepath.Join(t.TempDir(), "runtime") + require.NoError(t, os.WriteFile(sharedRuntime, []byte(`#!/bin/sh printf '%s\n' "$@" > "${0%/*}/args" for arg in "$@"; do case "$arg" in @@ -129,6 +123,14 @@ done printf '%s' "${HOST_JUDGE_KEY-}" > "${0%/*}/host-key" echo '{"type":"agent_choice","content":"hello"}' `), 0o755)) + t.Parallel() + + for _, setup := range []string{"", "echo setup"} { + t.Run(setup, func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + fakeRuntime := filepath.Join(dir, "runtime") + require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime)) runner := newRunner(config.NewFileSource(filepath.Join(dir, "agent.yaml")), &config.RuntimeConfig{ Config: config.Config{Providers: map[string]latest.ProviderConfig{"corporate": {Provider: "typesafe", TokenKey: "HOST_JUDGE_KEY"}}}, EnvProviderOverride: environment.NewMapEnvProvider(map[string]string{"HOST_JUDGE_KEY": "secret-value"}),