Skip to content

test(eval): avoid ETXTBSY in parallel fake-runtime tests #4556

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dgageot merged 1 commit into from
Oct 8, 2026
+26 −20
Merged

test(eval): avoid ETXTBSY in parallel fake-runtime tests #4556

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion pkg/evaluation/eval_judge_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ func TestEvaluateWithEvaluatorJudge(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("the fake container runtime is a POSIX shell script")
}
sharedRuntime := filepath.Join(t.TempDir(), "runtime")
writeFakeContainerRuntime(t, sharedRuntime, `{"type":"agent_choice","content":"hello"}`)
t.Parallel()

for _, tt := range []struct {
Expand Down Expand Up @@ -60,7 +62,7 @@ func TestEvaluateWithEvaluatorJudge(t *testing.T) {
require.NoError(t, os.Mkdir(evalsDir, 0o700))
require.NoError(t, os.WriteFile(filepath.Join(evalsDir, "session.json"), []byte(`{"evals":{`+tt.criteria+`}}`), 0o600))
fakeRuntime := filepath.Join(dir, "runtime")
writeFakeContainerRuntime(t, fakeRuntime, filepath.Join(dir, "args"), `{"type":"agent_choice","content":"hello"}`)
require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime))
run, err := Evaluate(t.Context(), &bytes.Buffer{}, &bytes.Buffer{}, false, "test",
&config.RuntimeConfig{EnvProviderOverride: environment.NewMapEnvProvider(map[string]string{"TYPESAFE_API_KEY": "secret"})},
Config{AgentFilename: agentPath, EvalsDir: evalsDir, JudgeType: JudgeTypeEvaluator, JudgeModel: "relevance", Concurrency: 1, ContainerRuntime: fakeRuntime})
Expand Down
20 changes: 10 additions & 10 deletions pkg/evaluation/eval_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ func TestEvaluateSkipsUnusedJudge(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("the fake container runtime executable is a POSIX shell script")
}
sharedRuntime := filepath.Join(t.TempDir(), "runtime")
writeFakeContainerRuntime(t, sharedRuntime, `{"type":"agent_choice","content":"ok"}`)
t.Parallel()

for _, tt := range []struct {
Expand Down Expand Up @@ -113,7 +115,7 @@ func TestEvaluateSkipsUnusedJudge(t *testing.T) {
}

fakeRuntime := filepath.Join(tmpDir, "fake-runtime")
writeFakeContainerRuntime(t, fakeRuntime, filepath.Join(tmpDir, "args"), `{"type":"agent_choice","content":"ok"}`)
require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime))

runConfig := &config.RuntimeConfig{
EnvProviderForTests: environment.NewNoEnvProvider(),
Expand Down Expand Up @@ -1226,12 +1228,10 @@ func TestContainerRuntimeOrDefault(t *testing.T) {
assert.Equal(t, "podman", custom.containerRuntimeOrDefault())
}

// writeFakeContainerRuntime writes a POSIX shell script standing in for a
// Docker-compatible container runtime CLI: it records its arguments to
// argsFile and prints output on stdout. No daemon is involved.
func writeFakeContainerRuntime(t *testing.T, path, argsFile, output string) {
// Write before t.Parallel so forked children cannot inherit a writable executable.
func writeFakeContainerRuntime(t *testing.T, path, output string) {
t.Helper()
script := "#!/bin/sh\necho \"$@\" > \"" + argsFile + "\"\necho '" + output + "'\n"
script := "#!/bin/sh\necho \"$@\" > \"${0%/*}/args\"\necho '" + output + "'\n"
require.NoError(t, os.WriteFile(path, []byte(script), 0o755))
}

Expand All @@ -1242,12 +1242,12 @@ func TestRunDockerAgentInContainerUsesConfiguredRuntime(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("the fake container runtime executable is a POSIX shell script")
}
t.Parallel()

tmpDir := t.TempDir()
argsFile := filepath.Join(tmpDir, "args")
fakeRuntime := filepath.Join(tmpDir, "fake-podman")
writeFakeContainerRuntime(t, fakeRuntime, argsFile, `{"type":"agent_choice","content":"ok"}`)
writeFakeContainerRuntime(t, fakeRuntime, `{"type":"agent_choice","content":"ok"}`)
t.Parallel()

runner := newRunner(
config.NewFileSource(filepath.Join(tmpDir, "agent.yaml")),
Expand Down Expand Up @@ -1275,12 +1275,12 @@ func TestBuildEvalImageUsesConfiguredRuntime(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("the fake container runtime executable is a POSIX shell script")
}
t.Parallel()

tmpDir := t.TempDir()
argsFile := filepath.Join(tmpDir, "args")
fakeRuntime := filepath.Join(tmpDir, "fake-podman")
writeFakeContainerRuntime(t, fakeRuntime, argsFile, "sha256:fake-image-id")
writeFakeContainerRuntime(t, fakeRuntime, "sha256:fake-image-id")
t.Parallel()

evalsDir := filepath.Join(tmpDir, "evals")
require.NoError(t, os.Mkdir(evalsDir, 0o755))
Expand Down
4 changes: 3 additions & 1 deletion pkg/evaluation/flavors_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ func TestEvalContainerForwardsOrderedFlavors(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("the fake container runtime executable is a POSIX shell script")
}
sharedRuntime := filepath.Join(t.TempDir(), "runtime")
require.NoError(t, os.WriteFile(sharedRuntime, []byte("#!/bin/sh\nprintf '%s\\n' \"$@\" > \"${0%/*}/args\"\necho '{\"type\":\"agent_choice\",\"content\":\"ok\"}'\n"), 0o755))
t.Parallel()

for _, setup := range []string{"", "echo setup"} {
Expand All @@ -26,7 +28,7 @@ func TestEvalContainerForwardsOrderedFlavors(t *testing.T) {
dir := t.TempDir()
argsFile := filepath.Join(dir, "args")
fake := filepath.Join(dir, "runtime")
require.NoError(t, os.WriteFile(fake, []byte("#!/bin/sh\nprintf '%s\\n' \"$@\" > \""+argsFile+"\"\necho '{\"type\":\"agent_choice\",\"content\":\"ok\"}'\n"), 0o755))
require.NoError(t, os.Symlink(sharedRuntime, fake))
runConfig := &config.RuntimeConfig{
Config: config.Config{Flavors: []string{"baseline", "fixtures,with spaces"}},
EnvProviderForTests: environment.NewNoEnvProvider(),
Expand Down
18 changes: 10 additions & 8 deletions pkg/evaluation/judge_container_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,14 +108,8 @@ func TestContainerMountsJudgeProviderWithoutCredentials(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("the fake container runtime is a POSIX shell script")
}
t.Parallel()

for _, setup := range []string{"", "echo setup"} {
t.Run(setup, func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
fakeRuntime := filepath.Join(dir, "runtime")
require.NoError(t, os.WriteFile(fakeRuntime, []byte(`#!/bin/sh
sharedRuntime := filepath.Join(t.TempDir(), "runtime")
require.NoError(t, os.WriteFile(sharedRuntime, []byte(`#!/bin/sh
printf '%s\n' "$@" > "${0%/*}/args"
for arg in "$@"; do
case "$arg" in
Expand All @@ -129,6 +123,14 @@ done
printf '%s' "${HOST_JUDGE_KEY-}" > "${0%/*}/host-key"
echo '{"type":"agent_choice","content":"hello"}'
`), 0o755))
t.Parallel()

for _, setup := range []string{"", "echo setup"} {
t.Run(setup, func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
fakeRuntime := filepath.Join(dir, "runtime")
require.NoError(t, os.Symlink(sharedRuntime, fakeRuntime))
runner := newRunner(config.NewFileSource(filepath.Join(dir, "agent.yaml")), &config.RuntimeConfig{
Config: config.Config{Providers: map[string]latest.ProviderConfig{"corporate": {Provider: "typesafe", TokenKey: "HOST_JUDGE_KEY"}}},
EnvProviderOverride: environment.NewMapEnvProvider(map[string]string{"HOST_JUDGE_KEY": "secret-value"}),
Expand Down
Loading