diff --git a/README.md b/README.md index 37e4e65..8ee4919 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,12 @@ tests whose coverage reaches a line are run for it; a package with no tests is a packages that import it. What the tests need besides the code is the environment's: spin runs it under its hack/testpg, which starts one PostgreSQL for every test process. +Each test process is held to its share of the memory free when the run starts (`-mem`). Run in a +cgroup systemd delegated it - `systemd-run --user --scope -p Delegate=yes go tool mutate` - it +starts each one in a child cgroup the kernel holds to that memory with no swap and to 1024 tasks, +and kills what the process leaves running; anywhere else it measures each process's tree from +/proc, which a test that starts itself without end escapes. + ## ctxlife, testquality, refs, vulngate diff --git a/cmd/mutate/main.go b/cmd/mutate/main.go index c922b3f..63f5bef 100644 --- a/cmd/mutate/main.go +++ b/cmd/mutate/main.go @@ -3,7 +3,9 @@ // A repository runs it as a Go tool (`tool github.com/spin-stack/go-tools/cmd/mutate` in its // go.mod, `go tool mutate`), from anywhere in its module. What its tests need besides the code - // a database they share, say - is the environment's: spin runs it under hack/testpg, which starts -// one PostgreSQL for every test process and says where in a variable. +// one PostgreSQL for every test process and says where in a variable. Run in a cgroup systemd +// delegated it (`systemd-run --user --scope -p Delegate=yes`), it starts each test process in a +// child the kernel holds to its memory and its tasks. package main import ( @@ -93,6 +95,7 @@ func main() { } mutate.Share(*jobs, int64(*mem*(1<<30))) + confine() started := time.Now() t, err := mutate.RunAll(chosen, importers, *timeout, *jobs, os.Stdout) if err != nil { @@ -135,6 +138,22 @@ func main() { os.Exit(1) } +// confine holds each test process in a cgroup of its own where this run was delegated one, and +// says which it does either way: a run that should have been given a cgroup and was not reads the +// same as one that was, until a test forks without end. +func confine() { + dir, err := mutate.Confine() + switch { + case err != nil: + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(2) + case dir != "": + fmt.Printf("mutate: each test process in a cgroup of its own under %s\n", dir) + default: + fmt.Printf("mutate: test memory measured from /proc; run under `systemd-run --user --scope -p Delegate=yes` to have the kernel hold it, and the processes a test starts\n") + } +} + // fromCache splits chosen into the edits to ask about and those the cache in dir says a run over // the same code refused; with no dir, there is no cache and every edit is asked. func fromCache(dir string, chosen []mutate.Mutation, importers map[string][]string) (*mutate.Cache, []mutate.Mutation, []mutate.Mutation, error) { diff --git a/internal/mutate/bound.go b/internal/mutate/bound.go index eb20508..fe76e6f 100644 --- a/internal/mutate/bound.go +++ b/internal/mutate/bound.go @@ -39,6 +39,28 @@ func memAvailable(meminfo []byte) int64 { return 0 } +// Confine holds every test process to its bound in a cgroup of its own when this run was +// delegated one, and says which; empty when the bound is measured from /proc. Call it once, +// after Share and before anything is asked. +// +// mutate-lane: TestATestProcessIsHeldByItsCgroup +func Confine() (string, error) { + dir, err := delegated() + testCgroup = dir + return dir, err +} + +// bound is what a test process is held to, as a report names it. +func bound() string { + switch { + case testCgroup == "": + return gib(memLimit) + case memLimit == 0: + return fmt.Sprintf("%d tasks", pidsMax) + } + return fmt.Sprintf("%s or %d tasks", gib(memLimit), pidsMax) +} + // gib is n bytes as people read it. func gib(n int64) string { return fmt.Sprintf("%.1f GiB", float64(n)/(1<<30)) @@ -48,6 +70,9 @@ func gib(n int64) string { // memLimit between them; exceeded says they were. A go test is the go command, its compiler and // the test binary, and the binary is the one that grows. func bounded(what string, cmd *exec.Cmd) (out []byte, exceeded bool, err error) { + if testCgroup != "" { + return confined(what, cmd) + } limit := memLimit if limit == 0 { out, err := combined(what, cmd) diff --git a/internal/mutate/bound_internal_test.go b/internal/mutate/bound_internal_test.go index dc953d6..dc1dbb1 100644 --- a/internal/mutate/bound_internal_test.go +++ b/internal/mutate/bound_internal_test.go @@ -1,6 +1,7 @@ package mutate import ( + "fmt" "slices" "testing" ) @@ -90,3 +91,21 @@ func TestATreeIsEverythingUnderItsRoot(t *testing.T) { t.Errorf("a process gone since it was listed walked as %v, holding %d", tree, rss) } } + +// A bound is named as it is held: memory alone when it is measured from /proc, and the tasks too +// in a cgroup, which may bound them alone. +func TestABoundIsNamedAsItIsHeld(t *testing.T) { + t.Cleanup(func() { testCgroup, memLimit = "", 0 }) + memLimit = 3 << 30 + if got := bound(); got != "3.0 GiB" { + t.Errorf("measured from /proc, the bound is named %q", got) + } + testCgroup = "/sys/fs/cgroup/run.scope" + if got, want := bound(), fmt.Sprintf("3.0 GiB or %d tasks", pidsMax); got != want { + t.Errorf("in a cgroup, the bound is named %q, want %q", got, want) + } + memLimit = 0 + if got, want := bound(), fmt.Sprintf("%d tasks", pidsMax); got != want { + t.Errorf("in a cgroup with no memory bound, the bound is named %q, want %q", got, want) + } +} diff --git a/internal/mutate/cgroup_internal_test.go b/internal/mutate/cgroup_internal_test.go new file mode 100644 index 0000000..3723c16 --- /dev/null +++ b/internal/mutate/cgroup_internal_test.go @@ -0,0 +1,85 @@ +//go:build linux + +package mutate + +import ( + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// inScope marks the run of this test that systemd-run started in a delegated scope. +const inScope = "MUTATE_TEST_IN_DELEGATED_SCOPE" + +// In a cgroup systemd delegated, a test process is held by the kernel: one that allocates past +// its memory, or forks past its tasks, is stopped and reported so, at once; what one leaves +// running when it ends is killed rather than waited on; and nothing of either is left after. +func TestATestProcessIsHeldByItsCgroup(t *testing.T) { + if !inDelegatedScope(t) { + return + } + dir, err := Confine() + if err != nil || dir == "" { + t.Fatalf("a delegated scope was taken as %q, %v", dir, err) + } + t.Cleanup(func() { testCgroup, memLimit, pidsMax = "", 0, 1024 }) + memLimit, pidsMax = 64<<20, 32 + run := func(script string) (string, bool, time.Duration, error) { + started := time.Now() + out, exceeded, err := bounded("test", exec.Command("sh", "-c", script)) + return string(out), exceeded, time.Since(started), err + } + + if out, exceeded, _, err := run("echo held"); exceeded || err != nil || out != "held\n" { + t.Errorf("a process within its bound answered %q, exceeded %v, %v", out, exceeded, err) + } + // tail -n 1 keeps a line until its end, and /dev/zero has none. + if _, exceeded, _, _ := run("head -c 512M /dev/zero | tail -n 1 >/dev/null"); !exceeded { + t.Error("a process that held 512 MiB under a bound of 64 was not stopped for it") + } + if _, exceeded, took, _ := run("for i in $(seq 100); do sleep 60 & done; wait"); !exceeded || took > 10*time.Second { + t.Errorf("a process that forked past 32 tasks: exceeded %v after %s", exceeded, took) + } + if out, exceeded, took, err := run("sleep 60 & echo left"); exceeded || err != nil || took > 10*time.Second { + t.Errorf("a process that left a child running: %q, exceeded %v, %v, after %s", out, exceeded, err, took) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if e.IsDir() && e.Name() != leadCgroup { + t.Errorf("%s is left in the delegated cgroup", e.Name()) + } + } + if procs, _ := os.ReadFile(dir + "/" + leadCgroup + "/cgroup.procs"); len(strings.Fields(string(procs))) != 1 { + t.Errorf("the run's own cgroup holds %q, want this process alone", procs) + } +} + +// inDelegatedScope is whether this is the run of t in a scope systemd delegated it; when it is +// not, it runs t again in one and reports what that run said, or skips where no user manager +// can start one. +func inDelegatedScope(t *testing.T) bool { + t.Helper() + if os.Getenv(inScope) != "" { + return true + } + if err := exec.Command("systemd-run", "--user", "--scope", "--quiet", "true").Run(); err != nil { + t.Skipf("no systemd user manager to delegate a scope: %v", err) + } + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + run := exec.Command("systemd-run", "--user", "--scope", "--quiet", "--collect", "-p", "Delegate=yes", + self, "-test.run=^"+t.Name()+"$", "-test.v", "-test.count=1") + run.Env = append(os.Environ(), inScope+"=1") + if out, err := run.CombinedOutput(); err != nil { + t.Fatalf("in a delegated scope: %v\n%s", err, out) + } + return false +} diff --git a/internal/mutate/cgroup_linux.go b/internal/mutate/cgroup_linux.go new file mode 100644 index 0000000..8f3f9b0 --- /dev/null +++ b/internal/mutate/cgroup_linux.go @@ -0,0 +1,207 @@ +package mutate + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "syscall" + "time" +) + +// pidsMax is how many tasks - processes and their threads - a test process and everything it +// starts may hold between them. A test binary is tens of threads. An edit that makes a binary +// start itself each time it starts - a supervisor handed "this executable" - is thousands of +// processes of tens of megabytes each, and no measure of a tree from /proc stops it: each +// process leaves the tree when its parent dies, so the tree never holds much. On 2026-10-22 such +// processes took the 30 GiB of a workstation until it hung. +var pidsMax = 1024 + +// testCgroup is the cgroup this run was delegated, each test process started in a child of it; +// empty where it was delegated none, and a test's memory is measured from /proc. What uses it is +// held by TestATestProcessIsHeldByItsCgroup, in a scope systemd delegated a process of its own, +// whose coverage a mutate run does not see: hence its functions' lane markers. +var testCgroup string + +// leadCgroup is the child of a delegated cgroup that the run's own processes move to: a cgroup +// that holds processes cannot give its children controllers of their own. +const leadCgroup = "mutate" + +// delegated is the directory of the cgroup this process runs in when systemd delegated it +// (`systemd-run --user --scope -p Delegate=yes`), with its processes moved to a child so its +// children can be bounded; empty for any other, which this run must not rearrange. +// +// mutate-lane: TestATestProcessIsHeldByItsCgroup +func delegated() (string, error) { + self, err := os.ReadFile("/proc/self/cgroup") + if err != nil { + return "", nil //nolint:nilerr // no /proc: no cgroup to be given + } + // cgroup v2 alone is one line, "0::"; v1 or a hybrid is more. + path, ok := strings.CutPrefix(strings.TrimSpace(string(self)), "0::") + if !ok || strings.Contains(path, "\n") { + return "", nil + } + said, err := exec.Command("systemctl", "--user", "show", "--property=Delegate", "--value", filepath.Base(path)).Output() + if err != nil || strings.TrimSpace(string(said)) != "yes" { + return "", nil //nolint:nilerr // no user manager, or a unit it did not delegate + } + dir := filepath.Join("/sys/fs/cgroup", path) + if err := takeOver(dir); err != nil { + return "", fmt.Errorf("mutate: the cgroup %s was delegated, and cannot bound its children: %w", dir, err) + } + return dir, nil +} + +// takeOver moves every process in dir to its lead child and gives dir's children the memory and +// pids controllers. +// +// mutate-lane: TestATestProcessIsHeldByItsCgroup +func takeOver(dir string) error { + lead := filepath.Join(dir, leadCgroup) + if err := os.Mkdir(lead, 0o755); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + // Again while it is busy: a process this run's parent starts between the move and the + // enabling lands in dir, and holds it. + var err error + for range 10 { + procs, readErr := os.ReadFile(filepath.Join(dir, "cgroup.procs")) + if readErr != nil { + return readErr + } + for _, pid := range strings.Fields(string(procs)) { + // ESRCH: gone since it was listed, which leaves nothing to move. + if err := set(lead, "cgroup.procs", pid); err != nil && !errors.Is(err, syscall.ESRCH) { + return err + } + } + err = set(dir, "cgroup.subtree_control", "+memory +pids") + if !errors.Is(err, syscall.EBUSY) { + return err + } + } + return err +} + +// confinedRuns names each test process's cgroup. +var confinedRuns atomic.Int64 + +// confined is bounded in a child of testCgroup: the kernel holds cmd and everything it starts to +// memLimit with no swap, and to pidsMax tasks, and what is left when cmd ends is killed with it. +// exceeded says they reached either. +// +// mutate-lane: TestATestProcessIsHeldByItsCgroup +func confined(what string, cmd *exec.Cmd) (out []byte, exceeded bool, err error) { + dir := filepath.Join(testCgroup, "t"+strconv.FormatInt(confinedRuns.Add(1), 10)) + if err := os.Mkdir(dir, 0o755); err != nil { + return nil, false, err + } + defer release(dir) + limits := [][2]string{ + {"pids.max", strconv.Itoa(pidsMax)}, + {"memory.swap.max", "0"}, + } + if memLimit > 0 { + limits = append(limits, [2]string{"memory.max", strconv.FormatInt(memLimit, 10)}) + } + for _, l := range limits { + if err := set(dir, l[0], l[1]); err != nil { + return nil, false, err + } + } + fd, err := os.Open(dir) + if err != nil { + return nil, false, err + } + defer func() { _ = fd.Close() }() + + var buf bytes.Buffer + cmd.Stdout, cmd.Stderr = &buf, &buf + cmd.SysProcAttr = &syscall.SysProcAttr{UseCgroupFD: true, CgroupFD: int(fd.Fd())} + // A process cmd started and left running holds its output open, and Wait would wait for it + // for ever; release kills it once cmd is done. + cmd.WaitDelay = time.Second + started := time.Now() + if err := cmd.Start(); err != nil { + account(what, cmd, time.Since(started)) + return nil, false, err + } + done, stopped := make(chan struct{}), make(chan bool) + go func() { stopped <- watchPids(dir, done) }() + err = cmd.Wait() + close(done) + stoppedAtPids := <-stopped + account(what, cmd, time.Since(started)) + if errors.Is(err, exec.ErrWaitDelay) { + err = nil // cmd itself succeeded; what it left is release's + } + return buf.Bytes(), stoppedAtPids || event(dir, "memory.events", "oom_kill") > 0, err +} + +// watchPids kills dir's processes once a fork was refused there, and says whether it did: what +// starts processes until it cannot otherwise waits on them, holding nothing, until its timeout. +// The memory bound needs no watch: the kernel kills what passes it. +// +// mutate-lane: TestATestProcessIsHeldByItsCgroup +func watchPids(dir string, done <-chan struct{}) bool { + tick := time.NewTicker(memPoll) + defer tick.Stop() + for { + select { + case <-done: + return false + case <-tick.C: + if event(dir, "pids.events", "max") > 0 { + _ = set(dir, "cgroup.kill", "1") + return true + } + } + } +} + +// release kills whatever is left in dir and removes it, once it is empty or after a few seconds +// of waiting for that. +// +// mutate-lane: TestATestProcessIsHeldByItsCgroup +func release(dir string) { + _ = set(dir, "cgroup.kill", "1") + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { + if event(dir, "cgroup.events", "populated") == 0 { + break + } + } + _ = os.Remove(dir) +} + +// set writes value to dir's file, which must be there: a cgroup's files are the kernel's, and +// one written where there is no cgroup is a file nobody reads. +// +// mutate-lane: TestATestProcessIsHeldByItsCgroup +func set(dir, file, value string) error { + f, err := os.OpenFile(filepath.Join(dir, file), os.O_WRONLY, 0) + if err != nil { + return err + } + _, err = f.WriteString(value) + return errors.Join(err, f.Close()) +} + +// event is key's count in dir's file of "key value" lines; 0 where it has none. +// +// mutate-lane: TestATestProcessIsHeldByItsCgroup +func event(dir, file, key string) int64 { + events, _ := os.ReadFile(filepath.Join(dir, file)) + for line := range strings.Lines(string(events)) { + if rest, ok := strings.CutPrefix(line, key+" "); ok { + n, _ := strconv.ParseInt(strings.TrimSpace(rest), 10, 64) + return n + } + } + return 0 +} diff --git a/internal/mutate/cgroup_other.go b/internal/mutate/cgroup_other.go new file mode 100644 index 0000000..f49ddb9 --- /dev/null +++ b/internal/mutate/cgroup_other.go @@ -0,0 +1,18 @@ +//go:build !linux + +package mutate + +import "os/exec" + +// testCgroup is never set where there are no cgroups: a test's memory is measured from /proc. +var testCgroup string + +// pidsMax bounds nothing where there are no cgroups. +var pidsMax = 0 + +func delegated() (string, error) { return "", nil } + +func confined(what string, cmd *exec.Cmd) ([]byte, bool, error) { + out, err := combined(what, cmd) + return out, false, err +} diff --git a/internal/mutate/cover.go b/internal/mutate/cover.go index 368a25f..2a98b65 100644 --- a/internal/mutate/cover.go +++ b/internal/mutate/cover.go @@ -116,7 +116,7 @@ func Cover(dir, timeout string, slots chan struct{}) (*Coverage, error) { // and say so, since an edit only it reaches then reads as one nothing holds. said, exceeded, runErr := bounded("cover: run one test", run) if exceeded { - runErr = fmt.Errorf("held over the %s a test process may, and stopped", gib(memLimit)) + runErr = fmt.Errorf("held over the %s a test process may, and stopped", bound()) } failed := runErr != nil if failed { diff --git a/internal/mutate/runner.go b/internal/mutate/runner.go index e563f2c..2ef9439 100644 --- a/internal/mutate/runner.go +++ b/internal/mutate/runner.go @@ -84,7 +84,7 @@ func RunAll(chosen []Mutation, importers map[string][]string, timeout string, jo // Named with its bound: the edit is refused, and the loop it made is worth knowing of. t.Killed++ t.Refused = append(t.Refused, m) - _, _ = fmt.Fprintf(w, " refused %s (its tests held over %s, and were stopped)\n", m, gib(memLimit)) + _, _ = fmt.Fprintf(w, " refused %s (its tests held over %s, and were stopped)\n", m, bound()) case Unbuildable: t.Unbuildable++ _, _ = fmt.Fprintf(w, " no edit %s (does not build)\n", m)