mutate: each test process in a cgroup of its own when the run was delegated one

README.md

@@ -143,6 +143,12 @@ tests whose coverage reaches a line are run for it; a package with no tests is a
 packages that import it. What the tests need besides the code is the environment's: spin runs it
 under its hack/testpg, which starts one PostgreSQL for every test process.
 
+Each test process is held to its share of the memory free when the run starts (`-mem`). Run in a
+cgroup systemd delegated it - `systemd-run --user --scope -p Delegate=yes go tool mutate` - it
+starts each one in a child cgroup the kernel holds to that memory with no swap and to 1024 tasks,
+and kills what the process leaves running; anywhere else it measures each process's tree from
+/proc, which a test that starts itself without end escapes.
+
 
 ## ctxlife, testquality, refs, vulngate
 

cmd/mutate/main.go

@@ -3,7 +3,9 @@
 // A repository runs it as a Go tool (`tool github.com/spin-stack/go-tools/cmd/mutate` in its
 // go.mod, `go tool mutate`), from anywhere in its module. What its tests need besides the code -
 // a database they share, say - is the environment's: spin runs it under hack/testpg, which starts
-// one PostgreSQL for every test process and says where in a variable.
+// one PostgreSQL for every test process and says where in a variable. Run in a cgroup systemd
+// delegated it (`systemd-run --user --scope -p Delegate=yes`), it starts each test process in a
+// child the kernel holds to its memory and its tasks.
 package main
 
 import (
@@ -93,6 +95,7 @@ func main() {
 	}
 
 	mutate.Share(*jobs, int64(*mem*(1<<30)))
+	confine()
 	started := time.Now()
 	t, err := mutate.RunAll(chosen, importers, *timeout, *jobs, os.Stdout)
 	if err != nil {
@@ -135,6 +138,22 @@ func main() {
 	os.Exit(1)
 }
 
+// confine holds each test process in a cgroup of its own where this run was delegated one, and
+// says which it does either way: a run that should have been given a cgroup and was not reads the
+// same as one that was, until a test forks without end.
+func confine() {
+	dir, err := mutate.Confine()
+	switch {
+	case err != nil:
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(2)
+	case dir != "":
+		fmt.Printf("mutate: each test process in a cgroup of its own under %s\n", dir)
+	default:
+		fmt.Printf("mutate: test memory measured from /proc; run under `systemd-run --user --scope -p Delegate=yes` to have the kernel hold it, and the processes a test starts\n")
+	}
+}
+
 // fromCache splits chosen into the edits to ask about and those the cache in dir says a run over
 // the same code refused; with no dir, there is no cache and every edit is asked.
 func fromCache(dir string, chosen []mutate.Mutation, importers map[string][]string) (*mutate.Cache, []mutate.Mutation, []mutate.Mutation, error) {

internal/mutate/bound.go

@@ -39,6 +39,28 @@ func memAvailable(meminfo []byte) int64 {
 	return 0
 }
 
+// Confine holds every test process to its bound in a cgroup of its own when this run was
+// delegated one, and says which; empty when the bound is measured from /proc. Call it once,
+// after Share and before anything is asked.
+//
+// mutate-lane: TestATestProcessIsHeldByItsCgroup
+func Confine() (string, error) {
+	dir, err := delegated()
+	testCgroup = dir
+	return dir, err
+}
+
+// bound is what a test process is held to, as a report names it.
+func bound() string {
+	switch {
+	case testCgroup == "":
+		return gib(memLimit)
+	case memLimit == 0:
+		return fmt.Sprintf("%d tasks", pidsMax)
+	}
+	return fmt.Sprintf("%s or %d tasks", gib(memLimit), pidsMax)
+}
+
 // gib is n bytes as people read it.
 func gib(n int64) string {
 	return fmt.Sprintf("%.1f GiB", float64(n)/(1<<30))
@@ -48,6 +70,9 @@ func gib(n int64) string {
 // memLimit between them; exceeded says they were. A go test is the go command, its compiler and
 // the test binary, and the binary is the one that grows.
 func bounded(what string, cmd *exec.Cmd) (out []byte, exceeded bool, err error) {
+	if testCgroup != "" {
+		return confined(what, cmd)
+	}
 	limit := memLimit
 	if limit == 0 {
 		out, err := combined(what, cmd)

internal/mutate/bound_internal_test.go

@@ -1,6 +1,7 @@
 package mutate
 
 import (
+	"fmt"
 	"slices"
 	"testing"
 )
@@ -90,3 +91,21 @@ func TestATreeIsEverythingUnderItsRoot(t *testing.T) {
 		t.Errorf("a process gone since it was listed walked as %v, holding %d", tree, rss)
 	}
 }
+
+// A bound is named as it is held: memory alone when it is measured from /proc, and the tasks too
+// in a cgroup, which may bound them alone.
+func TestABoundIsNamedAsItIsHeld(t *testing.T) {
+	t.Cleanup(func() { testCgroup, memLimit = "", 0 })
+	memLimit = 3 << 30
+	if got := bound(); got != "3.0 GiB" {
+		t.Errorf("measured from /proc, the bound is named %q", got)
+	}
+	testCgroup = "/sys/fs/cgroup/run.scope"
+	if got, want := bound(), fmt.Sprintf("3.0 GiB or %d tasks", pidsMax); got != want {
+		t.Errorf("in a cgroup, the bound is named %q, want %q", got, want)
+	}
+	memLimit = 0
+	if got, want := bound(), fmt.Sprintf("%d tasks", pidsMax); got != want {
+		t.Errorf("in a cgroup with no memory bound, the bound is named %q, want %q", got, want)
+	}
+}

internal/mutate/cgroup_internal_test.go

@@ -0,0 +1,85 @@
+//go:build linux
+
+package mutate
+
+import (
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+)
+
+// inScope marks the run of this test that systemd-run started in a delegated scope.
+const inScope = "MUTATE_TEST_IN_DELEGATED_SCOPE"
+
+// In a cgroup systemd delegated, a test process is held by the kernel: one that allocates past
+// its memory, or forks past its tasks, is stopped and reported so, at once; what one leaves
+// running when it ends is killed rather than waited on; and nothing of either is left after.
+func TestATestProcessIsHeldByItsCgroup(t *testing.T) {
+	if !inDelegatedScope(t) {
+		return
+	}
+	dir, err := Confine()
+	if err != nil || dir == "" {
+		t.Fatalf("a delegated scope was taken as %q, %v", dir, err)
+	}
+	t.Cleanup(func() { testCgroup, memLimit, pidsMax = "", 0, 1024 })
+	memLimit, pidsMax = 64<<20, 32
+	run := func(script string) (string, bool, time.Duration, error) {
+		started := time.Now()
+		out, exceeded, err := bounded("test", exec.Command("sh", "-c", script))
+		return string(out), exceeded, time.Since(started), err
+	}
+
+	if out, exceeded, _, err := run("echo held"); exceeded || err != nil || out != "held\n" {
+		t.Errorf("a process within its bound answered %q, exceeded %v, %v", out, exceeded, err)
+	}
+	// tail -n 1 keeps a line until its end, and /dev/zero has none.
+	if _, exceeded, _, _ := run("head -c 512M /dev/zero | tail -n 1 >/dev/null"); !exceeded {
+		t.Error("a process that held 512 MiB under a bound of 64 was not stopped for it")
+	}
+	if _, exceeded, took, _ := run("for i in $(seq 100); do sleep 60 & done; wait"); !exceeded || took > 10*time.Second {
+		t.Errorf("a process that forked past 32 tasks: exceeded %v after %s", exceeded, took)
+	}
+	if out, exceeded, took, err := run("sleep 60 & echo left"); exceeded || err != nil || took > 10*time.Second {
+		t.Errorf("a process that left a child running: %q, exceeded %v, %v, after %s", out, exceeded, err, took)
+	}
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, e := range entries {
+		if e.IsDir() && e.Name() != leadCgroup {
+			t.Errorf("%s is left in the delegated cgroup", e.Name())
+		}
+	}
+	if procs, _ := os.ReadFile(dir + "/" + leadCgroup + "/cgroup.procs"); len(strings.Fields(string(procs))) != 1 {
+		t.Errorf("the run's own cgroup holds %q, want this process alone", procs)
+	}
+}
+
+// inDelegatedScope is whether this is the run of t in a scope systemd delegated it; when it is
+// not, it runs t again in one and reports what that run said, or skips where no user manager
+// can start one.
+func inDelegatedScope(t *testing.T) bool {
+	t.Helper()
+	if os.Getenv(inScope) != "" {
+		return true
+	}
+	if err := exec.Command("systemd-run", "--user", "--scope", "--quiet", "true").Run(); err != nil {
+		t.Skipf("no systemd user manager to delegate a scope: %v", err)
+	}
+	self, err := os.Executable()
+	if err != nil {
+		t.Fatal(err)
+	}
+	run := exec.Command("systemd-run", "--user", "--scope", "--quiet", "--collect", "-p", "Delegate=yes",
+		self, "-test.run=^"+t.Name()+"$", "-test.v", "-test.count=1")
+	run.Env = append(os.Environ(), inScope+"=1")
+	if out, err := run.CombinedOutput(); err != nil {
+		t.Fatalf("in a delegated scope: %v\n%s", err, out)
+	}
+	return false
+}