diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dca71bc..ac2f3a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,14 +88,21 @@ jobs: probe_dir="$RUNNER_TEMP/runtime probe λ" mkdir -p "$probe_dir/state" go build -o "$probe_dir/runtime.exe" ./cmd/devsy-fake-runtime + go build -o "$probe_dir/supervisor.exe" ./cmd/devsy-runtime-supervisor go build -o "$probe_dir/probe.exe" ./cmd/devsy-runtime-spawn-probe "$probe_dir/probe.exe" --binary "$probe_dir/runtime.exe" \ --samples 100 --revision "$GITHUB_SHA" \ -- --state-dir "$probe_dir/state" > "$probe_dir/report.json" + "$probe_dir/probe.exe" --binary "$probe_dir/runtime.exe" \ + --supervisor-binary "$probe_dir/supervisor.exe" \ + --samples 100 --revision "$GITHUB_SHA" \ + -- --state-dir "$probe_dir/state" > "$probe_dir/supervised-report.json" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runtime-spawn-${{ matrix.os }} - path: ${{ runner.temp }}/runtime probe λ/report.json + path: | + ${{ runner.temp }}/runtime probe λ/report.json + ${{ runner.temp }}/runtime probe λ/supervised-report.json if-no-files-found: error release-please: name: Release Please diff --git a/README.md b/README.md index 0e24054..764c39d 100644 --- a/README.md +++ b/README.md @@ -197,16 +197,21 @@ this suite does not certify those policies or runtime-specific image semantics. ## Measuring plugin startup The host adapter's startup spike uses the real plugin transport and includes -process shutdown and reaping. Build both executables without race instrumentation +process shutdown and reaping. Build all three executables without race instrumentation (the race runtime's exit delay would distort process lifetime measurements): ```sh mkdir -p bin/probe-state go build -o bin/devsy-fake-runtime ./cmd/devsy-fake-runtime go build -o bin/devsy-runtime-spawn-probe ./cmd/devsy-runtime-spawn-probe +go build -o bin/devsy-runtime-supervisor ./cmd/devsy-runtime-supervisor "$PWD/bin/devsy-runtime-spawn-probe" \ --binary "$PWD/bin/devsy-fake-runtime" --samples 100 \ -- --state-dir "$PWD/bin/probe-state" > bin/spawn-report.json +"$PWD/bin/devsy-runtime-spawn-probe" \ + --binary "$PWD/bin/devsy-fake-runtime" \ + --supervisor-binary "$PWD/bin/devsy-runtime-supervisor" --samples 100 \ + -- --state-dir "$PWD/bin/probe-state" > bin/supervised-spawn-report.json ``` The probe requires an explicitly trusted absolute executable path and an unused @@ -225,7 +230,12 @@ initialization call. Find measures ordinary absence independently of Info. Reports retain raw stage timings in nanoseconds, total p50/p95/p99 using nearest-rank percentiles, process IDs and reaping confirmation, OS/architecture, Go version, CPU count, source revision, and the measured binary's SHA-256. -Arguments, full environment values, and executable paths are omitted. +The additive `launch_mode` field identifies `direct` or `supervised` launches; +`supervisor_sha256` identifies the helper in supervised reports. In direct mode, +`pid` identifies the plugin; in supervised mode it identifies the owning helper. +Startup and total durations in supervised mode include launching both processes, +and reaping includes waiting for supervisor cleanup. Arguments, full environment +values, and executable paths are omitted. The binary hash identifies the measured artifact; it is not an integrity check against a trusted expected checksum. Hashing also warms file caches. First @@ -237,9 +247,13 @@ plugin lifetime; this probe makes no ownership decision or latency threshold. CI runs the probe on Linux, macOS, and Windows with executable paths containing spaces and non-ASCII characters, and publishes `runtime-spawn-*` JSON artifacts. -These jobs gate release automation alongside the existing quality checks. -Cross-platform cancellation and descendant-process cleanup are the next host -hardening gates; successful startup measurements do not certify those behaviors. +Each artifact retains the direct `report.json` and `supervised-report.json` from +the same runner, runtime binary, operation, and sample count. Compare Info and +Find separately. These are sequential warm-cache experiments (direct runs first), +not randomized trials, cold-cache results, or latency acceptance tests. Repeat in +both orders on a representative host before drawing a performance conclusion. +These jobs gate release automation alongside the existing quality checks; +successful startup measurements alone do not certify descendant-process cleanup. ## Process ownership experiments diff --git a/cmd/devsy-runtime-spawn-probe/main.go b/cmd/devsy-runtime-spawn-probe/main.go index 27d9031..f0c0854 100644 --- a/cmd/devsy-runtime-spawn-probe/main.go +++ b/cmd/devsy-runtime-spawn-probe/main.go @@ -21,6 +21,12 @@ func main() { "", "absolute path to an explicitly trusted plugin executable", ) + flag.StringVar( + &config.SupervisorBinary, + "supervisor-binary", + "", + "absolute path to a trusted supervisor helper; empty measures direct launch", + ) flag.IntVar(&config.Samples, "samples", 100, "warm repetitions per operation") flag.DurationVar( &config.Timeout, diff --git a/internal/spawnprobe/probe.go b/internal/spawnprobe/probe.go index 409d948..3336406 100644 --- a/internal/spawnprobe/probe.go +++ b/internal/spawnprobe/probe.go @@ -11,23 +11,26 @@ import ( "os/exec" "path/filepath" "runtime" + "strconv" "time" sdkplugin "github.com/devsy-org/devsy-runtime-sdk/plugin" "github.com/devsy-org/devsy-runtime-sdk/runtimev1" + "github.com/devsy-org/devsy-runtime-sdk/supervisor" "github.com/hashicorp/go-hclog" hplugin "github.com/hashicorp/go-plugin" ) // Config supplies an explicitly trusted executable and bounded measurement count. type Config struct { - Binary string - Args []string - Samples int - Timeout time.Duration - WorkspaceID string - Revision string - Diagnostics io.Writer + SupervisorBinary string + Binary string + Args []string + Samples int + Timeout time.Duration + WorkspaceID string + Revision string + Diagnostics io.Writer } // Run measures one first launch and Samples warm launches for Info and Find. @@ -54,24 +57,56 @@ func Run(ctx context.Context, config Config) (*Report, error) { return &Report{ SchemaVersion: 1, GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, GoVersion: runtime.Version(), CPUs: runtime.NumCPU(), Revision: config.Revision, - BinarySHA256: checksum, RecordedAt: time.Now().UTC(), Info: info, Find: find, + LaunchMode: launchMode(config), BinarySHA256: checksum.runtime, + SupervisorSHA256: checksum.supervisor, RecordedAt: time.Now().UTC(), Info: info, Find: find, }, nil } -func validate(config Config) (string, error) { +type binaryHashes struct { + runtime string + supervisor string +} + +func launchMode(config Config) string { + if config.SupervisorBinary != "" { + return "supervised" + } + return "direct" +} + +func validate(config Config) (binaryHashes, error) { if !filepath.IsAbs(config.Binary) { - return "", errors.New("plugin binary must be an absolute path") + return binaryHashes{}, errors.New("plugin binary must be an absolute path") } if config.Samples < 1 || config.Samples > 10000 { - return "", errors.New("samples must be between 1 and 10000") + return binaryHashes{}, errors.New("samples must be between 1 and 10000") } if config.Timeout <= 0 { - return "", errors.New("operation timeout must be positive") + return binaryHashes{}, errors.New("operation timeout must be positive") } if config.WorkspaceID == "" { - return "", errors.New("workspace ID is required for Find") + return binaryHashes{}, errors.New("workspace ID is required for Find") } - return checksum(config.Binary) + return artifactHashes(config) +} + +func artifactHashes(config Config) (binaryHashes, error) { + hashes := binaryHashes{} + var err error + hashes.runtime, err = checksum(config.Binary) + if err != nil { + return hashes, err + } + if config.SupervisorBinary != "" { + if !filepath.IsAbs(config.SupervisorBinary) { + return hashes, errors.New("supervisor binary must be an absolute path") + } + hashes.supervisor, err = checksum(config.SupervisorBinary) + if err != nil { + return hashes, fmt.Errorf("supervisor: %w", err) + } + } + return hashes, nil } func checksum(binary string) (string, error) { @@ -133,34 +168,15 @@ func measureOne( } ctx, cancel := context.WithTimeout(parent, config.Timeout) defer cancel() - // #nosec G204 -- The caller supplies an explicitly trusted absolute executable. - command := exec.CommandContext(ctx, config.Binary, config.Args...) - client := hplugin.NewClient(&hplugin.ClientConfig{ - HandshakeConfig: sdkplugin.Handshake(), - VersionedPlugins: map[int]hplugin.PluginSet{ - sdkplugin.ProtocolVersion: sdkplugin.ClientPlugins(), - }, - Cmd: command, - AllowedProtocols: []hplugin.Protocol{hplugin.ProtocolGRPC}, - StartTimeout: config.Timeout, - Logger: hclog.NewNullLogger(), - Stderr: config.Diagnostics, - SyncStderr: config.Diagnostics, - }) + client := newClient(ctx, config) started := time.Now() defer func() { - reapStarted := time.Now() - client.Kill() - sample.Reaped = client.Exited() - if command.Process != nil { - sample.PID = command.Process.Pid - } - sample.ReapNS = time.Since(reapStarted).Nanoseconds() - sample.TotalNS = time.Since(started).Nanoseconds() - if resultErr == nil && !sample.Reaped { - resultErr = errors.New("plugin was not reaped") + cleanupErr := reap(client, &sample, started) + if resultErr == nil { + resultErr = cleanupErr } }() + rpc, err := client.Client() sample.StartupNS = time.Since(started).Nanoseconds() if err != nil { @@ -212,3 +228,45 @@ func call( } return nil } + +func newClient(ctx context.Context, config Config) *hplugin.Client { + clientConfig := &hplugin.ClientConfig{ + HandshakeConfig: sdkplugin.Handshake(), + VersionedPlugins: map[int]hplugin.PluginSet{ + sdkplugin.ProtocolVersion: sdkplugin.ClientPlugins(), + }, + AllowedProtocols: []hplugin.Protocol{hplugin.ProtocolGRPC}, + StartTimeout: config.Timeout, + Logger: hclog.NewNullLogger(), + Stderr: config.Diagnostics, + SyncStderr: config.Diagnostics, + } + if config.SupervisorBinary == "" { + // #nosec G204 -- Explicitly trusted absolute executable, validated before measurement. + clientConfig.Cmd = exec.CommandContext(ctx, config.Binary, config.Args...) + } else { + clientConfig.RunnerFunc = supervisor.Runner(supervisor.Options{ + SupervisorBinary: config.SupervisorBinary, + RuntimeBinary: config.Binary, + Args: config.Args, + }) + } + return hplugin.NewClient(clientConfig) +} + +func reap(client *hplugin.Client, sample *Sample, started time.Time) error { + reapStarted := time.Now() + pid, pidErr := strconv.Atoi(client.ID()) + sample.PID = pid + client.Kill() + sample.Reaped = client.Exited() + sample.ReapNS = time.Since(reapStarted).Nanoseconds() + sample.TotalNS = time.Since(started).Nanoseconds() + if pidErr != nil || sample.PID <= 0 { + return errors.New("runner did not report a process ID") + } + if !sample.Reaped { + return errors.New("runner was not reaped") + } + return nil +} diff --git a/internal/spawnprobe/probe_test.go b/internal/spawnprobe/probe_test.go index efaecbd..8376a1f 100644 --- a/internal/spawnprobe/probe_test.go +++ b/internal/spawnprobe/probe_test.go @@ -11,7 +11,7 @@ import ( "time" ) -var executable string +var executable, supervisorExecutable string func TestMain(m *testing.M) { dir, err := os.MkdirTemp("", "devsy-spawn-probe-") @@ -19,23 +19,34 @@ func TestMain(m *testing.M) { panic(err) } executable = filepath.Join(dir, "runtime λ fixture") + supervisorExecutable = filepath.Join(dir, "supervisor λ fixture") if runtime.GOOS == "windows" { executable += ".exe" + supervisorExecutable += ".exe" + } + for binary, packagePath := range map[string]string{ + executable: "../../cmd/devsy-fake-runtime", + supervisorExecutable: "../../cmd/devsy-runtime-supervisor", + } { + if err := buildFixture(binary, packagePath); err != nil { + _ = os.RemoveAll(dir) + os.Exit(1) + } } - // No race instrumentation in the measured executable: its shutdown delay - // would dominate the process lifetime measurement. - // #nosec G204 -- Fixed fixture package, locally-created output directory. - command := exec.Command("go", "build", "-o", executable, "../../cmd/devsy-fake-runtime") - command.Stdout, command.Stderr = os.Stdout, os.Stderr - if err := command.Run(); err != nil { - _ = os.RemoveAll(dir) - os.Exit(1) - } + code := m.Run() _ = os.RemoveAll(dir) os.Exit(code) } +func buildFixture(binary, packagePath string) error { + // No race instrumentation: its shutdown delay would distort lifetime measurements. + // #nosec G204 -- Fixed fixture packages and locally-created output directory. + command := exec.Command("go", "build", "-o", binary, packagePath) + command.Stdout, command.Stderr = os.Stdout, os.Stderr + return command.Run() +} + func testConfig(t *testing.T) Config { t.Helper() return Config{ @@ -45,7 +56,18 @@ func testConfig(t *testing.T) Config { } func TestRealPluginMeasurements(t *testing.T) { - report, err := Run(context.Background(), testConfig(t)) + for _, owned := range []bool{false, true} { + config := testConfig(t) + if owned { + config.SupervisorBinary = supervisorExecutable + } + t.Run(launchMode(config), func(t *testing.T) { checkRealMeasurements(t, config) }) + } +} + +func checkRealMeasurements(t *testing.T, config Config) { + t.Helper() + report, err := Run(context.Background(), config) if err != nil { t.Fatal(err) } @@ -53,6 +75,7 @@ func TestRealPluginMeasurements(t *testing.T) { len(report.BinarySHA256) != 64 { t.Fatalf("missing reproducibility metadata: %v", report) } + assertLaunchMetadata(t, config, report) if report.ColdCacheMeasured { t.Fatal("warm file cache was labeled cold") } @@ -83,6 +106,9 @@ func TestInvalidConfiguration(t *testing.T) { cases := []func(*Config){ func(c *Config) { c.Binary = "runtime-from-PATH" }, func(c *Config) { c.Binary = t.TempDir() }, + func(c *Config) { c.SupervisorBinary = "supervisor-from-PATH" }, + func(c *Config) { c.SupervisorBinary = t.TempDir() }, + func(c *Config) { c.SupervisorBinary = filepath.Join(t.TempDir(), "missing") }, func(c *Config) { c.Samples = 0 }, func(c *Config) { c.Samples = 10001 }, func(c *Config) { c.Timeout = 0 }, @@ -111,10 +137,46 @@ func TestCancellationAndMalformedInfo(t *testing.T) { } func TestStartupTimeout(t *testing.T) { - config := testConfig(t) + for _, owned := range []bool{false, true} { + config := testConfig(t) + if owned { + config.SupervisorBinary = supervisorExecutable + } + t.Run(launchMode(config), func(t *testing.T) { checkStartupTimeout(t, config) }) + } +} + +func checkStartupTimeout(t *testing.T, config Config) { + t.Helper() config.Args = append(config.Args, "--mode", "delay-handshake", "--handshake-delay", "1m") config.Timeout = 100 * time.Millisecond if _, err := Run(context.Background(), config); err == nil { t.Fatal("delayed startup accepted") } } + +func assertLaunchMetadata(t *testing.T, config Config, report *Report) { + t.Helper() + assertArtifactHash(t, config.Binary, report.BinarySHA256) + if config.SupervisorBinary != "" { + assertArtifactHash(t, config.SupervisorBinary, report.SupervisorSHA256) + } + if report.LaunchMode != launchMode(config) || + (report.SupervisorSHA256 != "") != (config.SupervisorBinary != "") { + t.Fatal("launch metadata does not identify the measurement mode") + } + if config.SupervisorBinary != "" && len(report.SupervisorSHA256) != 64 { + t.Fatal("supervisor checksum missing") + } +} + +func assertArtifactHash(t *testing.T, binary, got string) { + t.Helper() + expected, err := checksum(binary) + if err != nil { + t.Fatal(err) + } + if got != expected { + t.Fatal("report fingerprint does not match measured executable") + } +} diff --git a/internal/spawnprobe/report.go b/internal/spawnprobe/report.go index bf68d9d..27e4fc3 100644 --- a/internal/spawnprobe/report.go +++ b/internal/spawnprobe/report.go @@ -7,6 +7,7 @@ import ( ) // Sample records wall-clock nanoseconds for one complete plugin operation. +// PID identifies the direct plugin or, in supervised mode, its owning supervisor. type Sample struct { PID int `json:"pid"` Reaped bool `json:"reaped"` @@ -34,6 +35,8 @@ type Measurements struct { // Report contains reproducibility metadata, never executable arguments or environment values. type Report struct { + LaunchMode string `json:"launch_mode"` + SupervisorSHA256 string `json:"supervisor_sha256,omitempty"` SchemaVersion int `json:"schema_version"` GOOS string `json:"goos"` GOARCH string `json:"goarch"`