test: probe runtime cancellation and process ownership

.github/workflows/ci.yml

@@ -21,7 +21,17 @@ jobs:
       - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version: '1.26.8'
-      - run: go test -race ./...
+      - name: Run race tests and record process-ownership evidence
+        shell: bash
+        run: |
+          set -o pipefail
+          go test -race -json ./... | tee "$RUNNER_TEMP/test-report.json"
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        if: always()
+        with:
+          name: runtime-tests-${{ matrix.os }}
+          path: ${{ runner.temp }}/test-report.json
+          if-no-files-found: error
       - run: go vet ./...
   generated:
     name: Generated bindings

README.md

@@ -239,3 +239,36 @@ spaces and non-ASCII characters, and publishes `runtime-spawn-*` JSON artifacts.
 These jobs gate release automation alongside the existing quality checks.
 Cross-platform cancellation and descendant-process cleanup are the next host
 hardening gates; successful startup measurements do not certify those behaviors.
+
+## Process ownership experiments
+
+Run the real-process cancellation and crash probes with:
+
+```sh
+mise exec -- go test -race -v ./internal/processprobe
+```
+
+The fixture launches a host, a plugin, and a blocking runtime child from an
+executable path containing spaces and Unicode. Readiness acknowledgements
+precede cancellation, and independent observer connections answer liveness
+checks. The plugin acknowledges child reaping only after `exec.Cmd.Wait` returns.
+CI runs these probes on Linux, macOS, and Windows with the other race tests;
+`runtime-tests-*` artifacts retain their JSON test output.
+
+| Scenario | Behavior asserted by the spike |
+| --- | --- |
+| Cancel unary RPC or Exec | A cooperative plugin cancels and reaps its child |
+| Child ignores interruption | A bounded forced-kill fallback reaps the child; Unix also acknowledges the ignored signal |
+| Abrupt plugin death | The runtime child survives until the independent test observer kills it |
+| Abrupt host death | Transport loss cancels the RPC and reaps the child, but the plugin survives until the observer kills it |
+
+The last two tests deliberately record ownership gaps in the current transport.
+A passing probe suite does not mean abrupt process-tree cleanup is implemented.
+`exec.CommandContext` and `Client.Kill()` alone do not establish a complete
+process-tree policy. Windows cannot deliver `os.Interrupt` through
+`os.Process.Signal`, so cancellation exercises the forced-kill fallback there.
+
+These results block runtime cutover until explicit host/plugin/descendant
+ownership is designed and tested on all supported platforms. The fixture is an
+experiment, not an exported process supervisor. Streaming stress and executable
+trust/environment compatibility remain separate host-hardening work.

internal/processfixture/main.go

@@ -0,0 +1,217 @@
+// Process fixture only: exercises ownership failures through the real SDK transport.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"os/signal"
+	"syscall"
+	"time"
+
+	sdkplugin "github.com/devsy-org/devsy-runtime-sdk/plugin"
+	"github.com/devsy-org/devsy-runtime-sdk/runtimev1"
+	"github.com/devsy-org/devsy-runtime-sdk/server"
+	"github.com/hashicorp/go-hclog"
+	hplugin "github.com/hashicorp/go-plugin"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/status"
+)
+
+const (
+	childRole  = "child"
+	pluginRole = "plugin"
+)
+
+type event struct {
+	Role string `json:"role"`
+	Kind string `json:"kind"`
+	PID  int    `json:"pid"`
+}
+
+type fixture struct {
+	runtimev1.UnimplementedRuntimeDriverServer
+	observer string
+	ignore   bool
+	events   *reporter
+}
+
+func main() {
+	mode := flag.String("mode", pluginRole, "fixture role")
+	observer := flag.String("observer", "", "loopback observer address")
+	ignore := flag.Bool("ignore-interrupt", false, "child ignores normal termination")
+	flag.Parse()
+	if err := run(*mode, *observer, *ignore); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func run(mode, observer string, ignore bool) error {
+	conn, err := net.DialTimeout("tcp", observer, 5*time.Second)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = conn.Close() }()
+	events := &reporter{encoder: json.NewEncoder(conn), role: mode}
+	if err := events.Encode(event{Role: mode, Kind: "ready", PID: os.Getpid()}); err != nil {
+		return err
+	}
+	disconnected := make(chan struct{})
+	go events.respond(conn, disconnected)
+	switch mode {
+	case childRole:
+		return child(disconnected, events, ignore)
+	case "host":
+		return host(observer, ignore)
+	case pluginRole:
+		server.Serve(&fixture{observer: observer, ignore: ignore, events: events})
+		return nil
+	default:
+		return errors.New("unknown fixture role")
+	}
+}
+
+func child(disconnected <-chan struct{}, events *reporter, ignore bool) error {
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
+	defer signal.Stop(signals)
+	// Publish readiness only after the signal handler and observer connection exist.
+	if err := json.NewEncoder(os.Stdout).
+		Encode(event{Role: childRole, Kind: "armed", PID: os.Getpid()}); err != nil {
+		return err
+	}
+	for {
+		select {
+		case <-disconnected:
+			return nil
+		case <-signals:
+			if !ignore {
+				return nil
+			}
+			if err := events.Encode(
+				event{Role: childRole, Kind: "ignored", PID: os.Getpid()},
+			); err != nil {
+				return err
+			}
+		}
+	}
+}
+
+func (f *fixture) Info(
+	ctx context.Context,
+	_ *runtimev1.InfoRequest,
+) (*runtimev1.InfoResponse, error) {
+	if err := f.block(ctx); err != nil {
+		return nil, err
+	}
+	return nil, status.FromContextError(ctx.Err()).Err()
+}
+
+func (f *fixture) Exec(
+	stream grpc.BidiStreamingServer[runtimev1.ExecClientMessage, runtimev1.ExecServerMessage],
+) error {
+	first, err := stream.Recv()
+	if err != nil {
+		return err
+	}
+	if first.GetStart() == nil {
+		return errors.New("missing Exec start")
+	}
+	return f.block(stream.Context())
+}
+
+func (f *fixture) block(ctx context.Context) error {
+	executable, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	// #nosec G204 -- Self-executable fixture, with arguments controlled by the test harness.
+	cmd := exec.CommandContext(ctx, executable, "--mode", childRole, "--observer", f.observer,
+		fmt.Sprintf("--ignore-interrupt=%t", f.ignore))
+	cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) }
+	// Windows cannot deliver os.Interrupt to a child; WaitDelay also bounds this fallback.
+	cmd.WaitDelay = 100 * time.Millisecond
+	cmd.Stderr = os.Stderr
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	if err := cmd.Start(); err != nil {
+		_ = out.Close()
+		return err
+	}
+	var ready event
+	if err := json.NewDecoder(out).Decode(&ready); err != nil {
+		_ = cmd.Process.Kill()
+		_ = cmd.Wait()
+		return err
+	}
+	if err := f.events.Encode(
+		event{Role: pluginRole, Kind: "child-ready", PID: ready.PID},
+	); err != nil {
+		_ = cmd.Process.Kill()
+		_ = cmd.Wait()
+		return err
+	}
+	// Wait is required even when cancellation or forced termination has occurred.
+	waitErr := cmd.Wait()
+	if err := f.events.Encode(event{Role: pluginRole, Kind: "reaped", PID: ready.PID}); err != nil {
+		return err
+	}
+	if ctx.Err() != nil {
+		return status.FromContextError(ctx.Err()).Err()
+	}
+	return waitErr
+}
+
+func host(observer string, ignore bool) error {
+	executable, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	// #nosec G204 -- Self-executable fixture, with arguments controlled by the test harness.
+	cmd := exec.Command(executable, "--mode", pluginRole, "--observer", observer,
+		fmt.Sprintf("--ignore-interrupt=%t", ignore))
+	client := hplugin.NewClient(&hplugin.ClientConfig{
+		HandshakeConfig: sdkplugin.Handshake(),
+		VersionedPlugins: map[int]hplugin.PluginSet{
+			sdkplugin.ProtocolVersion: sdkplugin.ClientPlugins(),
+		},
+		AllowedProtocols: []hplugin.Protocol{hplugin.ProtocolGRPC},
+		Cmd:              cmd,
+		StartTimeout:     10 * time.Second,
+		Logger:           hclog.NewNullLogger(),
+		Stderr:           os.Stderr,
+		SyncStderr:       os.Stderr,
+	})
+	defer client.Kill()
+	rpc, err := client.Client()
+	if err != nil {
+		return err
+	}
+	raw, err := rpc.Dispense(sdkplugin.Name)
+	if err != nil {
+		return err
+	}
+	driver, ok := raw.(runtimev1.RuntimeDriverClient)
+	if !ok {
+		return errors.New("unexpected runtime client")
+	}
+	stream, err := driver.Exec(context.Background())
+	if err != nil {
+		return err
+	}
+	if err := stream.Send(&runtimev1.ExecClientMessage{Payload: &runtimev1.ExecClientMessage_Start{
+		Start: &runtimev1.ExecStart{WorkspaceId: "process-probe", Argv: []string{"block"}},
+	}}); err != nil {
+		return err
+	}
+	_, err = stream.Recv()
+	return err
+}

internal/processfixture/observer.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"encoding/json"
+	"net"
+	"os"
+	"sync"
+)
+
+// One writer serializes RPC lifecycle events and independent liveness replies.
+type reporter struct {
+	mu      sync.Mutex
+	encoder *json.Encoder
+	role    string
+}
+
+func (r *reporter) Encode(e event) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.encoder.Encode(e)
+}
+
+func (r *reporter) respond(conn net.Conn, disconnected chan<- struct{}) {
+	defer close(disconnected)
+	decoder := json.NewDecoder(conn)
+	for {
+		var e event
+		if err := decoder.Decode(&e); err != nil {
+			return
+		}
+		if e.Kind != "ping" {
+			return
+		}
+		if err := r.Encode(event{Role: r.role, Kind: "alive", PID: os.Getpid()}); err != nil {
+			return
+		}
+	}
+}