diff --git a/bgworker_test.go b/bgworker_test.go new file mode 100644 index 0000000000..ffa7904f0c --- /dev/null +++ b/bgworker_test.go @@ -0,0 +1,746 @@ +package frankenphp_test + +import ( + "bytes" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/dunglas/frankenphp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// requireFileEventually asserts that `path` appears on disk before the deadline +func requireFileEventually(t testing.TB, path string, msgAndArgs ...any) { + t.Helper() + require.Eventually(t, func() bool { + _, err := os.Stat(path) + return err == nil + }, 5*time.Second, 25*time.Millisecond, msgAndArgs...) +} + +// requireFileContentEventually waits for `path` to appear with content +func requireFileContentEventually(t *testing.T, path string) string { + t.Helper() + require.Eventually(t, func() bool { + b, err := os.ReadFile(path) + return err == nil && len(b) > 0 + }, 5*time.Second, 25*time.Millisecond, "file %q did not appear", path) + b, err := os.ReadFile(path) + require.NoError(t, err) + + return string(b) +} + +// asserts on the timing of Shutdown(), so it calls it itself rather than +// through initServers' t.Cleanup hook +func TestBackgroundWorkerLifecycle(t *testing.T) { + tmp := t.TempDir() + sentinel := filepath.Join(tmp, "bg-lifecycle.sentinel") + + t.Cleanup(frankenphp.Shutdown) + require.NoError(t, frankenphp.Init( + frankenphp.WithWorkers("bg-lifecycle", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + )) + + requireFileEventually(t, sentinel, "background worker did not touch sentinel") + + done := make(chan struct{}) + go func() { + frankenphp.Shutdown() + close(done) + }() + + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatalf("Shutdown did not return within 10s") + } +} + +func TestBackgroundWorkerCrashRestarts(t *testing.T) { + tmp := t.TempDir() + crashMarker := filepath.Join(tmp, "bg-crash.marker") + restarted := filepath.Join(tmp, "bg-crash.restarted") + + initServers(t, + frankenphp.WithWorkers("bg-crash", "testdata/bgworker/crash.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{ + "BG_CRASH_MARKER": crashMarker, + "BG_RESTARTED_SENTINEL": restarted, + }), + ), + frankenphp.WithNumThreads(2), + ) + + requireFileEventually(t, restarted, "background worker did not restart after crash") +} + +// the sentinel directory is declared on the server, not on the worker +func TestBackgroundWorkerOnServer(t *testing.T) { + tmp := t.TempDir() + + server, err := frankenphp.NewServer( + testDataDir, + frankenphp.WithServerName("sidekick-server"), + frankenphp.WithServerEnv(map[string]string{"BG_SENTINEL_DIR": tmp}), + ) + require.NoError(t, err) + + initServers(t, + frankenphp.WithServer(server), + frankenphp.WithWorkers("jobs", "testdata/bgworker/named.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(server), + ), + frankenphp.WithNumThreads(2), + ) + + // named.php touches "/" + requireFileEventually(t, filepath.Join(tmp, "jobs"), "background worker did not touch its per-name sentinel") + + body := serverGet(t, server, "http://example.com/index.php") + assert.Contains(t, body, "I am by birth a Genevese", "the server must still serve regular requests") +} + +func TestBackgroundWorkerValidation(t *testing.T) { + t.Cleanup(frankenphp.Shutdown) + + t.Run("early return without the handle fails startup", func(t *testing.T) { + err := frankenphp.Init( + frankenphp.WithWorkers("bg-early", "testdata/bgworker/early-return.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerMaxFailures(2), + ), + frankenphp.WithNumThreads(2), + ) + require.ErrorContains(t, err, "without calling WorkerHandle::tick()") + }) + + t.Run("fetching the handle without ticking fails startup", func(t *testing.T) { + err := frankenphp.Init( + frankenphp.WithWorkers("bg-no-tick", "testdata/bgworker/fetch-no-tick.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerMaxFailures(2), + ), + frankenphp.WithNumThreads(2), + ) + require.ErrorContains(t, err, "without calling WorkerHandle::tick()") + }) + + t.Run("max_threads is rejected", func(t *testing.T) { + err := frankenphp.Init( + frankenphp.WithWorkers("bg-scaled", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerMaxThreads(2), + ), + frankenphp.WithNumThreads(2), + ) + require.ErrorContains(t, err, "cannot set max_threads") + }) + + t.Run("request matchers are rejected", func(t *testing.T) { + err := frankenphp.Init( + frankenphp.WithWorkers("bg-matched", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerMatcher(func(*http.Request) bool { return true }), + ), + frankenphp.WithNumThreads(2), + ) + require.ErrorContains(t, err, "cannot match requests") + }) +} + +func TestBackgroundWorkerCannotHandleRequests(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, + frankenphp.WithServer(server), + frankenphp.WithWorkers("jobs", "testdata/bgworker/basic.php", 1, frankenphp.WithWorkerBackground(), frankenphp.WithWorkerServerScope(server)), + frankenphp.WithNumThreads(2), + ) + + err = server.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "http://example.com/index.php", nil), frankenphp.WithWorkerName("jobs")) + require.ErrorContains(t, err, `background worker "jobs" cannot handle requests`) +} + +// a blocking read is a wait too: it reports the worker ready, and the EOF of +// the drain unblocks it +func TestBackgroundWorkerParksOnRead(t *testing.T) { + tmp := t.TempDir() + sentinel := filepath.Join(tmp, "bg-read.sentinel") + + t.Cleanup(frankenphp.Shutdown) + require.NoError(t, frankenphp.Init( + frankenphp.WithWorkers("bg-read", "testdata/bgworker/read.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + )) + requireFileEventually(t, sentinel, "background worker parked on a read did not start") + + done := make(chan struct{}) + go func() { + frankenphp.Shutdown() + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Shutdown did not return within 10s: the read did not observe EOF") + } +} + +// a blocking receive reaches the stream through the transport API rather than +// the read op, so Init() would hang if only reads reported readiness +func TestBackgroundWorkerParksOnReceive(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "bg-recv.sentinel") + + t.Cleanup(frankenphp.Shutdown) + require.NoError(t, frankenphp.Init( + frankenphp.WithWorkers("bg-recv", "testdata/bgworker/recv.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + )) + requireFileEventually(t, sentinel, "background worker parked on a receive did not start") + + done := make(chan struct{}) + go func() { + frankenphp.Shutdown() + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Shutdown did not return within 10s: the receive did not observe EOF") + } +} + +func TestBackgroundWorkerRestartDrainsParkedScript(t *testing.T) { + tmp := t.TempDir() + countFile := filepath.Join(tmp, "bg-count.log") + + initServers(t, + frankenphp.WithWorkers("bg-count", "testdata/bgworker/count.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_COUNT_FILE": countFile}), + ), + frankenphp.WithNumThreads(2), + ) + runs := func() int { + b, _ := os.ReadFile(countFile) + return bytes.Count(b, []byte("\n")) + } + require.Eventually(t, func() bool { return runs() == 1 }, 5*time.Second, 25*time.Millisecond, "background worker did not start") + + frankenphp.RestartWorkers() + + require.Eventually(t, func() bool { return runs() == 2 }, 5*time.Second, 25*time.Millisecond, "background worker was not re-run after the restart") +} + +func TestWorkerHandleOutsideBackgroundWorker(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), frankenphp.WithNumThreads(1)) + + body := serverGet(t, server, "http://example.com/handle-outside.php") + + assert.Contains(t, body, `FrankenPHP\WorkerHandle can only be created from a background worker`) +} + +// without the wake-up sent at start, a script that only ticks when its handle +// is readable would keep Init() waiting until the drain +func TestBackgroundWorkerLoopTicksOnItsOwn(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "loop.sentinel") + initServers(t, + frankenphp.WithWorkers("bg-loop", "testdata/bgworker/loop.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + ) + requireFileEventually(t, sentinel, "background worker did not start") +} + +// the handle is readable at start and quiet once a tick returned, so a loop +// selecting on it blocks instead of spinning +func TestBackgroundWorkerTickLeavesTheHandleQuiet(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "readable.txt") + initServers(t, + frankenphp.WithWorkers("bg-readable", "testdata/bgworker/readable.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + ) + + assert.Equal(t, "start:readable after tick:quiet after second tick:quiet", requireFileContentEventually(t, sentinel)) +} + +// true while the worker runs, false once it is drained, and still false on the +// next call +func TestBackgroundWorkerTick(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "ticks.txt") + + t.Cleanup(frankenphp.Shutdown) + require.NoError(t, frankenphp.Init( + frankenphp.WithWorkers("bg-tick", "testdata/bgworker/tick.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + )) + assert.Equal(t, "true true", requireFileContentEventually(t, sentinel)) + + // Shutdown() waits for the script to leave, so the file is final after it + frankenphp.Shutdown() + b, err := os.ReadFile(sentinel) + require.NoError(t, err) + assert.Equal(t, "true true false false", string(b)) +} + +// an HTTP worker sees FRANKENPHP_WORKER as it always did, a background one its +// declared name in FRANKENPHP_WORKER_BACKGROUND and no FRANKENPHP_WORKER +func TestWorkerNameInServerVars(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "flag.txt") + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, + frankenphp.WithServer(server), + frankenphp.WithWorkers("web", testDataDir+"worker-name.php", 1, frankenphp.WithWorkerServerScope(server)), + frankenphp.WithWorkers("jobs", "testdata/bgworker/flag.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(server), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(3), + ) + + assert.Equal(t, "1 http", serverGet(t, server, "http://example.com/worker-name.php")) + + flag := requireFileContentEventually(t, sentinel) + assert.Contains(t, flag, "'worker' => 'unset'") + assert.Contains(t, flag, "'background' => 'jobs'") +} + +// the threads of a pool share the name, park on a handle each, and one drain +// wakes them all +func TestBackgroundWorkerPool(t *testing.T) { + dir := t.TempDir() + initServers(t, + frankenphp.WithWorkers("pool", "testdata/bgworker/pool.php", 3, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL_DIR": dir}), + ), + frankenphp.WithNumThreads(4), + ) + + require.Eventually(t, func() bool { + entries, _ := os.ReadDir(dir) + return len(entries) == 3 + }, 5*time.Second, 25*time.Millisecond, "the three pool threads did not all start") + + done := make(chan struct{}) + go func() { + frankenphp.Shutdown() + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Shutdown did not drain the whole pool within 10s") + } +} + +// two named background workers of one server may share a script, they are not +// matched by path +func TestBackgroundWorkerMultiEntrypoint(t *testing.T) { + tmp := t.TempDir() + first, second := filepath.Join(tmp, "first"), filepath.Join(tmp, "second") + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, + frankenphp.WithServer(server), + frankenphp.WithWorkers("first", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(server), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": first}), + ), + frankenphp.WithWorkers("second", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(server), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": second}), + ), + frankenphp.WithNumThreads(3), + ) + + requireFileEventually(t, first, "the first worker on the shared script did not start") + requireFileEventually(t, second, "the second worker on the shared script did not start") +} + +// a name left out is generated from the script, like an HTTP worker's +func TestBackgroundWorkerNameIsGenerated(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "flag.txt") + initServers(t, + frankenphp.WithWorkers("", "testdata/bgworker/flag.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(1), + ) + + script, err := filepath.Abs("testdata/bgworker/flag.php") + require.NoError(t, err) + // var_export() doubles the backslashes of a Windows path + assert.Contains(t, requireFileContentEventually(t, sentinel), "'background' => '"+strings.ReplaceAll(script, `\`, `\\`)+"'") +} + +// a background worker serves no requests, so num defaults to one thread rather +// than to the CPU count of an HTTP worker +func TestBackgroundWorkerDefaultsToOneThread(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "bg-default.sentinel") + initServers(t, + frankenphp.WithWorkers("bg-default", "testdata/bgworker/basic.php", 0, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(1), + ) + requireFileEventually(t, sentinel, "background worker did not start without an explicit num") + + debug := frankenphp.DebugState() + background := 0 + for _, thread := range debug.ThreadDebugStates { + if strings.Contains(thread.Name, "Background Worker") { + background++ + } + } + assert.Equal(t, 1, background) + // on top of num_threads, which keeps its regular thread + assert.Len(t, debug.ThreadDebugStates, 2) +} + +// max_execution_time applies until the first tick, a setup that outlives it +// fails Init(). The limit is PHP's, so the test only runs where its timers are +// known to fire under FrankenPHP: the max execution timers of Linux ZTS builds +func TestBackgroundWorkerBootstrapIsBounded(t *testing.T) { + if !frankenphp.Config().ZendMaxExecutionTimers { + t.Skip("max_execution_time is only reliable with Zend max execution timers") + } + + countFile := filepath.Join(t.TempDir(), "boots") + err := frankenphp.Init( + frankenphp.WithWorkers("bg-slow", "testdata/bgworker/slow-boot.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerMaxFailures(0), + frankenphp.WithWorkerEnv(map[string]string{"BG_COUNT_FILE": countFile}), + ), + frankenphp.WithNumThreads(2), + frankenphp.WithPhpIni(map[string]string{"max_execution_time": "1"}), + ) + if err == nil { + frankenphp.Shutdown() + } + require.ErrorContains(t, err, "keeps crashing") + + b, _ := os.ReadFile(countFile) + assert.Equal(t, 1, bytes.Count(b, []byte("\n")), "the limit should have ended the one and only boot") +} + +// past its first tick, a parked script is cut short by neither of the two +// limits it never disables itself: max_execution_time, disarmed by the tick, +// and default_socket_timeout, which the handle's stream overrides +func TestBackgroundWorkerParkingIsNotInterrupted(t *testing.T) { + countFile := filepath.Join(t.TempDir(), "runs") + initServers(t, + frankenphp.WithWorkers("bg-timeout", "testdata/bgworker/no-time-limit.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_COUNT_FILE": countFile}), + ), + frankenphp.WithNumThreads(2), + frankenphp.WithPhpIni(map[string]string{"max_execution_time": "1", "max_input_time": "1", "default_socket_timeout": "1"}), + ) + + runs := func() int { + b, _ := os.ReadFile(countFile) + + return bytes.Count(b, []byte("\n")) + } + require.Eventually(t, func() bool { return runs() == 1 }, 5*time.Second, 25*time.Millisecond, "background worker did not start") + // well past both limits + time.Sleep(2500 * time.Millisecond) + assert.Equal(t, 1, runs(), "the worker was restarted, so a limit interrupted its park") +} + +// the same stream every time, a fresh one once the script closed it, one per +// handle, and the drain reaches the script through all of them +func TestBackgroundWorkerStreamClosedAndFetchedAgain(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "refetch.txt") + + t.Cleanup(frankenphp.Shutdown) + require.NoError(t, frankenphp.Init( + frankenphp.WithWorkers("bg-refetch", "testdata/bgworker/refetch.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + )) + assert.Equal(t, "same then fresh then its own", requireFileContentEventually(t, sentinel)) + + done := make(chan struct{}) + go func() { + frankenphp.Shutdown() + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Shutdown did not return within 10s: the re-fetched handle missed the drain") + } +} + +// boot failures below max_consecutive_failures are retried with the backoff, +// and Init() still succeeds once a run reaches its ready point +func TestBackgroundWorkerBootFailuresThenSucceeds(t *testing.T) { + tmp := t.TempDir() + countFile, sentinel := filepath.Join(tmp, "boots"), filepath.Join(tmp, "ready") + initServers(t, + frankenphp.WithWorkers("bg-flaky", "testdata/bgworker/fail-then-succeed.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_COUNT_FILE": countFile, "BG_SENTINEL": sentinel, "BG_FAIL_UNTIL": "2"}), + ), + frankenphp.WithNumThreads(2), + ) + + requireFileEventually(t, sentinel, "background worker did not recover from its boot failures") + boots, err := os.ReadFile(countFile) + require.NoError(t, err) + assert.Equal(t, "3", string(boots), "two boot failures then a success") +} + +// a crash past the ready point restarts without counting toward +// max_consecutive_failures, and a drain cuts the backoff short +func TestBackgroundWorkerCrashAfterReadyRestarts(t *testing.T) { + countFile := filepath.Join(t.TempDir(), "runs") + initServers(t, + frankenphp.WithWorkers("bg-crashy", "testdata/bgworker/crash-after-ready.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerMaxFailures(2), + frankenphp.WithWorkerEnv(map[string]string{"BG_COUNT_FILE": countFile}), + ), + frankenphp.WithNumThreads(2), + ) + + require.Eventually(t, func() bool { + b, _ := os.ReadFile(countFile) + return bytes.Count(b, []byte("\n")) >= 5 + }, 5*time.Second, 25*time.Millisecond, "the worker was not restarted after crashing past its ready point") + + // past four crashes the wait is at its 1s cap + start := time.Now() + frankenphp.Shutdown() + assert.Less(t, time.Since(start), 500*time.Millisecond, "Shutdown() waited for the backoff") +} + +// a script returning right after its tick is re-run with the crash backoff +func TestBackgroundWorkerCleanExitIsPaced(t *testing.T) { + countFile := filepath.Join(t.TempDir(), "runs") + initServers(t, + frankenphp.WithWorkers("bg-exit", "testdata/bgworker/exit-after-tick.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_COUNT_FILE": countFile}), + ), + frankenphp.WithNumThreads(2), + ) + + time.Sleep(time.Second) + b, err := os.ReadFile(countFile) + require.NoError(t, err) + runs := bytes.Count(b, []byte("\n")) + assert.GreaterOrEqual(t, runs, 2, "the script was not re-run") + assert.LessOrEqual(t, runs, 8, "the re-runs were not paced") +} + +// the other side of the pacing: a worker that does some work and returns is +// re-run at its own pace +func TestBackgroundWorkerCyclingIsNotThrottled(t *testing.T) { + countFile := filepath.Join(t.TempDir(), "runs") + initServers(t, + frankenphp.WithWorkers("bg-cycle", "testdata/bgworker/cycle.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_COUNT_FILE": countFile}), + ), + frankenphp.WithNumThreads(2), + ) + + // the interval between two runs is the work plus whatever FrankenPHP + // waits: comparing the last to the first cancels how fast the machine + // is, and a backoff counting these runs would have grown it by a second + starts := func() []float64 { + b, _ := os.ReadFile(countFile) + var times []float64 + for _, line := range strings.Fields(string(b)) { + if t, err := strconv.ParseFloat(line, 64); err == nil { + times = append(times, t) + } + } + + return times + } + require.Eventually(t, func() bool { return len(starts()) >= 6 }, 15*time.Second, 50*time.Millisecond, "the worker did not run six times") + + times := starts() + first, last := times[1]-times[0], times[5]-times[4] + assert.Less(t, last-first, 0.5, "the interval between runs grew, the worker was throttled") +} + +// a script ignoring its handle does not stall RestartWorkers() past the reboot +// grace period, the force-kill ends it +func TestBackgroundWorkerRebootForceKillsStuckScript(t *testing.T) { + if runtime.GOOS != "linux" && runtime.GOOS != "freebsd" { + t.Skipf("force-kill cannot interrupt a blocking syscall on %s", runtime.GOOS) + } + + tmp := t.TempDir() + once, sentinel := filepath.Join(tmp, "once"), filepath.Join(tmp, "parked") + initServers(t, + frankenphp.WithWorkers("bg-stuck", "testdata/bgworker/stuck.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_ONCE": once, "BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + ) + requireFileEventually(t, once, "background worker never entered its sleep") + + start := time.Now() + frankenphp.RestartWorkers() + assert.WithinDuration(t, start, time.Now(), 10*time.Second, "the reboot must force-kill the stuck script within its grace period") + + requireFileEventually(t, sentinel, "the re-run script did not park") +} + +// the poll API of PHP 8.6 waits on the handle itself, no stream involved +func TestBackgroundWorkerPollHandle(t *testing.T) { + if frankenphp.Version().VersionID < 80600 { + t.Skip("the poll API needs PHP 8.6") + } + + sentinel := filepath.Join(t.TempDir(), "poll.backend") + initServers(t, + frankenphp.WithWorkers("bg-poll", "testdata/bgworker/poll.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + ) + + // the server only starts once the worker ticked, which it does through + // the context: a backend name means the wait came back on its own + assert.NotEmpty(t, requireFileContentEventually(t, sentinel)) +} + +// the handle implements Io\Poll\Handle on every version: PHP 8.6 declares the +// interface with the poll API, FrankenPHP declares it below that +func TestBackgroundWorkerHandleIsAPollHandle(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "poll.json") + initServers(t, + frankenphp.WithWorkers("bg-iface", "testdata/bgworker/poll-interface.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + ) + + assert.JSONEq(t, `{"handle":true,"internal":true}`, requireFileContentEventually(t, sentinel)) +} + +// the methods do not trust the constructor's guard: unserialize() is refused +// outright, and a handle Reflection built on a request thread throws instead +// of reaching the Go side, which would panic and take the process with it +func TestWorkerHandleBuiltBehindTheConstructor(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), frankenphp.WithNumThreads(1)) + + body := serverGet(t, server, "http://example.com/handle-bypass.php") + assert.Contains(t, body, "unserialize: Exception: Unserialization of 'FrankenPHP\\WorkerHandle' is not allowed") + assert.Contains(t, body, "reflection: ReflectionException:") + assert.Contains(t, body, "construct: RuntimeException: FrankenPHP\\WorkerHandle can only be created from a background worker") +} + +// a script that parks without ever ticking fails Init() instead of hanging it, +// the only failure mode available where Zend max execution timers are not +func TestBackgroundWorkerBootTimeout(t *testing.T) { + // a hang fails the test instead of the whole run + initWithin := func(t *testing.T, options ...frankenphp.Option) error { + t.Helper() + done := make(chan error, 1) + go func() { done <- frankenphp.Init(options...) }() + + select { + case err := <-done: + if err == nil { + frankenphp.Shutdown() + } + + return err + case <-time.After(10 * time.Second): + t.Fatal("Init() did not return") + + return nil + } + } + mute := func(name string, timeout time.Duration, env map[string]string) frankenphp.Option { + return frankenphp.WithWorkers(name, "testdata/bgworker/never-ticks.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerBootTimeout(timeout), + frankenphp.WithWorkerEnv(env), + ) + } + + t.Run("one worker", func(t *testing.T) { + err := initWithin(t, mute("bg-mute", 300*time.Millisecond, nil), frankenphp.WithNumThreads(2)) + require.ErrorContains(t, err, `background worker "bg-mute" did not call WorkerHandle::tick() within 300ms`) + }) + + // each timeout reports once and never waits for room to do so + t.Run("two workers timing out one after the other", func(t *testing.T) { + err := initWithin(t, + mute("bg-mute-a", 200*time.Millisecond, nil), + mute("bg-mute-b", time.Second, nil), + frankenphp.WithNumThreads(1), + ) + require.ErrorContains(t, err, `background worker "bg-mute-a" did not call WorkerHandle::tick() within 200ms`) + }) + + // a timeout landing between two failed boots stops the next one from starting + t.Run("timeout between two failed boots", func(t *testing.T) { + countFile := filepath.Join(t.TempDir(), "boots") + err := initWithin(t, + mute("bg-flaky", 300*time.Millisecond, map[string]string{"BG_COUNT_FILE": countFile, "BG_FAIL_UNTIL": "3"}), + frankenphp.WithNumThreads(1), + ) + require.ErrorContains(t, err, `background worker "bg-flaky" did not call WorkerHandle::tick() within 300ms`) + + b, err := os.ReadFile(countFile) + require.NoError(t, err) + boots, err := strconv.Atoi(string(b)) + require.NoError(t, err) + assert.LessOrEqual(t, boots, 3, "a run started after the timeout") + }) +} diff --git a/caddy/app.go b/caddy/app.go index 266322d788..1ef018362b 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -410,10 +410,13 @@ func (f *FrankenPHPApp) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { if len(wc.MatchPath) != 0 { return d.Errf(`"match" can only be used in a php_server worker block, not in a global one: %q`, wc.FileName) } - // check for duplicate workers - for _, existingWorker := range f.Workers { - if existingWorker.FileName == wc.FileName { - return d.Errf("global workers must not have duplicate filenames: %q", wc.FileName) + // check for duplicate workers; background workers are not + // matched by path, several may share a script + if !wc.Background { + for _, existingWorker := range f.Workers { + if !existingWorker.Background && existingWorker.FileName == wc.FileName { + return d.Errf("global workers must not have duplicate filenames: %q", wc.FileName) + } } } diff --git a/caddy/caddy_test.go b/caddy/caddy_test.go index 945f1a3d68..5424b9943b 100644 --- a/caddy/caddy_test.go +++ b/caddy/caddy_test.go @@ -788,6 +788,45 @@ func TestMetrics(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(ctx.GetMetricsRegistry(), strings.NewReader(expectedMetrics), "frankenphp_total_threads", "frankenphp_busy_threads")) } +// TestBackgroundWorkerFromCaddyfile starts a background worker from a +// Caddyfile and checks it runs: the sentinel its script touches appears +func TestBackgroundWorkerFromCaddyfile(t *testing.T) { + sentinel := filepath.ToSlash(filepath.Join(t.TempDir(), "bg.sentinel")) + tester := caddytest.NewTester(t) + initServer(t, tester, ` + { + skip_install_trust + admin localhost:2999 + http_port `+testPort+` + https_port 9443 + + frankenphp { + worker { + file ../testdata/bgworker/basic.php + num 1 + name bg-caddy + background + env BG_SENTINEL `+sentinel+` + } + } + } + + localhost:`+testPort+` { + route { + php { + root ../testdata + } + } + } + `, "caddyfile") + + require.Eventually(t, func() bool { + _, err := os.Stat(sentinel) + + return err == nil + }, 5*time.Second, 25*time.Millisecond, "the background worker declared in the Caddyfile did not run") +} + func TestWorkerMetrics(t *testing.T) { var wg sync.WaitGroup tester := caddytest.NewTester(t) @@ -870,7 +909,7 @@ func TestWorkerMetrics(t *testing.T) { # TYPE frankenphp_worker_request_count counter frankenphp_worker_request_count{worker="` + workerName + `"} 10 - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, WorkerHandle::tick() for background workers # TYPE frankenphp_ready_workers gauge frankenphp_ready_workers{worker="` + workerName + `"} 2 ` @@ -1027,7 +1066,7 @@ func TestNamedWorkerMetrics(t *testing.T) { # TYPE frankenphp_worker_request_count counter frankenphp_worker_request_count{worker="my_app"} 10 - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, WorkerHandle::tick() for background workers # TYPE frankenphp_ready_workers gauge frankenphp_ready_workers{worker="my_app"} 2 ` @@ -1123,7 +1162,7 @@ func TestAutoWorkerConfig(t *testing.T) { # TYPE frankenphp_worker_request_count counter frankenphp_worker_request_count{worker="` + workerName + `"} 10 - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, WorkerHandle::tick() for background workers # TYPE frankenphp_ready_workers gauge frankenphp_ready_workers{worker="` + workerName + `"} ` + workers + ` ` @@ -1498,7 +1537,7 @@ func TestMultiWorkersMetrics(t *testing.T) { # TYPE frankenphp_worker_request_count counter frankenphp_worker_request_count{worker="service1"} 10 - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, WorkerHandle::tick() for background workers # TYPE frankenphp_ready_workers gauge frankenphp_ready_workers{worker="service1"} 2 frankenphp_ready_workers{worker="service2"} 3 @@ -1652,7 +1691,7 @@ func TestWorkerRestart(t *testing.T) { // Check metrics expectedMetrics := ` - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, WorkerHandle::tick() for background workers # TYPE frankenphp_ready_workers gauge frankenphp_ready_workers{worker="service"} 1 # HELP frankenphp_total_workers Total number of PHP workers for this worker @@ -1680,7 +1719,7 @@ func TestWorkerRestart(t *testing.T) { // frankenphp_ready_workers should be back to 1 even after worker restarts expectedMetrics = ` - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, WorkerHandle::tick() for background workers # TYPE frankenphp_ready_workers gauge frankenphp_ready_workers{worker="service"} 1 # HELP frankenphp_total_workers Total number of PHP workers for this worker diff --git a/caddy/config_test.go b/caddy/config_test.go index 041c2a47bc..94db973fb9 100644 --- a/caddy/config_test.go +++ b/caddy/config_test.go @@ -92,6 +92,45 @@ func TestModuleWorkerDuplicateFilenamesFail(t *testing.T) { require.Contains(t, err.Error(), "must not have duplicate filenames", "Error message should mention duplicate filenames") } +// two global background workers may share a script, like their php_server +// counterparts and the Go API: they are not matched by path +func TestGlobalBackgroundWorkersShareAFilename(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + frankenphp { + worker { + name first + file ../testdata/worker-with-env.php + num 1 + background + } + worker { + name second + file ../testdata/worker-with-env.php + num 1 + background + } + } + }`) + app := &FrankenPHPApp{} + + require.NoError(t, app.UnmarshalCaddyfile(d)) + require.Len(t, app.Workers, 2) +} + +func TestGlobalWorkerDuplicateFilenamesFail(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + frankenphp { + worker ../testdata/worker-with-env.php + worker ../testdata/worker-with-env.php + } + }`) + app := &FrankenPHPApp{} + + require.ErrorContains(t, app.UnmarshalCaddyfile(d), "must not have duplicate filenames") +} + func TestModuleWorkersWithDifferentFilenames(t *testing.T) { // Create a test configuration with different worker filenames configWithDifferentFilenames := ` @@ -295,3 +334,151 @@ func TestCreateUniqueWorkerNamesQualifiedByServer(t *testing.T) { // workers without a server keep the numeric postfix behavior require.Equal(t, "queue_2", createUniqueWorkerName(usedWorkerNames, wc, "")) } + +func TestWorkerBackgroundConfig(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + php_server { + worker { + name jobs + file ../testdata/worker-with-env.php + num 2 + background + } + } + }`) + module := &FrankenPHPModule{} + + require.NoError(t, module.UnmarshalCaddyfile(d)) + require.Len(t, module.Workers, 1) + require.True(t, module.Workers[0].Background) + require.Equal(t, "jobs", module.Workers[0].Name) +} + +// the name is generated when the workers are collected, like for an HTTP worker +func TestWorkerBackgroundNameIsOptional(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + php_server { + worker { + file ../testdata/worker-with-env.php + background + } + } + }`) + module := &FrankenPHPModule{} + + require.NoError(t, module.UnmarshalCaddyfile(d)) + require.True(t, module.Workers[0].Background) + require.Empty(t, module.Workers[0].Name) +} + +func TestWorkerBackgroundVars(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + php_server { + worker { + file ../testdata/worker-with-env.php + background + vars redis feature_flags + } + } + }`) + module := &FrankenPHPModule{} + + require.NoError(t, module.UnmarshalCaddyfile(d)) + require.Equal(t, []string{"redis", "feature_flags"}, module.Workers[0].Vars) +} + +func TestWorkerVarsRequireBackground(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + php_server { + worker { + file ../testdata/worker-with-env.php + vars redis + } + } + }`) + module := &FrankenPHPModule{} + + require.ErrorContains(t, module.UnmarshalCaddyfile(d), `"vars" is only supported for background workers`) +} + +func TestWorkerBackgroundWithoutNumParses(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + php_server { + worker { + name jobs + file ../testdata/worker-with-env.php + background + } + } + }`) + module := &FrankenPHPModule{} + + // num is optional, it defaults to one thread when the workers start + require.NoError(t, module.UnmarshalCaddyfile(d)) +} + +func TestWorkerBootTimeout(t *testing.T) { + parse := func(t *testing.T, directives string) (*FrankenPHPModule, error) { + t.Helper() + module := &FrankenPHPModule{} + err := module.UnmarshalCaddyfile(caddyfile.NewTestDispenser(` + { + php_server { + worker { + file ../testdata/worker-with-env.php + ` + directives + ` + } + } + }`)) + + return module, err + } + + module, err := parse(t, "background\nboot_timeout 5s") + require.NoError(t, err) + require.Equal(t, caddy.Duration(5*time.Second), *module.Workers[0].BootTimeout) + + // zero waits forever + module, err = parse(t, "background\nboot_timeout 0") + require.NoError(t, err) + require.Equal(t, caddy.Duration(0), *module.Workers[0].BootTimeout) + + _, err = parse(t, "boot_timeout 5s") + require.ErrorContains(t, err, `"boot_timeout" is only supported for background workers`) + + _, err = parse(t, "background\nboot_timeout -1s") + require.ErrorContains(t, err, "boot_timeout must be >= 0") +} + +func TestWorkerBackgroundRejectsMatch(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + php_server { + worker { + name jobs + file ../testdata/worker-with-env.php + match /jobs/* + background + } + } + }`) + module := &FrankenPHPModule{} + + err := module.UnmarshalCaddyfile(d) + require.ErrorContains(t, err, `"match" is not supported for background workers`) +} + +// a JSON config skips the Caddyfile check: the core refuses vars on an HTTP +// worker instead of the option being dropped +func TestWorkerVarsReachTheCoreWithoutBackground(t *testing.T) { + wc := workerConfig{FileName: "../testdata/worker-with-env.php", Vars: []string{"cfg"}} + opts, err := wc.toWorkerOptions() + require.NoError(t, err) + + require.ErrorContains(t, frankenphp.Validate(frankenphp.WithWorkers("http", wc.FileName, 1, opts...)), "only background workers publish them") +} diff --git a/caddy/module.go b/caddy/module.go index 20dcec9ee2..0bab580086 100644 --- a/caddy/module.go +++ b/caddy/module.go @@ -315,6 +315,10 @@ func (f *FrankenPHPModule) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // Check if a worker with this filename already exists in this module fileNames := make(map[string]struct{}, len(f.Workers)) for _, w := range f.Workers { + // background workers are not matched by path, several may share a script + if w.Background { + continue + } if _, ok := fileNames[w.FileName]; ok { return fmt.Errorf(`workers in a single "php" or "php_server" block must not have duplicate filenames: %q`, w.FileName) } diff --git a/caddy/workerconfig.go b/caddy/workerconfig.go index f229c71487..38bc16f4e4 100644 --- a/caddy/workerconfig.go +++ b/caddy/workerconfig.go @@ -3,6 +3,7 @@ package caddy import ( "path/filepath" "strconv" + "time" "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" @@ -38,6 +39,12 @@ type workerConfig struct { MatchPath []string `json:"match_path,omitempty"` // MaxConsecutiveFailures sets the maximum number of consecutive failures before panicking (defaults to 6, set to -1 to never panick) MaxConsecutiveFailures *int `json:"max_consecutive_failures,omitempty"` + // BootTimeout bounds how long a background worker may take to reach its first WorkerHandle::tick() before the start fails (defaults to 30s on PHP builds without Zend max execution timers, where max_execution_time is disabled, set to 0 to wait forever) + BootTimeout *caddy.Duration `json:"boot_timeout,omitempty"` + // Background marks this worker as a background (non-HTTP) worker. + Background bool `json:"background,omitempty"` + // Vars lists the namespaces a background worker publishes with WorkerHandle::setVars(), read by frankenphp_get_vars(). + Vars []string `json:"vars,omitempty"` options []frankenphp.WorkerOption } @@ -139,8 +146,31 @@ func unmarshalWorker(d *caddyfile.Dispenser) (workerConfig, error) { } wc.MaxConsecutiveFailures = new(v) + case "boot_timeout": + if !d.NextArg() { + return wc, d.ArgErr() + } + + v, err := caddy.ParseDuration(d.Val()) + if err != nil { + return wc, d.WrapErr(err) + } + if v < 0 { + return wc, d.Errf("boot_timeout must be >= 0") + } + + timeout := caddy.Duration(v) + wc.BootTimeout = &timeout + case "background": + wc.Background = true + case "vars": + namespaces := d.RemainingArgs() + if len(namespaces) == 0 { + return wc, d.ArgErr() + } + wc.Vars = append(wc.Vars, namespaces...) default: - return wc, wrongSubDirectiveError("worker", "name, file, num, env, watch, match, max_consecutive_failures, max_threads", v) + return wc, wrongSubDirectiveError("worker", "name, file, num, env, watch, match, max_consecutive_failures, max_threads, background, boot_timeout, vars", v) } } @@ -148,6 +178,18 @@ func unmarshalWorker(d *caddyfile.Dispenser) (workerConfig, error) { return wc, d.Err(`the "file" argument must be specified`) } + if !wc.Background && wc.BootTimeout != nil { + return wc, d.Err(`"boot_timeout" is only supported for background workers`) + } + + if wc.Background && len(wc.MatchPath) != 0 { + return wc, d.Err(`"match" is not supported for background workers`) + } + + if !wc.Background && len(wc.Vars) != 0 { + return wc, d.Err(`"vars" is only supported for background workers`) + } + if frankenphp.EmbeddedAppPath != "" && filepath.IsLocal(wc.FileName) { wc.FileName = filepath.Join(frankenphp.EmbeddedAppPath, wc.FileName) } @@ -169,6 +211,20 @@ func (wc *workerConfig) toWorkerOptions() ([]frankenphp.WorkerOption, error) { // options collected while provisioning the module, e.g. the Mercure hub opts = append(opts, wc.options...) + if wc.Background { + opts = append(opts, frankenphp.WithWorkerBackground()) + + if wc.BootTimeout != nil { + opts = append(opts, frankenphp.WithWorkerBootTimeout(time.Duration(*wc.BootTimeout))) + } + } + + // whatever the kind of worker: the core refuses vars on an HTTP one, which + // a JSON config reaches without the Caddyfile check + if len(wc.Vars) > 0 { + opts = append(opts, frankenphp.WithWorkerVars(wc.Vars...)) + } + // copy the caddy match logic and create a unique matcher function for this worker // inject the matcher into frankenphp if len(wc.MatchPath) > 0 { diff --git a/docs/config.md b/docs/config.md index 3dc5a9cbbe..be8bf8cb02 100644 --- a/docs/config.md +++ b/docs/config.md @@ -106,11 +106,14 @@ You can also explicitly configure FrankenPHP using the [global option](https://c php_ini # Set a php.ini directive. Can be used several times to set multiple directives. worker { file # Sets the path to the worker script. - num # Sets the number of PHP threads to start, defaults to 2x the number of available CPUs. + num # Sets the number of PHP threads to start, defaults to 2x the number of available CPUs, 1 for background workers. env # Sets an extra environment variable to the given value. Can be specified more than once for multiple environment variables. watch # Sets the path to watch for file changes. Can be specified more than once for multiple paths. name # Sets the name of the worker, used in logs and metrics. Default: absolute path of worker file max_consecutive_failures # Sets the maximum number of consecutive failures before the worker is considered unhealthy, -1 means the worker will always restart. Default: 6. + boot_timeout # EXPERIMENTAL: how long a background worker may take to reach its first tick() before the start fails, 0 waits forever. Default: 30s on PHP builds without Zend max execution timers, none elsewhere. + background # EXPERIMENTAL: marks this worker as a background (non-HTTP) worker, whose script ticks its FrankenPHP\WorkerHandle at least once + vars # EXPERIMENTAL: the vars namespaces a background worker publishes, read with frankenphp_get_vars() } } } @@ -193,11 +196,14 @@ php_server [] { request_body_timeout # Sets an idle timeout on request body reads: a stalled (slow POST) client is cut off while a steady upload of any size succeeds. Default: 60s. Set to 0 to disable. worker { # Creates a worker specific to this server. Can be specified more than once for multiple workers. file # Sets the path to the worker script, can be relative to the php_server root - num # Sets the number of PHP threads to start, defaults to 2x the number of available + num # Sets the number of PHP threads to start, defaults to 2x the number of available CPUs, 1 for background workers. name # Sets the name for the worker, used in logs and metrics. Default: absolute path of worker file. Postfixed with a number if name is already in use. watch # Sets the path to watch for file changes. Can be specified more than once for multiple paths. env # Sets an extra environment variable to the given value. Can be specified more than once for multiple environment variables. Environment variables for this worker are also inherited from the php_server parent, but can be overwritten here. match # match the worker to a path pattern. Overrides try_files and can only be used in the php_server directive. + boot_timeout # EXPERIMENTAL: how long a background worker may take to reach its first tick() before the start fails, 0 waits forever. Default: 30s on PHP builds without Zend max execution timers, none elsewhere. + background # EXPERIMENTAL: marks this worker as a background (non-HTTP) worker, whose script ticks its FrankenPHP\WorkerHandle at least once + vars # EXPERIMENTAL: the vars namespaces a background worker publishes, read with frankenphp_get_vars() } worker # Can also use the short form like in the global frankenphp block. } diff --git a/docs/library.md b/docs/library.md index 968d2bd971..b21e04e67c 100644 --- a/docs/library.md +++ b/docs/library.md @@ -60,6 +60,8 @@ err := frankenphp.Init( Workers declared without a server scope are global: they match by file path on any server. Since a global worker has no set of requests to match against, combining `WithWorkerMatcher()` with a global worker is a configuration error and `Init()` rejects it. +`WithWorkerBackground()` declares a [background worker](worker.md#background-workers), which runs outside the request cycle. + ## Per-request options `Server.ServeHTTP()` accepts `RequestOption`s to override the server configuration for a single request, e.g. `WithRequestDocumentRoot()`, `WithRequestSplitPath()`, `WithRequestEnv()` or `WithRequestLogger()`. diff --git a/docs/metrics.md b/docs/metrics.md index b87d2231f4..0a68938959 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -19,7 +19,7 @@ When [Caddy metrics](https://caddyserver.com/docs/metrics) are enabled, FrankenP - `frankenphp_busy_workers{worker="[worker_name]"}`: The number of workers currently processing a request. - `frankenphp_worker_request_time{worker="[worker_name]"}`: The time spent processing requests by all workers. - `frankenphp_worker_request_count{worker="[worker_name]"}`: The number of requests processed by all workers. -- `frankenphp_ready_workers{worker="[worker_name]"}`: The number of workers that have called `frankenphp_handle_request` at least once. +- `frankenphp_ready_workers{worker="[worker_name]"}`: The number of workers that have reached their ready point at least once: `frankenphp_handle_request()` for HTTP workers, `WorkerHandle::tick()` for background workers. - `frankenphp_worker_crashes{worker="[worker_name]"}`: The number of times a worker has unexpectedly terminated. - `frankenphp_worker_restarts{worker="[worker_name]"}`: The number of times a worker has been deliberately restarted. - `frankenphp_worker_queue_depth{worker="[worker_name]"}`: The number of queued requests. diff --git a/docs/worker.md b/docs/worker.md index 1b6c86a225..f2cf8572cd 100644 --- a/docs/worker.md +++ b/docs/worker.md @@ -206,6 +206,110 @@ so the default `num_threads` value is `1 + 2x(number of available CPUs)`. Thus, in that mode if `num_threads` and `num` are explicitly set, `num_threads` must be strictly greater then `num`. +## Background workers + +This feature is experimental. + +A background worker runs its script in a loop outside the HTTP request cycle, on its own PHP thread. It is declared like any worker, with the `background` option; `num` defaults to one thread, and a `name` left out is generated from the script path as for HTTP workers: + +```caddyfile +php_server { + worker { + file jobs.php + num 1 + name jobs + background + } +} +``` + +The script takes a handle on the worker, `new FrankenPHP\WorkerHandle()`, and calls `tick()` on it once it is set up. The first call marks the worker ready: the server start waits for that point, and an exit before it counts as a failure. Until that call, `max_execution_time` applies as in any request, 30 seconds by default: a setup that outlives it is ended and counts as a boot failure, so raise the limit in `php_ini`, or call `set_time_limit()` in the script, when the setup legitimately takes longer. This bound only holds on PHP builds with Zend max execution timers (`--enable-zend-max-execution-timers`, see [the compilation guide](compile.md)): elsewhere FrankenPHP disables `max_execution_time`, and the `boot_timeout` of the worker ends the wait instead: 30 seconds by default, matching PHP's own `max_execution_time`, after which the start fails with the name of the worker that never ticked. Its script is then stopped, at once if it waits on its handle, or by the force-kill of the shutdown otherwise. Set `boot_timeout` to `0` to wait forever, with a warning logged after 10 seconds; it applies on every build when set. From the first call on, the run has no time limit, like the CLI. Every call returns `false` once FrankenPHP drains the worker, on shutdown, reboot or restart, and `true` otherwise. It never blocks and never hands out work: like `frankenphp_handle_request()`, it is where the runtime and the script meet. + +Between two calls the script waits on the handle, which is an `Io\Poll\Handle`: an `Io\Poll\Context` takes it as it is, next to whatever else the script waits on. + +```php +add($handle, [Event::Read]); +// plus the handles the script waits on + +while ($handle->tick()) { + foreach ($poll->wait() as $watcher) { + // the worker's handle triggered, or one of the script's + } + + doSomeWork(); +} + +// drained: return, FrankenPHP re-runs or stops the script +``` + +The handle becomes readable when FrankenPHP needs the script's attention, its drain included, and `tick()` consumes whatever was written there. What it carries is not part of the contract. It is readable once right after the script starts, so a loop that waits on it ticks by itself and the worker is ready as soon as its loop runs. + +`Io\Poll` comes with PHP 8.6 and polls with `epoll` or `kqueue`. On 8.2 to 8.5 the handle implements the same interface, so the loop above runs unchanged with [symfony/polyfill-io-poll](https://github.com/symfony/polyfill/tree/1.x/src/Io/Poll), which backs the context with `stream_select()`. + +An event loop takes the handle through its stream, `getStream()`. With [Revolt](https://revolt.run), the loop of amphp: + +```php +getStream(), function () use ($handle): void { + if (!$handle->tick()) { + // drained: stop the loop, the script returns and FrankenPHP moves on + EventLoop::getDriver()->stop(); + } +}); + +// the script's own watchers go here + +EventLoop::run(); +``` + +The wake-up sent at start makes the callback run as soon as the loop does, which is when the worker becomes ready. + +That stream is there for the libraries that take one, and waiting is all it supports: reading it steals the bytes `tick()` would have consumed, writing to it lands in a buffer FrankenPHP never drains, so a script that fills it blocks itself, and closing it is safe since the next `getStream()` on the same handle opens a fresh one over the same socket. A script with no loop of its own can wait on it directly, with a blocking read such as `fgets()` or with `stream_select()`, keeping in mind that `stream_select()` rejects a descriptor of `1024` or higher (`FD_SETSIZE`), which a worker re-run while the server holds more than a thousand connections will have. `tick()` itself uses `poll()` and is not affected. + +`$_SERVER['FRANKENPHP_WORKER_BACKGROUND']` holds the name of the worker. `FRANKENPHP_WORKER`, the variable of HTTP workers, is not set, so a script serving both roles tests which of the two is set. Their threads count like the threads of HTTP workers, on top of `num_threads` and within `max_threads`; they don't autoscale, so `max_threads` is not allowed on them. From Go, declare one with `WithWorkerBackground()`. + +### Sharing state with background workers + +A background worker publishes snapshots in vars namespaces, declared with `vars` on the worker, and requests and other workers read them with `frankenphp_get_vars()`, by namespace: + +```caddyfile +php_server { + worker { + file config.php + background + vars redis feature_flags + } +} +``` + +```php +// background worker +$handle = new FrankenPHP\WorkerHandle(); +$handle->setVars('redis', ['dsn' => 'redis://cache:6379']); +$handle->setVars('feature_flags', ['beta' => true]); +// ... + +// request, HTTP worker or another background worker +$flags = frankenphp_get_vars('feature_flags'); +``` + +A namespace has one publisher in a `php_server`, and one among the global workers; a reader gets the one of its `php_server` first, then the global one. One worker may publish as many namespaces as the application wants, so splitting them across workers is a matter of threads, failure caps and timeouts, not of addressing. A worker publishing vars runs a single thread, so each snapshot has one writer, and `getNamespaces()` on its handle returns what the configuration declared. + +Values must be null, scalars, arrays or enums. Each call replaces the whole snapshot of its namespace and readers get a copy, so the worker can publish at any time and a request always sees a consistent one. Every declared namespace is published before the first tick of a run, which `tick()` enforces: since the server accepts requests once the background workers are ready, requests never wait for a snapshot, and while a worker restarts they keep getting its last ones. + +`frankenphp_get_vars()` can only block while workers boot, a background worker or the bootstrap of an HTTP worker waiting for the publisher of a namespace. A cycle between background workers throws instead of hanging, and so does a wait once the start failed. It also throws when no worker publishes the namespace. + ## Superglobals behavior [PHP superglobals](https://www.php.net/manual/language.variables.superglobals.php) (`$_SERVER`, `$_ENV`, `$_GET`...) diff --git a/frankenphp.c b/frankenphp.c index 4066845349..563d5e1ad2 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -17,11 +17,14 @@ #include #ifdef PHP_WIN32 #include +#include #else #include #endif +#include
#include #include +#include #include #include #include @@ -32,6 +35,7 @@ #include #include #ifndef PHP_WIN32 +#include #include #endif #if defined(__linux__) @@ -55,13 +59,23 @@ #include "emulate_php_cli.h" #include "_cgo_export.h" + +#if PHP_VERSION_ID < 80400 +/* gen_stub.php registers classes with an API that only exists since 8.4. */ +static zend_always_inline zend_class_entry * +zend_register_internal_class_with_flags(zend_class_entry *class_entry, + zend_class_entry *parent_ce, + uint32_t flags) { + zend_class_entry *ce = + zend_register_internal_class_ex(class_entry, parent_ce); + ce->ce_flags |= flags; + + return ce; +} +#endif + #include "frankenphp_arginfo.h" -#ifdef FRANKENPHP_TEST -/* The persistent_zval helpers are only compiled in when a consumer needs - * them. The step that lands the first real caller (background workers) - * will drop this guard. */ #include "zval.h" -#endif #if defined(PHP_WIN32) && defined(ZTS) ZEND_TSRMLS_CACHE_DEFINE() @@ -130,6 +144,14 @@ HashTable *main_thread_env = NULL; static THREAD_LOCAL uintptr_t thread_index; static THREAD_LOCAL bool is_worker_thread = false; +static THREAD_LOCAL bool is_background_worker = false; +/* Stop socket pair of a background worker thread: [0] is the script's end, + * exposed via WorkerHandle::getStream(); [1] is transferred to the Go + * side, which closes it to signal a drain. */ +static THREAD_LOCAL php_socket_t worker_stop_socks[2] = {SOCK_ERR, SOCK_ERR}; +/* set by the first WorkerHandle::tick() of the current run, the ready + * point of a background worker */ +static THREAD_LOCAL bool worker_ticked = false; /* putenv() overrides layered over the other env tables, an IS_NULL entry * unsets the variable */ static THREAD_LOCAL HashTable *sandboxed_env = NULL; @@ -361,7 +383,203 @@ void frankenphp_release_thread_for_kill(force_kill_slot slot) { #endif } +/* Stop channel of background workers: a socket pair, whose Go end is closed + * on drain so the script's end reaches EOF. A pair rather than a pipe + * because on Windows php_select() only really waits on sockets: before 8.5 + * it reports any other handle as always ready. */ +static void frankenphp_worker_close_sock(php_socket_t s) { + if (s == SOCK_ERR) { + return; + } + /* php_network.h maps closesocket to close outside Windows */ + closesocket(s); +} + +/* keep the pair out of processes the script may spawn: a child holding the Go + * end would keep the script's end from ever reaching EOF */ +static void frankenphp_worker_sock_no_inherit(php_socket_t s) { +#ifdef PHP_WIN32 + SetHandleInformation((HANDLE)s, HANDLE_FLAG_INHERIT, 0); +#else + fcntl(s, F_SETFD, FD_CLOEXEC); +#endif +} + +static void frankenphp_worker_close_stop_socks(void) { + for (int i = 0; i < 2; i++) { + frankenphp_worker_close_sock(worker_stop_socks[i]); + worker_stop_socks[i] = SOCK_ERR; + } +} + +/* Resets the background worker state of the calling thread. The streams of + * WorkerHandle::getStream() do not own the socket, and request shutdown + * destroyed the handles they belong to. */ +static void frankenphp_reset_background_worker(void) { + is_background_worker = false; + worker_ticked = false; + frankenphp_worker_close_stop_socks(); +} + +#ifdef PHP_WIN32 +/* Windows has no socketpair() and PHP's emulation is not one: it binds a + * listener to INADDR_ANY, reachable off the machine, and hands back whichever + * connection arrives first. The pair is formed by hand instead, the way + * libevent and Tor do it: loopback alone, and a connection is kept only when + * it comes from the socket we connected with. */ +#define FRANKENPHP_SOCK_PAIR_TRIES 8 + +static int frankenphp_sock_pair_win32(php_socket_t socks[2]) { + SOCKET listener = INVALID_SOCKET, client = INVALID_SOCKET, + accepted = INVALID_SOCKET; + struct sockaddr_in addr, local, peer; + int addr_len = sizeof(addr), local_len = sizeof(local), peer_len, + exclusive = 1; + + socks[0] = SOCK_ERR; + socks[1] = SOCK_ERR; + + listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (listener == INVALID_SOCKET) { + return -1; + } + + /* nobody else may take this port from under the listener */ + setsockopt(listener, SOL_SOCKET, SO_EXCLUSIVEADDRUSE, + (const char *)&exclusive, sizeof(exclusive)); + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; + + if (bind(listener, (struct sockaddr *)&addr, sizeof(addr)) != 0 || + getsockname(listener, (struct sockaddr *)&addr, &addr_len) != 0 || + listen(listener, 1) != 0) { + goto error; + } + + client = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (client == INVALID_SOCKET || + connect(client, (struct sockaddr *)&addr, sizeof(addr)) != 0 || + getsockname(client, (struct sockaddr *)&local, &local_len) != 0) { + goto error; + } + + for (int attempt = 0; attempt < FRANKENPHP_SOCK_PAIR_TRIES; attempt++) { + memset(&peer, 0, sizeof(peer)); + peer_len = sizeof(peer); + accepted = accept(listener, (struct sockaddr *)&peer, &peer_len); + if (accepted == INVALID_SOCKET) { + goto error; + } + if (peer.sin_port == local.sin_port && + peer.sin_addr.s_addr == local.sin_addr.s_addr) { + closesocket(listener); + socks[0] = accepted; + socks[1] = client; + + return 0; + } + + /* somebody else got in: drop them and wait for our own connection */ + closesocket(accepted); + accepted = INVALID_SOCKET; + } + +error: + if (listener != INVALID_SOCKET) { + closesocket(listener); + } + if (client != INVALID_SOCKET) { + closesocket(client); + } + if (accepted != INVALID_SOCKET) { + closesocket(accepted); + } + + return -1; +} +#endif + +static int frankenphp_worker_open_stop_pair(void) { +#ifdef PHP_WIN32 + if (frankenphp_sock_pair_win32(worker_stop_socks) != 0) { + return -1; + } +#else +#ifdef SOCK_CLOEXEC + int type = SOCK_STREAM | SOCK_CLOEXEC; +#else + int type = SOCK_STREAM; +#endif + if (socketpair(AF_UNIX, type, 0, worker_stop_socks) != 0) { + worker_stop_socks[0] = SOCK_ERR; + worker_stop_socks[1] = SOCK_ERR; + + return -1; + } +#endif + /* redundant where SOCK_CLOEXEC applied; a fork()ed child (pcntl) still + * inherits both ends and delays the EOF until it exits */ + frankenphp_worker_sock_no_inherit(worker_stop_socks[0]); + frankenphp_worker_sock_no_inherit(worker_stop_socks[1]); + + return 0; +} + +/* Marks the calling thread as a background worker, opens its stop socket pair + * and transfers the Go side's end to the caller, clearing the TLS slot so a + * later recycle won't double-close it. Returns -1 if the pair could not be + * created. max_execution_time stays armed until the first tick(), see there. */ +intptr_t frankenphp_set_background_worker_and_get_stop_sock(void) { + frankenphp_reset_background_worker(); + is_background_worker = true; + + if (frankenphp_worker_open_stop_pair() != 0) { + return -1; + } + + /* One wake-up right away, so a script that hands its handle to an event + * loop ticks on its own: readiness then means the loop serviced the handle + * once. Nothing to do on failure, the script has to tick by itself. */ + const char wakeup = '\n'; +#ifdef MSG_NOSIGNAL + send(worker_stop_socks[1], &wakeup, 1, MSG_NOSIGNAL); +#else + send(worker_stop_socks[1], &wakeup, 1, 0); +#endif + + intptr_t s = (intptr_t)worker_stop_socks[1]; + worker_stop_socks[1] = SOCK_ERR; + + return s; +} + +/* Closes the Go side's end of a stop socket pair, which lands as EOF on the + * script's end so its stream_select() or blocking read returns promptly. */ +void frankenphp_worker_close_stop_sock(intptr_t s) { + if (s < 0) { + return; + } + /* A pcntl_fork() child inherits every descriptor of the process, and while + * it holds a copy of this end closing it lands no EOF: shutting the write + * direction down sends the FIN regardless. */ +#ifdef PHP_WIN32 + shutdown((php_socket_t)s, SD_SEND); +#else + shutdown((php_socket_t)s, SHUT_WR); +#endif + frankenphp_worker_close_sock((php_socket_t)s); +} + void frankenphp_update_local_thread_context(bool is_worker) { + /* A thread that ran a background worker can be recycled into an HTTP or a + * request thread: reset the TLS so the handle rejects callers again. */ + if (is_background_worker) { + frankenphp_reset_background_worker(); + } + is_worker_thread = is_worker; /* workers should keep running if the user aborts the connection */ @@ -967,6 +1185,15 @@ PHP_FUNCTION(frankenphp_handle_request) { RETURN_THROWS(); } + if (is_background_worker) { + /* background workers never receive HTTP requests */ + zend_throw_exception( + spl_ce_RuntimeException, + "frankenphp_handle_request() cannot be called from a background worker", + 0); + RETURN_THROWS(); + } + #ifdef ZEND_MAX_EXECUTION_TIMERS /* Disable timeouts while waiting for a request to handle */ zend_unset_timeout(); @@ -1165,6 +1392,375 @@ static void frankenphp_opcache_restart_hook(int reason) { } #endif +/* Handles of FrankenPHP: objects a script waits on. On PHP 8.6 they implement + * Io\Poll\Handle, so an Io\Poll\Context waits on them next to the script's + * own handles; the interface is a marker, the contract is the three hooks + * below, and the descriptor never reaches userland. Before 8.6 the same hooks + * carry the object's lifetime, in the layout the poll API expects. */ +#if PHP_VERSION_ID >= 80600 +#include +typedef php_poll_handle_object frankenphp_handle_obj; +typedef php_poll_handle_ops frankenphp_handle_ops; +#define FRANKENPHP_HANDLE_OF(zobj) PHP_POLL_HANDLE_OBJ_FROM_ZOBJ(zobj) +#else +typedef struct frankenphp_handle_obj frankenphp_handle_obj; +typedef struct { + php_socket_t (*get_fd)(frankenphp_handle_obj *handle); + int (*is_valid)(frankenphp_handle_obj *handle); + void (*cleanup)(frankenphp_handle_obj *handle); +} frankenphp_handle_ops; +struct frankenphp_handle_obj { + frankenphp_handle_ops *ops; + void *handle_data; + zend_object std; +}; +#define FRANKENPHP_HANDLE_OF(zobj) \ + ((frankenphp_handle_obj *)((char *)(zobj) - \ + offsetof(frankenphp_handle_obj, std))) +#endif + +/* The state a handle carries, NULL when it needs none. */ +#define FRANKENPHP_HANDLE_DATA(zthis) \ + (FRANKENPHP_HANDLE_OF(Z_OBJ_P(zthis))->handle_data) + +static zend_object_handlers frankenphp_handle_obj_handlers; + +static zend_object *frankenphp_handle_obj_create(zend_class_entry *ce, + frankenphp_handle_ops *ops) { +#if PHP_VERSION_ID >= 80600 + frankenphp_handle_obj *obj = + php_poll_handle_object_create(sizeof(*obj), ce, ops); +#else + frankenphp_handle_obj *obj = zend_object_alloc(sizeof(*obj), ce); + + zend_object_std_init(&obj->std, ce); + object_properties_init(&obj->std, ce); + obj->ops = ops; + obj->handle_data = NULL; +#endif + obj->std.handlers = &frankenphp_handle_obj_handlers; + + return &obj->std; +} + +static void frankenphp_handle_obj_free(zend_object *object) { + frankenphp_handle_obj *obj = FRANKENPHP_HANDLE_OF(object); + + if (obj->ops != NULL && obj->ops->cleanup != NULL) { + obj->ops->cleanup(obj); + } + zend_object_std_dtor(&obj->std); +} + +/* Declares Io\Poll\Handle on a handle class: the interface accepts internal + * classes only. PHP 8.6 brings it with the poll API, below it FrankenPHP + * declares it, and the classmap-autoloaded declaration of + * symfony/polyfill-io-poll then never loads. */ +static void frankenphp_handle_implements_poll(zend_class_entry *ce) { + zend_class_entry *poll_handle_ce = zend_hash_str_find_ptr( + CG(class_table), "io\\poll\\handle", sizeof("io\\poll\\handle") - 1); + +#if PHP_VERSION_ID < 80600 + if (poll_handle_ce == NULL) { + zend_class_entry poll_handle; + INIT_NS_CLASS_ENTRY(poll_handle, "Io\\Poll", "Handle", NULL); + poll_handle_ce = zend_register_internal_interface(&poll_handle); + } +#endif + + if (poll_handle_ce != NULL) { + zend_class_implements(ce, 1, poll_handle_ce); + } +} + +/* The socket of the current background worker, invalid outside one and + * once the thread let it go: is_valid() then reports what tick() does. */ +static php_socket_t +frankenphp_worker_handle_get_fd(frankenphp_handle_obj *handle) { + (void)handle; + + return is_background_worker ? worker_stop_socks[0] : SOCK_ERR; +} + +static int frankenphp_worker_handle_is_valid(frankenphp_handle_obj *handle) { + return frankenphp_worker_handle_get_fd(handle) != SOCK_ERR; +} + +/* The stream a script took from this handle, so asking twice hands out the + * same one and a loop does not grow the resource list of a run that never + * ends. On the handle rather than the thread, so it goes away with it. */ +static void frankenphp_worker_handle_cleanup(frankenphp_handle_obj *handle) { + if (handle->handle_data == NULL) { + return; + } + + zend_list_delete(handle->handle_data); + handle->handle_data = NULL; +} + +static frankenphp_handle_ops frankenphp_worker_handle_poll_ops = { + .get_fd = frankenphp_worker_handle_get_fd, + .is_valid = frankenphp_worker_handle_is_valid, + .cleanup = frankenphp_worker_handle_cleanup, +}; + +static zend_object *frankenphp_worker_handle_new(zend_class_entry *ce) { + return frankenphp_handle_obj_create(ce, &frankenphp_worker_handle_poll_ops); +} + +/* Ops of the streams returned by WorkerHandle::getStream(): the socket ops, + * except that closing a stream leaves the socket alone. It belongs to the + * thread, every handle of a run shares it, and it is closed at the next run + * setup or on thread exit. Initialized in MINIT. */ +static php_stream_ops frankenphp_worker_handle_ops; + +static int frankenphp_worker_handle_close(php_stream *stream, + int close_handle) { + (void)close_handle; + + /* free the stream data only, never the shared socket */ + return php_stream_socket_ops.close(stream, 0); +} + +/* The constructor is not a gate: unserialize() and Reflection make an + * instance without it, so every method checks the thread it runs on. */ +static bool frankenphp_worker_handle_usable(void) { + if (is_background_worker && worker_stop_socks[0] != SOCK_ERR) { + return true; + } + + zend_throw_exception( + spl_ce_RuntimeException, + "FrankenPHP\\WorkerHandle can only be used from a background worker", 0); + + return false; +} + +/* The handle of the current background worker: the stream to wait on and the + * tick that reports the drain. Constructing it outside a background worker + * throws, a run may create as many as it likes. */ +ZEND_METHOD(FrankenPHP_WorkerHandle, __construct) { + ZEND_PARSE_PARAMETERS_NONE(); + + if (!is_background_worker) { + zend_throw_exception( + spl_ce_RuntimeException, + "FrankenPHP\\WorkerHandle can only be created from a background worker", + 0); + RETURN_THROWS(); + } +} + +ZEND_METHOD(FrankenPHP_WorkerHandle, getStream) { + ZEND_PARSE_PARAMETERS_NONE(); + + if (!frankenphp_worker_handle_usable()) { + RETURN_THROWS(); + } + + /* The same resource until the script closes it, so asking for it in a loop + * does not grow the resource list of a run that never ends. The stream does + * not own the socket (see frankenphp_worker_handle_ops), so closing one + * leaves the streams of the other handles alone. */ + frankenphp_handle_obj *handle = FRANKENPHP_HANDLE_OF(Z_OBJ_P(ZEND_THIS)); + if (handle->handle_data != NULL) { + zend_resource *cached = handle->handle_data; + if (cached->type == php_file_le_stream()) { + GC_ADDREF(cached); + RETURN_RES(cached); + } + /* closed by the script: drop the handle's ref */ + zend_list_delete(cached); + handle->handle_data = NULL; + } + + php_stream *stream = + php_stream_sock_open_from_socket(worker_stop_socks[0], NULL); + if (stream == NULL) { + zend_throw_exception(spl_ce_RuntimeException, + "failed to create a stream over the stop socket", 0); + RETURN_THROWS(); + } + + /* a blocking read is a valid way to park: wait without the + * default_socket_timeout wake-ups */ + struct timeval no_timeout = {-1, 0}; + php_stream_set_option(stream, PHP_STREAM_OPTION_READ_TIMEOUT, 0, &no_timeout); + + stream->ops = &frankenphp_worker_handle_ops; + + php_stream_to_zval(stream, return_value); + handle->handle_data = Z_RES_P(return_value); + GC_ADDREF(Z_RES_P(return_value)); +} + +/* The ready point of a background worker and its liveness check, the analog + * of frankenphp_handle_request(): the first call of a run reports the worker + * ready, and every call returns false once FrankenPHP drains it. It never + * blocks and never hands out work, the script waits on the stream of + * getStream() and calls this when it is readable. Whatever the runtime wrote + * there to wake the script up is consumed here, so the protocol stays + * private. */ +ZEND_METHOD(FrankenPHP_WorkerHandle, tick) { + ZEND_PARSE_PARAMETERS_NONE(); + + if (!frankenphp_worker_handle_usable()) { + RETURN_THROWS(); + } + + if (!worker_ticked) { + /* refused while a declared vars namespace has no snapshot of this run */ + char *error = + go_frankenphp_background_worker_ready(frankenphp_thread_index()); + if (error != NULL) { + zend_throw_exception(spl_ce_LogicException, error, 0); + free(error); + RETURN_THROWS(); + } + worker_ticked = true; + /* The bootstrap ran under max_execution_time like any request, the loop + * that starts now has no time limit, like the CLI. Nothing re-arms the + * timer past this point. */ + zend_unset_timeout(); + } + + /* consume the wake-ups without blocking; EOF is the drain */ + char buf[64]; + for (;;) { + if (php_pollfd_for_ms(worker_stop_socks[0], PHP_POLLREADABLE, 0) <= 0) { + /* nothing pending, or a transient poll error: still running */ + RETURN_TRUE; + } + +#ifdef PHP_WIN32 + int n = recv(worker_stop_socks[0], buf, (int)sizeof(buf), 0); +#else + ssize_t n = recv(worker_stop_socks[0], buf, sizeof(buf), 0); +#endif + if (n == 0) { + /* the Go side closed its end: drained */ + RETURN_FALSE; + } + if (n < 0) { + int err = php_socket_errno(); + if (err == EINTR || PHP_IS_TRANSIENT_ERROR(err)) { + RETURN_TRUE; + } + + /* a broken socket carries no drain anymore, stop the loop */ + RETURN_FALSE; + } + } +} + +/* Shared vars of background workers, see WorkerHandle::setVars() and + * frankenphp_get_vars(): the persistent tables live in slots owned by the + * Go side, which copies and frees them through these two helpers. */ +void frankenphp_vars_to_request(zval *return_value, HashTable *table) { + zval persistent; + ZVAL_ARR(&persistent, table); + persistent_zval_to_request(return_value, &persistent); +} + +void frankenphp_vars_free(HashTable *table) { + zval persistent; + ZVAL_ARR(&persistent, table); + persistent_zval_free(&persistent); +} + +void frankenphp_array_append_string(zval *array, char *str, size_t len) { + add_next_index_stringl(array, str, len); +} + +ZEND_METHOD(FrankenPHP_WorkerHandle, getNamespaces) { + ZEND_PARSE_PARAMETERS_NONE(); + + if (!frankenphp_worker_handle_usable()) { + RETURN_THROWS(); + } + + zval vars; + array_init(&vars); + go_frankenphp_worker_vars(frankenphp_thread_index(), &vars); + + array_init(return_value); + add_assoc_zval(return_value, "vars", &vars); +} + +ZEND_METHOD(FrankenPHP_WorkerHandle, setVars) { + zend_string *namespace; + zval *vars; + ZEND_PARSE_PARAMETERS_START(2, 2) + Z_PARAM_STR(namespace) + Z_PARAM_ARRAY(vars) + ZEND_PARSE_PARAMETERS_END(); + + if (!frankenphp_worker_handle_usable()) { + RETURN_THROWS(); + } + + /* validate the whole tree first: persist and free recurse without a + * guard of their own */ + if (!persistent_zval_validate(vars)) { + zend_value_error( + "FrankenPHP\\WorkerHandle::setVars(): values must be null, scalars, " + "arrays or enums, nested no deeper than %d levels", + PERSISTENT_ZVAL_MAX_DEPTH); + RETURN_THROWS(); + } + + zval persistent; + persistent_zval_persist(&persistent, vars); + + HashTable *old = NULL; + char *error = + go_frankenphp_set_vars(frankenphp_thread_index(), ZSTR_VAL(namespace), + ZSTR_LEN(namespace), Z_ARRVAL(persistent), &old); + if (error != NULL) { + frankenphp_vars_free(Z_ARRVAL(persistent)); + zend_throw_exception(spl_ce_LogicException, error, 0); + free(error); + RETURN_THROWS(); + } + if (old != NULL) { + frankenphp_vars_free(old); + } +} + +PHP_FUNCTION(frankenphp_get_vars) { + zend_string *namespace; + ZEND_PARSE_PARAMETERS_START(1, 1) + Z_PARAM_STR(namespace) + ZEND_PARSE_PARAMETERS_END(); + + HashTable *table = NULL; + uintptr_t snapshot = 0; + char *error = + go_frankenphp_get_vars(frankenphp_thread_index(), ZSTR_VAL(namespace), + ZSTR_LEN(namespace), &table, &snapshot); + if (error != NULL) { + zend_throw_exception(spl_ce_RuntimeException, error, 0); + free(error); + RETURN_THROWS(); + } + + /* the copy may bail out, past memory_limit or in an autoloader an enum + * triggers: the snapshot is let go of whatever happens, so its publisher + * can replace and free it */ + bool bailed_out = false; + zend_try { frankenphp_vars_to_request(return_value, table); } + zend_catch { bailed_out = true; } + zend_end_try(); + + HashTable *unused = go_frankenphp_vars_done(snapshot); + if (unused != NULL) { + frankenphp_vars_free(unused); + } + if (bailed_out) { + zend_bailout(); + } +} + /* {{{ thread-safe opcache reset */ PHP_FUNCTION(frankenphp_opcache_reset) { go_schedule_opcache_reset(frankenphp_thread_index()); @@ -1223,7 +1819,20 @@ static const zend_function_entry frankenphp_test_hook_functions[] = { #endif PHP_MINIT_FUNCTION(frankenphp) { + frankenphp_worker_handle_ops = php_stream_socket_ops; + frankenphp_worker_handle_ops.label = "FrankenPHP worker handle"; + frankenphp_worker_handle_ops.close = frankenphp_worker_handle_close; + + frankenphp_handle_obj_handlers = std_object_handlers; + frankenphp_handle_obj_handlers.offset = offsetof(frankenphp_handle_obj, std); + frankenphp_handle_obj_handlers.free_obj = frankenphp_handle_obj_free; + frankenphp_handle_obj_handlers.clone_obj = NULL; + register_frankenphp_symbols(module_number); + + zend_class_entry *worker_handle_ce = register_class_FrankenPHP_WorkerHandle(); + worker_handle_ce->create_object = frankenphp_worker_handle_new; + frankenphp_handle_implements_poll(worker_handle_ce); #ifndef PHP_WIN32 /* MINIT runs once per ZTS thread — guard the atfork registration */ static pthread_once_t atfork_once = PTHREAD_ONCE_INIT; @@ -1858,6 +2467,13 @@ static void *php_thread(void *arg) { } zend_end_try(); + /* frankenphp_update_local_thread_context() only releases the stop socket on + * recycle: close it here too, so it does not outlive the thread on shutdown, + * reboot or an unhealthy exit. */ + if (is_background_worker) { + frankenphp_reset_background_worker(); + } + /* Must precede ts_free_thread: that frees the TSRM storage backing * the slot's &EG() pointers. Clearing first means any concurrent * force-kill either ran before us or sees a zero slot. */ diff --git a/frankenphp.go b/frankenphp.go index e560031e62..35274c75fd 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -28,6 +28,7 @@ import ( "os" "os/signal" "runtime" + "slices" "strings" "sync" "sync/atomic" @@ -221,7 +222,11 @@ func calculateMaxThreads(opt *opt) (numWorkers int, _ error) { maxThreadsFromWorkers := 0 for i, w := range opt.workers { - if w.num <= 0 { + if w.num <= 0 && w.isBackgroundWorker { + // a background worker serves no requests, so it does not scale + // with the CPUs like an HTTP one + opt.workers[i].num = 1 + } else if w.num <= 0 { // https://github.com/php/frankenphp/issues/126 opt.workers[i].num = maxProcs } @@ -323,25 +328,24 @@ func Validate(options ...Option) error { } takenNames := make(map[string]bool, len(opt.workers)) - takenPaths := make(map[*Server]map[string]bool, 1) - nameTaken := func(name string) bool { return takenNames[name] } - pathTaken := func(server *Server, path string) bool { return takenPaths[server][path] } + paths, vars := scopeSet{}, scopeSet{} + taken := declarationIndex{names: func(name string) bool { return takenNames[name] }, paths: paths.has, vars: vars.has} for _, w := range opt.workers { w, err := resolveWorkerFile(w) if err != nil { return err } - if err := checkWorkerDeclaration(w, nameTaken, pathTaken); err != nil { + if err := checkWorkerDeclaration(w, taken); err != nil { return err } takenNames[w.name] = true - if w.matchRequest == nil { - if takenPaths[w.server] == nil { - takenPaths[w.server] = make(map[string]bool) - } - takenPaths[w.server][w.fileName] = true + if !w.isBackgroundWorker && w.matchRequest == nil { + paths.add(w.server, w.fileName) + } + for _, ns := range w.vars { + vars.add(w.server, ns) } } @@ -421,6 +425,11 @@ func Init(options ...Option) error { } } else { opt.numThreads = 1 + // a background worker would take the only thread + if slices.ContainsFunc(opt.workers, func(w workerOpt) bool { return w.isBackgroundWorker }) { + shutdown() + return errors.New("background workers need a ZTS build of PHP, this one runs a single thread") + } if globalLogger.Enabled(globalCtx, slog.LevelWarn) { globalLogger.LogAttrs(globalCtx, slog.LevelWarn, `ZTS is not enabled, only 1 thread will be available, recompile PHP using the "--enable-zts" configuration option or performance will be degraded`) diff --git a/frankenphp.h b/frankenphp.h index 5eb6316947..81d2d0fcdd 100644 --- a/frankenphp.h +++ b/frankenphp.h @@ -219,6 +219,13 @@ typedef enum { FRANKENPHP_MERCURE_UNSUPPORTED = 4, } frankenphp_mercure_status; +/* Background worker primitives. */ +intptr_t frankenphp_set_background_worker_and_get_stop_sock(void); +void frankenphp_worker_close_stop_sock(intptr_t s); +void frankenphp_vars_to_request(zval *return_value, HashTable *table); +void frankenphp_vars_free(HashTable *table); +void frankenphp_array_append_string(zval *array, char *str, size_t len); + void register_extensions(zend_module_entry **m, int len); // This function is exported for use by extensions, as proposed in PR #2305 diff --git a/frankenphp.stub.php b/frankenphp.stub.php index d61397caaa..681cf4d81e 100644 --- a/frankenphp.stub.php +++ b/frankenphp.stub.php @@ -2,55 +2,127 @@ /** @generate-class-entries */ -/** @var int */ -const FRANKENPHP_LOG_LEVEL_DEBUG = -4; - -/** @var int */ -const FRANKENPHP_LOG_LEVEL_INFO = 0; - -/** @var int */ -const FRANKENPHP_LOG_LEVEL_WARN = 4; - -/** @var int */ -const FRANKENPHP_LOG_LEVEL_ERROR = 8; - -function frankenphp_handle_request(callable $callback): bool {} - -function headers_send(int $status = 200): int {} - -function frankenphp_finish_request(): bool {} - -/** - * @alias frankenphp_finish_request - */ -function fastcgi_finish_request(): bool {} - -function frankenphp_request_headers(): array {} - -/** - * @alias frankenphp_request_headers - */ -function apache_request_headers(): array {} - -/** - * @alias frankenphp_request_headers -*/ -function getallheaders(): array {} - -function frankenphp_response_headers(): array|bool {} - -/** - * @alias frankenphp_response_headers - */ -function apache_response_headers(): array|bool {} - -/** - * @param string|string[] $topics The first topic is the canonical one, the others are alternate topics - */ -function mercure_publish(string|array $topics, string $data = '', bool $private = false, ?string $id = null, ?string $type = null, ?int $retry = null): string {} - -/** - * @param int $level The importance or severity of a log event. The higher the level, the more important or severe the event. For more details, see: https://pkg.go.dev/log/slog#Level - * array $context Values of the array will be converted to the corresponding Go type (if supported by FrankenPHP) and added to the context of the structured logs using https://pkg.go.dev/log/slog#Attr - */ -function frankenphp_log(string $message, int $level = 0, array $context = []): void {} +namespace { + /** @var int */ + const FRANKENPHP_LOG_LEVEL_DEBUG = -4; + + /** @var int */ + const FRANKENPHP_LOG_LEVEL_INFO = 0; + + /** @var int */ + const FRANKENPHP_LOG_LEVEL_WARN = 4; + + /** @var int */ + const FRANKENPHP_LOG_LEVEL_ERROR = 8; + + function frankenphp_handle_request(callable $callback): bool {} + + function headers_send(int $status = 200): int {} + + function frankenphp_finish_request(): bool {} + + /** + * @alias frankenphp_finish_request + */ + function fastcgi_finish_request(): bool {} + + function frankenphp_request_headers(): array {} + + /** + * @alias frankenphp_request_headers + */ + function apache_request_headers(): array {} + + /** + * @alias frankenphp_request_headers + */ + function getallheaders(): array {} + + function frankenphp_response_headers(): array|bool {} + + /** + * @alias frankenphp_response_headers + */ + function apache_response_headers(): array|bool {} + + /** + * @param string|string[] $topics The first topic is the canonical one, the others are alternate topics + */ + function mercure_publish(string|array $topics, string $data = '', bool $private = false, ?string $id = null, ?string $type = null, ?int $retry = null): string {} + + /** + * @param int $level The importance or severity of a log event. The higher the level, the more important or severe the event. For more details, see: https://pkg.go.dev/log/slog#Level + * array $context Values of the array will be converted to the corresponding Go type (if supported by FrankenPHP) and added to the context of the structured logs using https://pkg.go.dev/log/slog#Attr + */ + function frankenphp_log(string $message, int $level = 0, array $context = []): void {} + /** + * Returns a copy of the snapshot last published in a vars namespace, + * resolved within the current php_server, then among global workers. + * Blocks until the publishing worker reached its ready point. Throws if no + * background worker publishes the namespace, or if background workers wait + * on each other in a cycle. + */ + function frankenphp_get_vars(string $namespace): array {} +} + +namespace FrankenPHP { + /** + * @strict-properties + * @not-serializable + * + * EXPERIMENTAL: the handle of the current background worker, the one + * point where the script and FrankenPHP meet. Constructing it outside + * a background worker throws. It implements Io\Poll\Handle, so an + * Io\Poll\Context waits on it directly, the one of PHP 8.6 or the one + * of symfony/polyfill-io-poll below that. + */ + final class WorkerHandle + { + public function __construct() {} + + /** + * The ready point and liveness check of a background worker, the + * background analog of frankenphp_handle_request(). The first call + * of a run marks the worker ready: the server start waits for it, + * and an exit before it counts as a failure. max_execution_time + * applies until that call and not after. It returns false once + * FrankenPHP drains the worker, on shutdown, reboot or restart, so + * the script can leave its loop, and true otherwise. It never + * blocks and never hands out work: the script waits on the stream + * of getStream() and calls this when it is readable, which also + * consumes whatever FrankenPHP wrote there. + */ + public function tick(): bool {} + + /** + * The stream to wait on, alone or with the script's own streams: it + * becomes readable when FrankenPHP needs the script's attention, + * its drain included. Only waiting on it is supported, through + * stream_select() or a blocking read; what it carries is not part + * of the contract and tick() consumes it. Reading it steals those + * bytes from tick(), writing to it goes nowhere. Closing it is + * safe: a handle hands out one stream, a fresh one over the same + * socket once the script closed it. + * + * @return resource + */ + public function getStream() {} + + /** + * Returns the namespaces this background worker serves, as declared in + * its configuration. + * + * @return array{vars: list} + */ + public function getNamespaces(): array {} + + /** + * Publishes a snapshot in one of the vars namespaces of this background + * worker: the array replaces the previous snapshot of that namespace, + * atomically for readers, which get copies. Values must be null, + * scalars, arrays or enums. Every declared namespace is published + * before the first tick() of a run. + */ + public function setVars(string $namespace, array $vars): void {} + } +} diff --git a/frankenphp_arginfo.h b/frankenphp_arginfo.h index 052cfc67c6..260dc31c98 100644 --- a/frankenphp_arginfo.h +++ b/frankenphp_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: 85fabd314ab885307e0032b9f7635fa13cc0c4b5 */ + * Stub hash: cf6b08b8a5459dc24b22ac90056be5860dfcc72d */ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_frankenphp_handle_request, 0, 1, _IS_BOOL, 0) ZEND_ARG_TYPE_INFO(0, callback, IS_CALLABLE, 0) @@ -41,6 +41,23 @@ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_frankenphp_log, 0, 1, IS_VOID, 0 ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, context, IS_ARRAY, 0, "[]") ZEND_END_ARG_INFO() +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_frankenphp_get_vars, 0, 1, IS_ARRAY, 0) + ZEND_ARG_TYPE_INFO(0, namespace, IS_STRING, 0) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_class_FrankenPHP_WorkerHandle___construct, 0, 0, 0) +ZEND_END_ARG_INFO() + +#define arginfo_class_FrankenPHP_WorkerHandle_tick arginfo_frankenphp_finish_request + +#define arginfo_class_FrankenPHP_WorkerHandle_getStream arginfo_class_FrankenPHP_WorkerHandle___construct + +#define arginfo_class_FrankenPHP_WorkerHandle_getNamespaces arginfo_frankenphp_request_headers + +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_FrankenPHP_WorkerHandle_setVars, 0, 2, IS_VOID, 0) + ZEND_ARG_TYPE_INFO(0, namespace, IS_STRING, 0) + ZEND_ARG_TYPE_INFO(0, vars, IS_ARRAY, 0) +ZEND_END_ARG_INFO() ZEND_FUNCTION(frankenphp_handle_request); ZEND_FUNCTION(headers_send); @@ -49,7 +66,12 @@ ZEND_FUNCTION(frankenphp_request_headers); ZEND_FUNCTION(frankenphp_response_headers); ZEND_FUNCTION(mercure_publish); ZEND_FUNCTION(frankenphp_log); - +ZEND_FUNCTION(frankenphp_get_vars); +ZEND_METHOD(FrankenPHP_WorkerHandle, __construct); +ZEND_METHOD(FrankenPHP_WorkerHandle, tick); +ZEND_METHOD(FrankenPHP_WorkerHandle, getStream); +ZEND_METHOD(FrankenPHP_WorkerHandle, getNamespaces); +ZEND_METHOD(FrankenPHP_WorkerHandle, setVars); static const zend_function_entry ext_functions[] = { ZEND_FE(frankenphp_handle_request, arginfo_frankenphp_handle_request) @@ -63,6 +85,16 @@ static const zend_function_entry ext_functions[] = { ZEND_FALIAS(apache_response_headers, frankenphp_response_headers, arginfo_apache_response_headers) ZEND_FE(mercure_publish, arginfo_mercure_publish) ZEND_FE(frankenphp_log, arginfo_frankenphp_log) + ZEND_FE(frankenphp_get_vars, arginfo_frankenphp_get_vars) + ZEND_FE_END +}; + +static const zend_function_entry class_FrankenPHP_WorkerHandle_methods[] = { + ZEND_ME(FrankenPHP_WorkerHandle, __construct, arginfo_class_FrankenPHP_WorkerHandle___construct, ZEND_ACC_PUBLIC) + ZEND_ME(FrankenPHP_WorkerHandle, tick, arginfo_class_FrankenPHP_WorkerHandle_tick, ZEND_ACC_PUBLIC) + ZEND_ME(FrankenPHP_WorkerHandle, getStream, arginfo_class_FrankenPHP_WorkerHandle_getStream, ZEND_ACC_PUBLIC) + ZEND_ME(FrankenPHP_WorkerHandle, getNamespaces, arginfo_class_FrankenPHP_WorkerHandle_getNamespaces, ZEND_ACC_PUBLIC) + ZEND_ME(FrankenPHP_WorkerHandle, setVars, arginfo_class_FrankenPHP_WorkerHandle_setVars, ZEND_ACC_PUBLIC) ZEND_FE_END }; @@ -73,3 +105,13 @@ static void register_frankenphp_symbols(int module_number) REGISTER_LONG_CONSTANT("FRANKENPHP_LOG_LEVEL_WARN", 4, CONST_PERSISTENT); REGISTER_LONG_CONSTANT("FRANKENPHP_LOG_LEVEL_ERROR", 8, CONST_PERSISTENT); } + +static zend_class_entry *register_class_FrankenPHP_WorkerHandle(void) +{ + zend_class_entry ce, *class_entry; + + INIT_NS_CLASS_ENTRY(ce, "FrankenPHP", "WorkerHandle", class_FrankenPHP_WorkerHandle_methods); + class_entry = zend_register_internal_class_with_flags(&ce, NULL, ZEND_ACC_FINAL|ZEND_ACC_NO_DYNAMIC_PROPERTIES|ZEND_ACC_NOT_SERIALIZABLE); + + return class_entry; +} diff --git a/metrics.go b/metrics.go index 852305aa35..a27e02630d 100644 --- a/metrics.go +++ b/metrics.go @@ -11,7 +11,7 @@ import ( const ( StopReasonCrash = iota StopReasonRestart - StopReasonBootFailure // worker crashed before reaching frankenphp_handle_request + StopReasonBootFailure // worker exited before reaching its ready point: frankenphp_handle_request, or WorkerHandle::tick() for background workers ) type StopReason int @@ -161,7 +161,7 @@ func (m *PrometheusMetrics) StopWorker(name string, reason StopReason) { m.totalWorkers.WithLabelValues(name).Dec() - // only decrement readyWorkers if the worker actually reached frankenphp_handle_request + // only decrement readyWorkers if the worker actually reached its ready point if reason != StopReasonBootFailure { m.readyWorkers.WithLabelValues(name).Dec() } @@ -194,7 +194,7 @@ func (m *PrometheusMetrics) TotalWorkers(string, int) { m.readyWorkers = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: ns, Name: "ready_workers", - Help: "Running workers that have successfully called frankenphp_handle_request at least once", + Help: "Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, WorkerHandle::tick() for background workers", }, basicLabels) m.mustRegister(m.readyWorkers) } diff --git a/metrics_test.go b/metrics_test.go index 4d5ec1a637..5910c67808 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -188,7 +188,7 @@ func TestPrometheusMetrics_TestStopReasonCrash(t *testing.T) { name: "Testing ReadyWorkers", c: m.readyWorkers, metadata: ` - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, WorkerHandle::tick() for background workers # TYPE frankenphp_ready_workers gauge `, expect: ` diff --git a/options.go b/options.go index 40f50a17ee..9b10bd01be 100644 --- a/options.go +++ b/options.go @@ -2,9 +2,11 @@ package frankenphp import ( "context" + "errors" "fmt" "log/slog" "net/http" + "slices" "time" ) @@ -58,6 +60,10 @@ type workerOpt struct { onServerStartup func() onServerShutdown func() server *Server + isBackgroundWorker bool + bootTimeout time.Duration + bootTimeoutIsSet bool + vars []string } // WithContext sets the main context to use. @@ -242,6 +248,52 @@ func WithWorkerServerScope(s *Server) WorkerOption { } } +// EXPERIMENTAL: WithWorkerBackground marks this worker as a background +// (non-HTTP) worker: it shares the PHP runtime with the HTTP threads but +// never receives requests. Its script ticks a FrankenPHP\WorkerHandle: the +// first tick() marks the worker ready, and tick() returns false once +// FrankenPHP drains the worker, on shutdown or restart. +func WithWorkerBackground() WorkerOption { + return func(w *workerOpt) error { + w.isBackgroundWorker = true + + return nil + } +} + +// EXPERIMENTAL: WithWorkerBootTimeout bounds how long a background worker may +// take to reach its ready point, its first WorkerHandle::tick(), before Init() +// gives up and returns an error. Zero waits for ever. Defaults to +// DefaultWorkerBootTimeout on PHP builds without Zend max execution timers, +// and to zero elsewhere, where max_execution_time bounds the bootstrap. It has +// no effect on HTTP workers. +func WithWorkerBootTimeout(timeout time.Duration) WorkerOption { + return func(w *workerOpt) error { + if timeout < 0 { + return fmt.Errorf("worker boot timeout must be >= 0, got %s", timeout) + } + w.bootTimeout = timeout + w.bootTimeoutIsSet = true + + return nil + } +} + +// EXPERIMENTAL: WithWorkerVars declares the vars namespaces a background +// worker publishes with WorkerHandle::setVars(), which frankenphp_get_vars() +// reads by namespace. A namespace has one publisher within a server, and one +// among the global workers. +func WithWorkerVars(namespaces ...string) WorkerOption { + return func(w *workerOpt) error { + if slices.Contains(namespaces, "") { + return errors.New("a vars namespace cannot be empty") + } + w.vars = append(w.vars, namespaces...) + + return nil + } +} + // WithWorkerMaxFailures sets the maximum number of consecutive failures before panicking func WithWorkerMaxFailures(maxFailures int) WorkerOption { return func(w *workerOpt) error { diff --git a/phpmainthread.go b/phpmainthread.go index 23a544ced8..55ef42d8b8 100644 --- a/phpmainthread.go +++ b/phpmainthread.go @@ -105,6 +105,8 @@ func drainPHPThreads() { } doneWG.Wait() + // no PHP thread can read them anymore, and the engine is still up + freeWorkerVars() mainThread.state.Set(state.Done) mainThread.state.WaitFor(state.Reserved) C.frankenphp_destroy_thread_metrics() @@ -161,7 +163,7 @@ func (mainThread *phpMainThread) rebootAllThreads() bool { for _, thread := range rebootingThreads { rebootWg.Go(func() { - close(thread.drainChan) + thread.drain() if thread.state.WaitForStateWithTimeout(rebootGracePeriod, state.YieldingForReboot) { return } diff --git a/phpmainthread_test.go b/phpmainthread_test.go index d8a5b6742d..a58a361ade 100644 --- a/phpmainthread_test.go +++ b/phpmainthread_test.go @@ -357,6 +357,13 @@ func TestCorrectThreadCalculation(t *testing.T) { testThreadCalculation(t, 2, 5, &opt{maxThreads: 5, workers: []workerOpt{{num: 1, maxThreads: 3}}}) testThreadCalculation(t, 3, 5, &opt{maxThreads: 5, workers: []workerOpt{{num: 1, maxThreads: 4}, {num: 1, maxThreads: 4}}}) + // a background worker counts like any worker, with one thread unless told otherwise + oneBackgroundWorker := []workerOpt{{isBackgroundWorker: true}} + testThreadCalculation(t, maxProcs, maxProcs, &opt{workers: oneBackgroundWorker}) + testThreadCalculation(t, 2, 2, &opt{numThreads: 1, workers: oneBackgroundWorker}) + testThreadCalculation(t, 2, 4, &opt{maxThreads: 4, workers: oneBackgroundWorker}) + testThreadCalculationError(t, &opt{numThreads: 1, maxThreads: 1, workers: oneBackgroundWorker}) + // a worker never eats into num_threads, so there is no floor to miss; // max_threads still has to fit what starts testThreadCalculationError(t, &opt{numThreads: 1, maxThreads: 1, workers: oneWorkerThread}) diff --git a/phpthread.go b/phpthread.go index 6a285e0452..2dc47ccf73 100644 --- a/phpthread.go +++ b/phpthread.go @@ -39,11 +39,8 @@ type threadHandler interface { beforeScriptExecution() string afterScriptExecution(exitStatus int) frankenPHPContext() *frankenPHPContext - // drain is a hook called by drainWorkerThreads right before drainChan is - // closed. Handlers that need to wake up a thread parked in a blocking C - // call (e.g. by closing a stop pipe) plug their signal in here. All - // current handlers are no-ops; this is the seam later handler types use - // without having to modify drainWorkerThreads. + // called right before drainChan is closed, for the handlers that have a + // thread parked in a blocking C call to wake up; a no-op for the others drain() } @@ -123,7 +120,7 @@ func (thread *phpThread) shutdown() { return } - close(thread.drainChan) + thread.drain() thread.waitForExit(state.Done) @@ -168,7 +165,7 @@ func (thread *phpThread) setHandler(handler threadHandler) { return } - close(thread.drainChan) + thread.drain() thread.state.WaitFor(state.TransitionInProgress) thread.handler = handler @@ -176,6 +173,14 @@ func (thread *phpThread) setHandler(handler threadHandler) { thread.state.Set(state.TransitionComplete) } +// drain tells the handler to yield: drainChan wakes it up from a Go wait, the +// handler hook from a blocking C call, so a thread parked either way leaves +// without waiting for the force-kill +func (thread *phpThread) drain() { + thread.handler.drain() + close(thread.drainChan) +} + // transition to a new handler safely // is triggered by setHandler and executed on the PHP thread func (thread *phpThread) transitionToNewHandler() string { diff --git a/requestoptions.go b/requestoptions.go index 962727562f..5c05eee0e3 100644 --- a/requestoptions.go +++ b/requestoptions.go @@ -2,6 +2,7 @@ package frankenphp import ( "errors" + "fmt" "log/slog" "net/http" "path/filepath" @@ -208,10 +209,16 @@ func WithRequestBodyTimeout(timeout time.Duration) RequestOption { // WithWorkerName sets the worker that should handle the request func WithWorkerName(name string) RequestOption { return func(o *frankenPHPContext) error { - if name != "" { - o.worker = workersByName[name] + if name == "" { + return nil } + w := workersByName[name] + if w != nil && w.isBackgroundWorker { + return fmt.Errorf("background worker %q cannot handle requests", name) + } + o.worker = w + return nil } } diff --git a/server.go b/server.go index e757ab37f6..ef25cdc30f 100644 --- a/server.go +++ b/server.go @@ -132,6 +132,12 @@ func (s *Server) Name() string { func (s *Server) addWorker(w *worker) error { s.workers = append(s.workers, w) + + // background workers never serve requests, so they are not matched at all + if w.isBackgroundWorker { + return nil + } + if w.matchRequest != nil { s.workersWithRequestMatcher = append(s.workersWithRequestMatcher, w) return nil diff --git a/testdata/bgworker/bad-vars.php b/testdata/bgworker/bad-vars.php new file mode 100644 index 0000000000..b788550e79 --- /dev/null +++ b/testdata/bgworker/bad-vars.php @@ -0,0 +1,27 @@ +getNamespaces()['vars'][0]; +$errors = []; +try { + $handle->setVars($namespace, ['object' => new stdClass()]); +} catch (\Throwable $e) { + $errors[] = get_class($e) . ': ' . $e->getMessage(); +} +try { + $handle->setVars('undeclared', []); +} catch (\Throwable $e) { + $errors[] = get_class($e) . ': ' . $e->getMessage(); +} +file_put_contents($_SERVER['BG_SENTINEL'], implode("\n", $errors)); +$handle->setVars($namespace, []); +$stream = $handle->getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/basic.php b/testdata/bgworker/basic.php new file mode 100644 index 0000000000..fcfaffc7df --- /dev/null +++ b/testdata/bgworker/basic.php @@ -0,0 +1,21 @@ +getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/consumer.php b/testdata/bgworker/consumer.php new file mode 100644 index 0000000000..32cdee5b5a --- /dev/null +++ b/testdata/bgworker/consumer.php @@ -0,0 +1,21 @@ +getMessage(); +} +file_put_contents($_SERVER['BG_SENTINEL'], $result); +$handle = new \FrankenPHP\WorkerHandle(); +foreach ($handle->getNamespaces()['vars'] as $namespace) { + $handle->setVars($namespace, ['consumed' => $result]); +} +$stream = $handle->getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/count.php b/testdata/bgworker/count.php new file mode 100644 index 0000000000..cb7d1e2f86 --- /dev/null +++ b/testdata/bgworker/count.php @@ -0,0 +1,15 @@ +getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/crash-after-ready.php b/testdata/bgworker/crash-after-ready.php new file mode 100644 index 0000000000..633f7af61c --- /dev/null +++ b/testdata/bgworker/crash-after-ready.php @@ -0,0 +1,9 @@ +tick(); +exit(1); diff --git a/testdata/bgworker/crash.php b/testdata/bgworker/crash.php new file mode 100644 index 0000000000..30eb27d664 --- /dev/null +++ b/testdata/bgworker/crash.php @@ -0,0 +1,31 @@ +getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/cycle.php b/testdata/bgworker/cycle.php new file mode 100644 index 0000000000..a171626ea6 --- /dev/null +++ b/testdata/bgworker/cycle.php @@ -0,0 +1,8 @@ +tick(); +usleep(150000); diff --git a/testdata/bgworker/early-return.php b/testdata/bgworker/early-return.php new file mode 100644 index 0000000000..590cd686ff --- /dev/null +++ b/testdata/bgworker/early-return.php @@ -0,0 +1,5 @@ +setVars('enum', ['suit' => EnumSuit::Hearts, 'n' => ++$publications]); +$stream = $handle->getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, 0, 10000); + $handle->setVars('enum', ['suit' => EnumSuit::Hearts, 'n' => ++$publications]); +} diff --git a/testdata/bgworker/exit-after-tick.php b/testdata/bgworker/exit-after-tick.php new file mode 100644 index 0000000000..27ba019e3d --- /dev/null +++ b/testdata/bgworker/exit-after-tick.php @@ -0,0 +1,7 @@ +tick(); diff --git a/testdata/bgworker/fail-then-succeed.php b/testdata/bgworker/fail-then-succeed.php new file mode 100644 index 0000000000..6897a237e0 --- /dev/null +++ b/testdata/bgworker/fail-then-succeed.php @@ -0,0 +1,18 @@ +getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/fetch-no-tick.php b/testdata/bgworker/fetch-no-tick.php new file mode 100644 index 0000000000..92c7e8c639 --- /dev/null +++ b/testdata/bgworker/fetch-no-tick.php @@ -0,0 +1,7 @@ + $_SERVER['FRANKENPHP_WORKER'] ?? 'unset', + 'background' => $_SERVER['FRANKENPHP_WORKER_BACKGROUND'] ?? 'unset', +], true)); +$handle = new \FrankenPHP\WorkerHandle(); +$stream = $handle->getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/loop.php b/testdata/bgworker/loop.php new file mode 100644 index 0000000000..8e9c8d9b12 --- /dev/null +++ b/testdata/bgworker/loop.php @@ -0,0 +1,16 @@ +getStream(); +do { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} while ($handle->tick()); diff --git a/testdata/bgworker/multi-vars.php b/testdata/bgworker/multi-vars.php new file mode 100644 index 0000000000..67d9dcde81 --- /dev/null +++ b/testdata/bgworker/multi-vars.php @@ -0,0 +1,16 @@ +getNamespaces(); +foreach ($namespaces['vars'] as $namespace) { + $handle->setVars($namespace, ['namespace' => $namespace, 'all' => $namespaces]); +} +$stream = $handle->getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/named.php b/testdata/bgworker/named.php new file mode 100644 index 0000000000..3daf46d4dd --- /dev/null +++ b/testdata/bgworker/named.php @@ -0,0 +1,20 @@ +getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/never-ticks.php b/testdata/bgworker/never-ticks.php new file mode 100644 index 0000000000..5f33d5f1cb --- /dev/null +++ b/testdata/bgworker/never-ticks.php @@ -0,0 +1,18 @@ +getStream(); +fgets($stream); +fgets($stream); diff --git a/testdata/bgworker/no-time-limit.php b/testdata/bgworker/no-time-limit.php new file mode 100644 index 0000000000..3585f8a19f --- /dev/null +++ b/testdata/bgworker/no-time-limit.php @@ -0,0 +1,12 @@ +tick(); +fgets($handle->getStream()); diff --git a/testdata/bgworker/poll-interface.php b/testdata/bgworker/poll-interface.php new file mode 100644 index 0000000000..917ea897f1 --- /dev/null +++ b/testdata/bgworker/poll-interface.php @@ -0,0 +1,16 @@ + $handle instanceof \Io\Poll\Handle, + 'internal' => (new ReflectionClass(\Io\Poll\Handle::class))->isInternal(), +])); +$stream = $handle->getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/poll.php b/testdata/bgworker/poll.php new file mode 100644 index 0000000000..ee8d50a8a5 --- /dev/null +++ b/testdata/bgworker/poll.php @@ -0,0 +1,14 @@ +add($handle, [\Io\Poll\Event::Read]); + +while ($handle->tick()) { + file_put_contents($_SERVER['BG_SENTINEL'], $poll->getBackend()->name); + $poll->wait(); +} diff --git a/testdata/bgworker/pool.php b/testdata/bgworker/pool.php new file mode 100644 index 0000000000..d9d53024ef --- /dev/null +++ b/testdata/bgworker/pool.php @@ -0,0 +1,13 @@ + 1 threads share the name; each touches a file of +// its own under BG_SENTINEL_DIR, then parks on its own handle. +set_time_limit(0); +@touch($_SERVER['BG_SENTINEL_DIR'] . DIRECTORY_SEPARATOR . bin2hex(random_bytes(8))); +$handle = new \FrankenPHP\WorkerHandle(); +$stream = $handle->getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/publisher.php b/testdata/bgworker/publisher.php new file mode 100644 index 0000000000..10fa2ef869 --- /dev/null +++ b/testdata/bgworker/publisher.php @@ -0,0 +1,22 @@ +setVars($handle->getNamespaces()['vars'][0], [ + 'answer' => 42, + 'value' => $_SERVER['BG_PUBLISH_VALUE'] ?? 'default', + 'nested' => ['a' => 1, 'list' => [true, null, 1.5, 'x']], + 'worker' => $_SERVER['FRANKENPHP_WORKER_BACKGROUND'], +]); +$stream = $handle->getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/read.php b/testdata/bgworker/read.php new file mode 100644 index 0000000000..f90affa1f1 --- /dev/null +++ b/testdata/bgworker/read.php @@ -0,0 +1,11 @@ +tick(); +fgets($handle->getStream()); diff --git a/testdata/bgworker/readable.php b/testdata/bgworker/readable.php new file mode 100644 index 0000000000..86db0d0761 --- /dev/null +++ b/testdata/bgworker/readable.php @@ -0,0 +1,26 @@ +getStream(); +$poll = static function () use ($stream): string { + $read = [$stream]; + $write = $except = null; + + return stream_select($read, $write, $except, 0) > 0 ? 'readable' : 'quiet'; +}; + +$seen = ['start:' . $poll()]; +$handle->tick(); +$seen[] = 'after tick:' . $poll(); +$handle->tick(); +$seen[] = 'after second tick:' . $poll(); +file_put_contents($_SERVER['BG_SENTINEL'], implode(' ', $seen)); + +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/recv.php b/testdata/bgworker/recv.php new file mode 100644 index 0000000000..048066c6a7 --- /dev/null +++ b/testdata/bgworker/recv.php @@ -0,0 +1,12 @@ +tick(); +stream_socket_recvfrom($handle->getStream(), 1); diff --git a/testdata/bgworker/refetch.php b/testdata/bgworker/refetch.php new file mode 100644 index 0000000000..a0948e4e44 --- /dev/null +++ b/testdata/bgworker/refetch.php @@ -0,0 +1,21 @@ +getStream(); +$again = $handle->getStream(); +$result = $first === $again ? 'same' : 'different'; + +fclose($first); +$third = $handle->getStream(); +$result .= $third === $first ? ' then same' : ' then fresh'; + +$other = (new \FrankenPHP\WorkerHandle())->getStream(); +$result .= $other === $third ? ' then shared' : ' then its own'; + +file_put_contents($_SERVER['BG_SENTINEL'], $result); +$handle->tick(); +fgets($third); diff --git a/testdata/bgworker/republish.php b/testdata/bgworker/republish.php new file mode 100644 index 0000000000..2589e34b3d --- /dev/null +++ b/testdata/bgworker/republish.php @@ -0,0 +1,18 @@ +setVars($handle->getNamespaces()['vars'][0], ['answer' => 42, 'nested' => ['a' => 1, 'list' => [true, null, 1.5, 'x']]]); +$stream = $handle->getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/slow-boot.php b/testdata/bgworker/slow-boot.php new file mode 100644 index 0000000000..56fc7b4a9c --- /dev/null +++ b/testdata/bgworker/slow-boot.php @@ -0,0 +1,11 @@ +tick(); diff --git a/testdata/bgworker/stuck.php b/testdata/bgworker/stuck.php new file mode 100644 index 0000000000..578986908d --- /dev/null +++ b/testdata/bgworker/stuck.php @@ -0,0 +1,19 @@ +tick(); +if (!file_exists($_SERVER['BG_ONCE'])) { + @touch($_SERVER['BG_ONCE']); + sleep(60); +} +@touch($_SERVER['BG_SENTINEL']); +$stream = $handle->getStream(); +while ($handle->tick()) { + $read = [$stream]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/tick.php b/testdata/bgworker/tick.php new file mode 100644 index 0000000000..ed9f24d02e --- /dev/null +++ b/testdata/bgworker/tick.php @@ -0,0 +1,17 @@ +tick() ? 'true' : 'false'; +$seen[] = $handle->tick() ? 'true' : 'false'; +file_put_contents($_SERVER['BG_SENTINEL'], implode(' ', $seen)); + +$read = [$handle->getStream()]; +$write = $except = null; +stream_select($read, $write, $except, null); + +$seen[] = $handle->tick() ? 'true' : 'false'; +$seen[] = $handle->tick() ? 'true' : 'false'; +file_put_contents($_SERVER['BG_SENTINEL'], implode(' ', $seen)); diff --git a/testdata/enum-suit.php b/testdata/enum-suit.php new file mode 100644 index 0000000000..f7c35e4abb --- /dev/null +++ b/testdata/enum-suit.php @@ -0,0 +1,7 @@ +getMessage(), "\n"; + } +}; + +$report('unserialize', static fn () => unserialize('O:23:"FrankenPHP\WorkerHandle":0:{}')); +$report('reflection', static fn () => (new ReflectionClass(\FrankenPHP\WorkerHandle::class))->newInstanceWithoutConstructor()); +$report('construct', static fn () => new \FrankenPHP\WorkerHandle()); diff --git a/testdata/handle-outside.php b/testdata/handle-outside.php new file mode 100644 index 0000000000..12f6141aa7 --- /dev/null +++ b/testdata/handle-outside.php @@ -0,0 +1,8 @@ +getMessage(); +} diff --git a/testdata/persist-roundtrip.php b/testdata/persist-roundtrip.php index f2bcec0627..fa78a876ed 100644 --- a/testdata/persist-roundtrip.php +++ b/testdata/persist-roundtrip.php @@ -86,3 +86,16 @@ function same(mixed $actual, mixed $expected, string $label): void { } catch (\LogicException) { echo "OK nested stdClass rejected\n"; } + +// A literal array is opcache-immutable, which the persistent tree copies +// rather than borrows: round-trip the same literal more times than the +// refcount of 2 opcache keeps on it, so nothing shared is ever released. +for ($i = 0; $i < 3; ++$i) { + $out = $rt(['immutable' => ['nested' => [1, 2, 3]], 'i' => 'literal']); + if ($out !== ['immutable' => ['nested' => [1, 2, 3]], 'i' => 'literal']) { + echo "FAIL immutable literal exposed repeatedly (round $i)\n"; + return; + } + unset($out); +} +echo "OK immutable literal exposed repeatedly\n"; diff --git a/testdata/vars-at-boot.php b/testdata/vars-at-boot.php new file mode 100644 index 0000000000..1eaa90bc35 --- /dev/null +++ b/testdata/vars-at-boot.php @@ -0,0 +1,13 @@ +getMessage(); +} +while (frankenphp_handle_request(function () use ($vars): void { + echo json_encode($vars); +})) { +} diff --git a/testdata/vars-enum.php b/testdata/vars-enum.php new file mode 100644 index 0000000000..d562d921cf --- /dev/null +++ b/testdata/vars-enum.php @@ -0,0 +1,15 @@ +name, ' kept:', implode(',', array_map('hex2bin', array_keys($seen))); diff --git a/testdata/vars-fatal.php b/testdata/vars-fatal.php new file mode 100644 index 0000000000..a83d7a00c4 --- /dev/null +++ b/testdata/vars-fatal.php @@ -0,0 +1,12 @@ +getMessage(); +} diff --git a/testdata/worker-name.php b/testdata/worker-name.php new file mode 100644 index 0000000000..b9f15a047d --- /dev/null +++ b/testdata/worker-name.php @@ -0,0 +1,11 @@ += 0 { + C.frankenphp_worker_close_stop_sock(C.intptr_t(s)) + } +} + +func (handler *backgroundWorkerThread) beforeScriptExecution() string { + return handler.workerLifecycle.beforeScriptExecution(handler.startScript) +} + +// startScript keeps trying to start the script: unlike an HTTP worker, whose +// setup cannot fail, a background worker needs a socket pair per run +func (handler *backgroundWorkerThread) startScript() string { + for { + err := handler.setupScript() + if err == nil { + return handler.worker.fileName + } + + if globalLogger.Enabled(globalCtx, slog.LevelError) { + globalLogger.LogAttrs(globalCtx, slog.LevelError, "failed to start background worker", slog.String("worker", handler.worker.name), slog.Int("thread", handler.thread.threadIndex), slog.Any("error", err)) + } + + // fail fast during startup so Init() surfaces the error to the + // operator; past startup, back off and retry like a crash + if reportStartupFailure(err) { + handler.thread.state.Set(state.ShuttingDown) + + return handler.beforeScriptExecution() + } + + handler.backoff() + if !handler.state.Is(state.Ready) && !handler.state.Is(state.TransitionComplete) { + // drained during the backoff (shutdown, reboot, transition) + return handler.beforeScriptExecution() + } + } +} + +// setupScript marks the thread as a background worker on the C side and takes +// ownership of the Go side's end of its stop socket pair. +func (handler *backgroundWorkerThread) setupScript() error { + s := int64(C.frankenphp_set_background_worker_and_get_stop_sock()) + if s < 0 { + return fmt.Errorf("failed to create the stop socket pair of background worker %q", handler.worker.name) + } + handler.stopSock.Store(s) + + switch handler.state.Get() { + case state.ShuttingDown, state.Rebooting, state.ForceRebooting, state.TransitionRequested: + // a concurrent drain may have run before the socket was published; + // close it now so the script observes EOF immediately + handler.drain() + } + + fc, err := newWorkerDummyContext(handler.worker) + if err != nil { + handler.drain() + return err + } + handler.thread.contextMu.Lock() + handler.context = fc + handler.thread.contextMu.Unlock() + + handler.isBootingScript = true + handler.publishedVars = make(map[string]bool, len(handler.worker.varsNamespaces)) + handler.runStartedAt = time.Now() + metrics.StartWorker(handler.worker.name) + // the run's logger and context, not the globals: a shutdown finishing + // meanwhile resets those, and Stop() does not wait for this callback + logger, ctx, name, threadIndex := fc.logger, fc.ctx, handler.worker.name, handler.thread.threadIndex + handler.bootTimer = time.AfterFunc(backgroundBootWarnDelay, func() { + if logger.Enabled(ctx, slog.LevelWarn) { + logger.LogAttrs(ctx, slog.LevelWarn, "background worker has not called WorkerHandle::tick() yet, it is not ready", slog.String("worker", name), slog.Int("thread", threadIndex)) + } + }) + + if fc.logger.Enabled(fc.ctx, slog.LevelDebug) { + fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "starting background worker", slog.String("worker", handler.worker.name), slog.Int("thread", handler.thread.threadIndex)) + } + + // the thread stays in TransitionComplete until the script calls + // WorkerHandle::tick(), see go_frankenphp_background_worker_ready + + return nil +} + +func (handler *backgroundWorkerThread) afterScriptExecution(exitStatus int) { + // release the socket on every exit path so the next run gets a fresh + // pair; drain() already took it when the exit was drain-triggered + handler.drain() + worker := handler.worker + handler.thread.contextMu.Lock() + handler.context = nil + handler.thread.contextMu.Unlock() + + handler.stopBootTimer() + handler.state.MarkAsWaiting(false) + + // exit past the ready point, cooperative or a crash: re-run the script + // without counting toward max_consecutive_failures, that cap is about a + // script that never boots. An HTTP worker is paced by traffic, a + // background one reaches its ready point on its own, so pace it here + if !handler.isBootingScript { + if exitStatus == 0 { + metrics.StopWorker(worker.name, StopReasonRestart) + + if globalLogger.Enabled(globalCtx, slog.LevelDebug) { + globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "restarting background worker", slog.String("worker", worker.name), slog.Int("thread", handler.thread.threadIndex)) + } + } else { + metrics.StopWorker(worker.name, StopReasonCrash) + + if globalLogger.Enabled(globalCtx, slog.LevelWarn) { + globalLogger.LogAttrs(globalCtx, slog.LevelWarn, "background worker crashed, restarting", slog.String("worker", worker.name), slog.Int("thread", handler.thread.threadIndex), slog.Int("exit_status", exitStatus), slog.Int("crashes", handler.crashCount)) + } + } + + // a worker processing a batch and returning is not a worker spinning + // on an immediate exit; a crash never resets the count, however long + // the run lasted + if exitStatus == 0 && time.Since(handler.runStartedAt) > minHealthyRun { + handler.crashCount = 0 + } + handler.wait(restartBackoff(handler.crashCount)) + handler.crashCount++ + + return + } + + // the script exited before calling WorkerHandle::tick(), a clean exit + // included; StopReasonBootFailure skips the ready-gauge decrement, + // matching the ReadyWorker call that never happened + metrics.StopWorker(worker.name, StopReasonBootFailure) + + // max_consecutive_failures only fails hard during startup, through + // startupFailChan; past it a failing worker keeps restarting with a + // louder log line rather than leave the server in a broken half-state. + // A worker Init() gave up on stops here whatever the cap says, its + // timeout already reported it + pastCap := worker.maxConsecutiveFailures >= 0 && handler.failureCount >= worker.maxConsecutiveFailures + if worker.bootTimedOut.Load() { + handler.thread.state.Set(state.ShuttingDown) + + return + } + if pastCap && !watcherIsEnabled { + var err error + if exitStatus == 0 { + err = fmt.Errorf("background worker %s exits without calling WorkerHandle::tick()", worker.fileName) + } else { + err = fmt.Errorf("too many consecutive failures: background worker %s keeps crashing", worker.fileName) + } + if reportStartupFailure(err) { + handler.thread.state.Set(state.ShuttingDown) + return + } + } + + logLevel := slog.LevelWarn + logMsg := "background worker failed before calling WorkerHandle::tick(), restarting" + if exitStatus == 0 { + logMsg = "background worker exited without calling WorkerHandle::tick(), restarting" + } + if pastCap { + logLevel = slog.LevelError + logMsg = "background worker exceeded max_consecutive_failures, still restarting" + } + if globalLogger.Enabled(globalCtx, logLevel) { + globalLogger.LogAttrs(globalCtx, logLevel, logMsg, slog.String("worker", worker.name), slog.Int("thread", handler.thread.threadIndex), slog.Int("failures", handler.failureCount), slog.Int("exit_status", exitStatus)) + } + + handler.backoff() +} + +func (handler *backgroundWorkerThread) stopBootTimer() { + if handler.bootTimer != nil { + handler.bootTimer.Stop() + handler.bootTimer = nil + } +} + +//export go_frankenphp_background_worker_ready +func go_frankenphp_background_worker_ready(threadIndex C.uintptr_t) *C.char { + // called on the PHP thread by the first WorkerHandle::tick() of a run; + // that function throws on every other thread kind + handler, ok := phpThreads[threadIndex].handler.(*backgroundWorkerThread) + if !ok { + panic("WorkerHandle::tick() called on a thread that is not a background worker") + } + + if handler.isBootingScript { + // readers never wait for a snapshot past the ready point + for _, ns := range handler.worker.varsNamespaces { + if !handler.publishedVars[ns] { + return C.CString("FrankenPHP\\WorkerHandle::tick(): publish the vars namespace " + strconv.Quote(ns) + " with setVars() before the first tick") + } + } + + handler.isBootingScript = false + // the boot succeeded, only consecutive boot failures count + handler.failureCount = 0 + handler.stopBootTimer() + handler.worker.markReady() + metrics.ReadyWorker(handler.worker.name) + // parked from now on as far as the threads state endpoint is concerned + handler.state.MarkAsWaiting(true) + + // initWorkers() waits for this state, so a script that fails before + // its first tick still fails Init() + if handler.state.Is(state.TransitionComplete) { + handler.state.Set(state.Ready) + } + } + + return nil +} + +// backoff waits before the next run of a crashed script, see restartBackoff +func (handler *backgroundWorkerThread) backoff() { + handler.wait(restartBackoff(handler.failureCount)) + handler.failureCount++ +} + +// wait sleeps between two runs, cut short by a drain, which the next +// beforeScriptExecution() picks up from the state +func (handler *backgroundWorkerThread) wait(d time.Duration) { + select { + case <-handler.thread.drainChan: + case <-time.After(d): + } +} diff --git a/threadworker.go b/threadworker.go index 8a308b4d87..8e823fbbe7 100644 --- a/threadworker.go +++ b/threadworker.go @@ -15,9 +15,8 @@ import ( // executes the PHP worker script in a loop // implements the threadHandler interface type workerThread struct { - state *state.ThreadState - thread *phpThread - worker *worker + workerLifecycle + dummyFrankenPHPContext *frankenPHPContext workerFrankenPHPContext *frankenPHPContext isBootingScript bool // true if the worker has not reached frankenphp_handle_request yet @@ -26,49 +25,19 @@ type workerThread struct { } func convertToWorkerThread(thread *phpThread, worker *worker) { - thread.setHandler(&workerThread{ - state: thread.state, - thread: thread, - worker: worker, - }) + thread.setHandler(&workerThread{workerLifecycle: newWorkerLifecycle(thread, worker)}) worker.attachThread(thread) } -// beforeScriptExecution returns the name of the script or an empty string on shutdown func (handler *workerThread) beforeScriptExecution() string { - switch handler.state.Get() { - case state.TransitionRequested: - if handler.worker.onThreadShutdown != nil { - handler.worker.onThreadShutdown(handler.thread.threadIndex) - } - handler.worker.detachThread(handler.thread) - return handler.thread.transitionToNewHandler() - case state.Ready, state.TransitionComplete: - handler.thread.updateContext(true) - if handler.worker.onThreadReady != nil { - handler.worker.onThreadReady(handler.thread.threadIndex) - } - - setupWorkerScript(handler, handler.worker) + return handler.workerLifecycle.beforeScriptExecution(handler.startScript) +} - return handler.worker.fileName - case state.Rebooting, state.ForceRebooting: - return "" - case state.RebootReady: - handler.requestCount = 0 - handler.state.Set(state.Ready) - return handler.beforeScriptExecution() - case state.ShuttingDown: - if handler.worker.onThreadShutdown != nil { - handler.worker.onThreadShutdown(handler.thread.threadIndex) - } - handler.worker.detachThread(handler.thread) +// startScript runs the worker script; it always has one to run +func (handler *workerThread) startScript() string { + setupWorkerScript(handler, handler.worker) - // signal to stop - return "" - default: - panic("unexpected state: " + handler.state.Name()) - } + return handler.worker.fileName } func (handler *workerThread) afterScriptExecution(exitStatus int) { @@ -149,8 +118,8 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) { return } - if worker.maxConsecutiveFailures >= 0 && startupFailChan != nil && !watcherIsEnabled && handler.failureCount >= worker.maxConsecutiveFailures { - startupFailChan <- fmt.Errorf("too many consecutive failures: worker %s has not reached frankenphp_handle_request()", worker.fileName) + if worker.maxConsecutiveFailures >= 0 && !watcherIsEnabled && handler.failureCount >= worker.maxConsecutiveFailures && + reportStartupFailure(fmt.Errorf("too many consecutive failures: worker %s has not reached frankenphp_handle_request()", worker.fileName)) { handler.thread.state.Set(state.ShuttingDown) return } @@ -168,13 +137,29 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) { } } - // wait a bit and try again (exponential backoff) - backoffDuration := time.Duration(handler.failureCount*handler.failureCount*100) * time.Millisecond - if backoffDuration > time.Second { - backoffDuration = time.Second - } + // wait a bit and try again + time.Sleep(restartBackoff(handler.failureCount)) handler.failureCount++ - time.Sleep(backoffDuration) +} + +// maxRestartBackoff is the longest wait between two runs of a failing script +const maxRestartBackoff = time.Second + +// minHealthyRun is how long a run has to last to count as one that did +// something: a shorter one paces the next, see restartBackoff +const minHealthyRun = 100 * time.Millisecond + +// restartBackoff is the wait before a worker script is re-run after a +// failure: quadratic in the number of consecutive failures, capped at one +// second; shared by HTTP and background workers. The cap comes before the +// multiplication, which overflows a duration past some 300k failures, a +// count a crash loop reaches on its own in a few days +func restartBackoff(failures int) time.Duration { + if failures >= 4 { + return maxRestartBackoff + } + + return time.Duration(failures*failures*100) * time.Millisecond } // waitForWorkerRequest is called during frankenphp_handle_request in the php worker script. diff --git a/threadworker_test.go b/threadworker_test.go new file mode 100644 index 0000000000..24e5a2e410 --- /dev/null +++ b/threadworker_test.go @@ -0,0 +1,20 @@ +package frankenphp + +import ( + "math" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestRestartBackoff(t *testing.T) { + assert.Equal(t, time.Duration(0), restartBackoff(0)) + assert.Equal(t, 100*time.Millisecond, restartBackoff(1)) + assert.Equal(t, 400*time.Millisecond, restartBackoff(2)) + assert.Equal(t, 900*time.Millisecond, restartBackoff(3)) + assert.Equal(t, time.Second, restartBackoff(4)) + // a crash loop counts without bound, the quadratic must not overflow + assert.Equal(t, time.Second, restartBackoff(303701)) + assert.Equal(t, time.Second, restartBackoff(math.MaxInt32)) +} diff --git a/worker.go b/worker.go index 8f1fe524f2..1c88856714 100644 --- a/worker.go +++ b/worker.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "sync" "sync/atomic" "time" @@ -31,10 +32,27 @@ type worker struct { threads []*phpThread threadMutex sync.RWMutex maxConsecutiveFailures int - onThreadReady func(int) - onThreadShutdown func(int) - queuedRequests atomic.Int32 - server *Server + bootTimeout time.Duration + // Init() gave up waiting for the ready point: the run is drained and must not start again + bootTimedOut atomic.Bool + onThreadReady func(int) + onThreadShutdown func(int) + queuedRequests atomic.Int32 + server *Server + isBackgroundWorker bool + // readyOnce is closed the first time a thread of a background worker + // reaches its ready point; frankenphp_get_vars() readers wait on it + readyOnce chan struct{} + readyClose sync.Once + // the vars namespaces the worker publishes, in declaration order, and + // the snapshot of each, see WorkerHandle::setVars() + varsNamespaces []string + vars map[string]*varsSlot +} + +// markReady records that the background worker reached its ready point once +func (worker *worker) markReady() { + worker.readyClose.Do(func() { close(worker.readyOnce) }) } var ( @@ -43,9 +61,23 @@ var ( globalWorkersByPath map[string]*worker watcherIsEnabled bool startupFailChan chan error + // the only window where a boot failure reaches startupFailChan: past it, the handlers log and keep restarting + startupPhase atomic.Bool + // closed by the first boot failure of the startup, so the threads waiting + // on the ready point of another worker give up, see waitVarsReady() + startupFailed chan struct{} + startupFailedOnce *sync.Once ) +// DefaultWorkerBootTimeout is how long a background worker may take to reach +// its first WorkerHandle::tick() before Init() fails, on PHP builds without +// Zend max execution timers: FrankenPHP disables max_execution_time there, +// and this matches its default. Elsewhere max_execution_time bounds the +// bootstrap instead. WithWorkerBootTimeout() sets it on any build. +const DefaultWorkerBootTimeout = 30 * time.Second + func initWorkers(opts []workerOpt) error { + varsOwners = namespaceOwners{} if len(opts) == 0 { return nil } @@ -68,28 +100,63 @@ func initWorkers(opts []workerOpt) error { totalThreadsToStart += w.num workers = append(workers, w) workersByName[w.name] = w - if w.server == nil { + varsOwners.add(w, w.varsNamespaces) + if w.server != nil { + if err := w.server.addWorker(w); err != nil { + return err + } + } else if !w.isBackgroundWorker { + // background workers are never matched against requests globalWorkersByPath[w.fileName] = w globalWorkersByPath[w.declaredFileName] = w - } else if err := w.server.addWorker(w); err != nil { - return err } } startupFailChan = make(chan error, totalThreadsToStart) + startupFailed, startupFailedOnce = make(chan struct{}), &sync.Once{} + startupPhase.Store(true) for _, w := range workers { for range w.num { thread := getInactivePHPThread() - convertToWorkerThread(thread, w) + if w.isBackgroundWorker { + convertToBackgroundWorkerThread(thread, w) + } else { + convertToWorkerThread(thread, w) + } + bootTimeout := time.Duration(0) + if w.isBackgroundWorker { + bootTimeout = w.bootTimeout + } workersReady.Go(func() { - thread.state.WaitFor(state.Ready, state.ShuttingDown, state.Done) + if bootTimeout <= 0 { + thread.state.WaitFor(state.Ready, state.ShuttingDown, state.Done) + + return + } + + if thread.state.WaitForStateWithTimeout(bootTimeout, state.Ready, state.ShuttingDown, state.Done) { + return + } + + // the script never ticked: stop the thread, so no run starts + // after this one, which ends at once if it waits on its handle + // and through the force-kill of the shutdown otherwise + if !thread.state.CompareAndSwap(state.TransitionComplete, state.ShuttingDown) { + return + } + w.bootTimedOut.Store(true) + reportStartupFailure(fmt.Errorf("background worker %q did not call WorkerHandle::tick() within %s", w.name, bootTimeout)) + if handler, ok := thread.handler.(*backgroundWorkerThread); ok { + handler.drain() + } }) } } workersReady.Wait() + startupPhase.Store(false) select { case err := <-startupFailChan: @@ -97,12 +164,28 @@ func initWorkers(opts []workerOpt) error { return fmt.Errorf("failed to initialize workers: %w", err) default: // all workers started successfully - startupFailChan = nil } return nil } +// reportStartupFailure hands a boot failure to initWorkers() while it waits, +// so Init() fails, and reports whether it did: past the startup phase the +// handler logs it and keeps restarting. It never blocks. +func reportStartupFailure(err error) bool { + if !startupPhase.Load() { + return false + } + + select { + case startupFailChan <- err: + default: + } + startupFailedOnce.Do(func() { close(startupFailed) }) + + return true +} + // resolveWorkerFile turns a declared filename into the path a worker runs, // and names the worker after it when the declaration left the name out. // @@ -138,12 +221,68 @@ func resolveWorkerFile(o workerOpt) (workerOpt, error) { return o, nil } +// declarationIndex tells what the workers declared before took: names are +// global, paths and namespaces belong to a scope, a nil server standing for +// the global workers +type declarationIndex struct { + names func(name string) bool + paths, vars func(server *Server, key string) bool +} + +// scopeSet records keys per scope, a nil server for the global workers +type scopeSet map[*Server]map[string]bool + +func (s scopeSet) has(server *Server, key string) bool { return s[server][key] } + +func (s scopeSet) add(server *Server, key string) { + if s[server] == nil { + s[server] = make(map[string]bool) + } + s[server][key] = true +} + +// namespaceOwners maps the namespaces declared in a scope to the worker that +// owns them, a nil server standing for the global workers +type namespaceOwners map[*Server]map[string]*worker + +func (o namespaceOwners) add(w *worker, namespaces []string) { + for _, ns := range namespaces { + if o[w.server] == nil { + o[w.server] = make(map[string]*worker) + } + o[w.server][ns] = w + } +} + +// resolve finds the owner of a namespace within a server first, then among +// the global workers +func (o namespaceOwners) resolve(server *Server, ns string) *worker { + if w := o[server][ns]; w != nil { + return w + } + + return o[nil][ns] +} + // checkWorkerDeclaration holds the rules a set of workers must follow, so -// Validate() and newWorker() answer the same way; pathTaken is asked for the -// global workers with a nil server. -func checkWorkerDeclaration(o workerOpt, nameTaken func(string) bool, pathTaken func(*Server, string) bool) error { - if o.server == nil { - if pathTaken(nil, o.fileName) { +// Validate() and newWorker() answer the same way; the paths and namespaces of +// the global workers are asked for with a nil server. +func checkWorkerDeclaration(o workerOpt, taken declarationIndex) error { + if o.isBackgroundWorker { + // never matched against requests, so several may share a script + if o.matchRequest != nil { + return fmt.Errorf("background worker %q cannot match requests", o.name) + } + if o.maxThreads > 0 { + return fmt.Errorf("background worker %q cannot set max_threads, it does not autoscale", o.name) + } + // Workers.SendRequest() and SendMessage() dispatch on requestChan, + // which no background thread reads + if o.extensionWorkers != nil { + return fmt.Errorf("background worker %q cannot be an extension worker, those handle requests", o.name) + } + } else if o.server == nil { + if taken.paths(nil, o.fileName) { return fmt.Errorf("two global workers cannot have the same filename: %q", o.fileName) } @@ -151,15 +290,38 @@ func checkWorkerDeclaration(o workerOpt, nameTaken func(string) bool, pathTaken if o.matchRequest != nil { return fmt.Errorf("worker %q has a request matcher but no server scope, use WithWorkerServerScope()", o.name) } - } else if o.matchRequest == nil && pathTaken(o.server, o.fileName) { + } else if o.matchRequest == nil && taken.paths(o.server, o.fileName) { // a matcher tells the workers of a server sharing a file apart return fmt.Errorf("two workers in a server cannot have the same filename: %q", o.fileName) } - if nameTaken(o.name) { + if taken.names(o.name) { return fmt.Errorf("two workers cannot have the same name: %q", o.name) } + scope := "two global workers" + if o.server != nil { + scope = "two workers in a server" + } + + if len(o.vars) > 0 { + if !o.isBackgroundWorker { + return fmt.Errorf("worker %q declares vars, only background workers publish them", o.name) + } + // one thread, so a snapshot has a single writer + if o.num > 1 { + return fmt.Errorf("background worker %q publishes vars, it runs a single thread", o.name) + } + } + for i, ns := range o.vars { + if slices.Contains(o.vars[:i], ns) { + return fmt.Errorf("worker %q declares the vars namespace %q twice", o.name, ns) + } + if taken.vars(o.server, ns) { + return fmt.Errorf("%s cannot publish the same vars namespace: %q", scope, ns) + } + } + return nil } @@ -169,16 +331,19 @@ func newWorker(o workerOpt) (*worker, error) { return nil, err } - nameTaken := func(name string) bool { return workersByName[name] != nil } - pathTaken := func(server *Server, path string) bool { - if server == nil { - return globalWorkersByPath[path] != nil - } + taken := declarationIndex{ + names: func(name string) bool { return workersByName[name] != nil }, + paths: func(server *Server, path string) bool { + if server == nil { + return globalWorkersByPath[path] != nil + } - return server.workersByPath[path] != nil + return server.workersByPath[path] != nil + }, + vars: func(server *Server, ns string) bool { return varsOwners[server][ns] != nil }, } - if err := checkWorkerDeclaration(o, nameTaken, pathTaken); err != nil { + if err := checkWorkerDeclaration(o, taken); err != nil { return nil, err } @@ -198,7 +363,22 @@ func newWorker(o workerOpt) (*worker, error) { } } - o.env["FRANKENPHP_WORKER\x00"] = "1" + // a script serving both roles tests which of the two is set + if o.isBackgroundWorker { + o.env["FRANKENPHP_WORKER_BACKGROUND\x00"] = o.name + } else { + o.env["FRANKENPHP_WORKER\x00"] = "1" + } + + // a background worker that never reaches its ready point would keep Init() + // waiting for ever where max_execution_time is disabled, see initWorkers() + bootTimeout := time.Duration(0) + if !Config().ZendMaxExecutionTimers { + bootTimeout = DefaultWorkerBootTimeout + } + if o.bootTimeoutIsSet { + bootTimeout = o.bootTimeout + } w := &worker{ name: o.name, @@ -211,9 +391,17 @@ func newWorker(o workerOpt) (*worker, error) { requestChan: make(chan *frankenPHPContext), threads: make([]*phpThread, 0, o.num), maxConsecutiveFailures: o.maxConsecutiveFailures, + bootTimeout: bootTimeout, onThreadReady: o.onThreadReady, onThreadShutdown: o.onThreadShutdown, server: o.server, + isBackgroundWorker: o.isBackgroundWorker, + readyOnce: make(chan struct{}), + varsNamespaces: o.vars, + vars: make(map[string]*varsSlot, len(o.vars)), + } + for _, ns := range o.vars { + w.vars[ns] = &varsSlot{} } w.configureMercure(&o) diff --git a/workerextension_test.go b/workerextension_test.go index e861ec97a3..dd268da702 100644 --- a/workerextension_test.go +++ b/workerextension_test.go @@ -77,6 +77,16 @@ func TestWorkerExtensionSendMessage(t *testing.T) { assert.Equal(t, "received message: Hello Workers", ret) } +// background workers never read requestChan, so an extension cannot send +// them requests or messages +func TestErrorIfExtensionWorkerIsBackground(t *testing.T) { + t.Cleanup(Shutdown) + + _, o := WithExtensionWorkers("backgroundExtension", "testdata/bgworker/basic.php", 1, WithWorkerBackground()) + + require.ErrorContains(t, Init(o), "cannot be an extension worker") +} + func TestErrorIf2WorkersHaveSameName(t *testing.T) { _, o1 := WithExtensionWorkers("duplicateWorker", "testdata/worker.php", 1) _, o2 := WithExtensionWorkers("duplicateWorker", "testdata/worker2.php", 1) diff --git a/workerlifecycle.go b/workerlifecycle.go new file mode 100644 index 0000000000..4d1809ee22 --- /dev/null +++ b/workerlifecycle.go @@ -0,0 +1,58 @@ +package frankenphp + +import "github.com/dunglas/frankenphp/internal/state" + +// workerLifecycle is the part of a worker thread that HTTP and background +// workers share: the thread, its worker, and the states both walk through +// between two runs of the script. Handlers embed it and pass the one step +// that differs, how a run starts. +type workerLifecycle struct { + state *state.ThreadState + thread *phpThread + worker *worker +} + +func newWorkerLifecycle(thread *phpThread, worker *worker) workerLifecycle { + return workerLifecycle{state: thread.state, thread: thread, worker: worker} +} + +// beforeScriptExecution returns the name of the script to run, or an empty +// string to stop the thread; startScript prepares a run and returns the +// script to execute, or an empty string to stop the thread +func (l *workerLifecycle) beforeScriptExecution(startScript func() string) string { + switch l.state.Get() { + case state.TransitionRequested: + l.detach() + + return l.thread.transitionToNewHandler() + case state.Ready, state.TransitionComplete: + l.thread.updateContext(true) + if l.worker.onThreadReady != nil { + l.worker.onThreadReady(l.thread.threadIndex) + } + + return startScript() + case state.Rebooting, state.ForceRebooting: + return "" + case state.RebootReady: + l.state.Set(state.Ready) + + return l.beforeScriptExecution(startScript) + case state.ShuttingDown: + l.detach() + + // signal to stop + return "" + default: + panic("unexpected state: " + l.state.Name()) + } +} + +// detach takes the thread off its worker, on the paths that stop running its +// script for good +func (l *workerLifecycle) detach() { + if l.worker.onThreadShutdown != nil { + l.worker.onThreadShutdown(l.thread.threadIndex) + } + l.worker.detachThread(l.thread) +} diff --git a/workervars.go b/workervars.go new file mode 100644 index 0000000000..c2d160588f --- /dev/null +++ b/workervars.go @@ -0,0 +1,233 @@ +package frankenphp + +// #include "frankenphp.h" +import "C" +import ( + "errors" + "runtime/cgo" + "strconv" + "sync" + "sync/atomic" +) + +// varsSnapshot is one publication of a vars namespace: a persistent +// HashTable, copied into request memory by each frankenphp_get_vars() +// reader. The copy runs in C with no lock held, a bailout may end it, so refs +// counts the slot and the readers holding the snapshot, and the last one to +// let go frees it. +type varsSnapshot struct { + table *C.HashTable + refs atomic.Int32 +} + +// release lets go of a reference, and hands back the table to free once +// nobody holds it anymore +func (s *varsSnapshot) release() *C.HashTable { + if s.refs.Add(-1) == 0 { + return s.table + } + + return nil +} + +// varsSlot holds the current snapshot of a vars namespace, published through +// WorkerHandle::setVars(). It belongs to the worker, not to a thread, so it +// survives script restarts and serves stale data meanwhile. +type varsSlot struct { + mu sync.RWMutex + current *varsSnapshot +} + +var ( + // the publisher of each vars namespace, per scope + varsOwners namespaceOwners + + // booting background workers blocked in frankenphp_get_vars() on other + // workers, keyed by waiter: a cycle would deadlock Init(), refuse it + varsWaitMu sync.Mutex + varsWaitOn = map[*worker]map[*worker]int{} +) + +// varsOwner resolves the publisher of a vars namespace: within the caller's +// server first, then among global workers +func varsOwner(fc *frankenPHPContext, ns string) (*worker, error) { + var server *Server + if fc != nil { + server = fc.server + } + w := varsOwners.resolve(server, ns) + if w == nil { + return nil, errors.New("frankenphp_get_vars(): no background worker publishes the vars namespace " + strconv.Quote(ns)) + } + + return w, nil +} + +// waitVarsReady blocks until target reached its ready point once. Requests +// cannot get here before that (activateServers() runs after initWorkers()), +// so a blocked caller is a worker still booting, a background one or the +// bootstrap of an HTTP one. Waits between background workers form a graph, +// and a cycle is refused instead of deadlocking Init(), as is any wait once +// the start failed +func waitVarsReady(target, caller *worker) error { + select { + case <-target.readyOnce: + return nil + default: + } + + if caller != nil { + varsWaitMu.Lock() + if caller == target || varsWaitReaches(target, caller) { + varsWaitMu.Unlock() + + return errors.New("frankenphp_get_vars(): circular dependency between background workers " + strconv.Quote(caller.name) + " and " + strconv.Quote(target.name)) + } + if varsWaitOn[caller] == nil { + varsWaitOn[caller] = map[*worker]int{} + } + varsWaitOn[caller][target]++ + varsWaitMu.Unlock() + + defer func() { + varsWaitMu.Lock() + if varsWaitOn[caller][target]--; varsWaitOn[caller][target] == 0 { + delete(varsWaitOn[caller], target) + } + if len(varsWaitOn[caller]) == 0 { + delete(varsWaitOn, caller) + } + varsWaitMu.Unlock() + }() + } + + select { + case <-target.readyOnce: + return nil + case <-mainThread.done: + return errors.New("frankenphp_get_vars(): FrankenPHP is shutting down") + case <-startupFailed: + return errors.New("frankenphp_get_vars(): FrankenPHP failed to start, " + strconv.Quote(target.name) + " is not ready") + } +} + +// varsWaitReaches reports whether from waits, transitively, on to; called +// with varsWaitMu held +func varsWaitReaches(from, to *worker) bool { + for next := range varsWaitOn[from] { + if next == to || varsWaitReaches(next, to) { + return true + } + } + + return false +} + +// freeWorkerVars releases the snapshots once no PHP thread can read them +// and before the engine goes away +func freeWorkerVars() { + for _, w := range workers { + for _, slot := range w.vars { + slot.mu.Lock() + if slot.current != nil { + if table := slot.current.release(); table != nil { + C.frankenphp_vars_free(table) + } + slot.current = nil + } + slot.mu.Unlock() + } + } +} + +// go_frankenphp_set_vars stores a snapshot in a namespace of the calling +// worker and hands back the one it replaces, for the C side to free unless a +// reader still copies it; on error, the C side frees the snapshot it passed +// +//export go_frankenphp_set_vars +func go_frankenphp_set_vars(threadIndex C.uintptr_t, namespace *C.char, namespaceLen C.size_t, table *C.HashTable, old **C.HashTable) *C.char { + handler, ok := phpThreads[threadIndex].handler.(*backgroundWorkerThread) + if !ok { + // refused on the C side already + return C.CString("FrankenPHP\\WorkerHandle::setVars() can only be called from a background worker") + } + + ns := C.GoStringN(namespace, C.int(namespaceLen)) + slot := handler.worker.vars[ns] + if slot == nil { + return C.CString("FrankenPHP\\WorkerHandle::setVars(): this worker does not publish the vars namespace " + strconv.Quote(ns)) + } + + s := &varsSnapshot{table: table} + s.refs.Store(1) + + slot.mu.Lock() + previous := slot.current + slot.current = s + slot.mu.Unlock() + if previous != nil { + *old = previous.release() + } + handler.publishedVars[ns] = true + + return nil +} + +// go_frankenphp_get_vars hands the current snapshot of a namespace to the C +// side, which copies it into request memory and then lets go of it through +// go_frankenphp_vars_done +// +//export go_frankenphp_get_vars +func go_frankenphp_get_vars(threadIndex C.uintptr_t, namespace *C.char, namespaceLen C.size_t, table **C.HashTable, snapshot *C.uintptr_t) *C.char { + thread := phpThreads[threadIndex] + ns := C.GoStringN(namespace, C.int(namespaceLen)) + target, err := varsOwner(thread.handler.frankenPHPContext(), ns) + if err != nil { + return C.CString(err.Error()) + } + + var caller *worker + if handler, ok := thread.handler.(*backgroundWorkerThread); ok && handler.isBootingScript { + caller = handler.worker + } + if err := waitVarsReady(target, caller); err != nil { + return C.CString(err.Error()) + } + + // a worker is ready only once every namespace it declares has a snapshot + slot := target.vars[ns] + slot.mu.RLock() + s := slot.current + s.refs.Add(1) + slot.mu.RUnlock() + + *table = s.table + *snapshot = C.uintptr_t(cgo.NewHandle(s)) + + return nil +} + +// go_frankenphp_vars_done lets go of the snapshot a frankenphp_get_vars() +// call copied, and hands back its table for the C side to free when its slot +// moved on meanwhile +// +//export go_frankenphp_vars_done +func go_frankenphp_vars_done(snapshot C.uintptr_t) *C.HashTable { + h := cgo.Handle(snapshot) + s := h.Value().(*varsSnapshot) + h.Delete() + + return s.release() +} + +// go_frankenphp_worker_vars lists the vars namespaces of the calling worker, +// for WorkerHandle::getNamespaces() +// +//export go_frankenphp_worker_vars +func go_frankenphp_worker_vars(threadIndex C.uintptr_t, array *C.zval) { + if handler, ok := phpThreads[threadIndex].handler.(*backgroundWorkerThread); ok { + for _, ns := range handler.worker.varsNamespaces { + C.frankenphp_array_append_string(array, toUnsafeChar(ns), C.size_t(len(ns))) + } + } +} diff --git a/workervars_test.go b/workervars_test.go new file mode 100644 index 0000000000..3ec6d27c7c --- /dev/null +++ b/workervars_test.go @@ -0,0 +1,293 @@ +package frankenphp_test + +import ( + "path/filepath" + "testing" + "time" + + "github.com/dunglas/frankenphp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// bgWorker declares a background worker from testdata/bgworker publishing the +// vars namespace named like it, scoped to server when one is given +func bgWorker(name, file string, env map[string]string, server *frankenphp.Server, extra ...frankenphp.WorkerOption) frankenphp.Option { + opts := []frankenphp.WorkerOption{frankenphp.WithWorkerBackground(), frankenphp.WithWorkerEnv(env), frankenphp.WithWorkerVars(name)} + if server != nil { + opts = append(opts, frankenphp.WithWorkerServerScope(server)) + } + + return frankenphp.WithWorkers(name, "testdata/bgworker/"+file, 1, append(opts, extra...)...) +} + +func TestVarsRoundTrip(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("publisher", "publisher.php", nil, server), frankenphp.WithNumThreads(2)) + + body := serverGet(t, server, "http://example.com/vars.php?ns=publisher") + assert.JSONEq(t, `{"answer":42,"value":"default","nested":{"a":1,"list":[true,null,1.5,"x"]},"worker":"publisher"}`, body) + + // a second read is a fresh copy of the same snapshot + assert.JSONEq(t, body, serverGet(t, server, "http://example.com/vars.php?ns=publisher")) + assert.Contains(t, serverGet(t, server, "http://example.com/vars.php?ns=nope"), `no background worker publishes the vars namespace "nope"`) +} + +// one worker publishes several namespaces, and readers address the +// namespace, never the worker +func TestVarsNamespaces(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, + frankenphp.WithServer(server), + frankenphp.WithWorkers("config", "testdata/bgworker/multi-vars.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(server), + frankenphp.WithWorkerVars("redis", "feature_flags"), + ), + frankenphp.WithNumThreads(2), + ) + + all := `{"vars":["redis","feature_flags"]}` + assert.JSONEq(t, `{"namespace":"redis","all":`+all+`}`, serverGet(t, server, "http://example.com/vars.php?ns=redis")) + assert.JSONEq(t, `{"namespace":"feature_flags","all":`+all+`}`, serverGet(t, server, "http://example.com/vars.php?ns=feature_flags")) + assert.Contains(t, serverGet(t, server, "http://example.com/vars.php?ns=config"), "no background worker publishes") +} + +// a namespace resolves within the server of the reader first, then among the +// global workers +func TestVarsScopedToServer(t *testing.T) { + server1, _ := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("one")) + server2, _ := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("two")) + server3, _ := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("three")) + // worker names are global, the namespace is what the three share + publisher := func(name, value string, server *frankenphp.Server) frankenphp.Option { + opts := []frankenphp.WorkerOption{ + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerVars("cfg"), + frankenphp.WithWorkerEnv(map[string]string{"BG_PUBLISH_VALUE": value}), + } + if server != nil { + opts = append(opts, frankenphp.WithWorkerServerScope(server)) + } + + return frankenphp.WithWorkers(name, "testdata/bgworker/publisher.php", 1, opts...) + } + initServers(t, + frankenphp.WithServer(server1), + frankenphp.WithServer(server2), + frankenphp.WithServer(server3), + publisher("cfg-one", "one", server1), + publisher("cfg-two", "two", server2), + publisher("cfg-global", "global", nil), + frankenphp.WithNumThreads(3), + ) + + assert.Contains(t, serverGet(t, server1, "http://example.com/vars.php?ns=cfg"), `"value":"one"`) + assert.Contains(t, serverGet(t, server2, "http://example.com/vars.php?ns=cfg"), `"value":"two"`) + assert.Contains(t, serverGet(t, server3, "http://example.com/vars.php?ns=cfg"), `"value":"global"`) +} + +// a worker booting before its dependency has published blocks in +// frankenphp_get_vars() until the dependency is ready +func TestVarsBlockUntilReady(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "consumer.json") + initServers(t, + bgWorker("consumer", "consumer.php", map[string]string{"BG_CONSUME": "publisher", "BG_SENTINEL": sentinel}, nil), + bgWorker("publisher", "publisher.php", map[string]string{"BG_PUBLISH_DELAY_MS": "500"}, nil), + frankenphp.WithNumThreads(3), + ) + + assert.Contains(t, requireFileContentEventually(t, sentinel), `"answer":42`) +} + +func TestVarsCycleIsRefused(t *testing.T) { + tmp := t.TempDir() + s1, s2 := filepath.Join(tmp, "c1.txt"), filepath.Join(tmp, "c2.txt") + initServers(t, + bgWorker("c1", "consumer.php", map[string]string{"BG_CONSUME": "c2", "BG_SENTINEL": s1}, nil), + bgWorker("c2", "consumer.php", map[string]string{"BG_CONSUME": "c1", "BG_SENTINEL": s2}, nil), + frankenphp.WithNumThreads(3), + ) + + // one side sees the cycle and publishes the exception, the other reads it + results := requireFileContentEventually(t, s1) + "\n" + requireFileContentEventually(t, s2) + assert.Contains(t, results, "circular dependency") + assert.Contains(t, results, `{"consumed":"RuntimeException: frankenphp_get_vars(): circular dependency`) +} + +// the first tick of a run waits for every declared namespace, so readers +// never ask a ready worker for a snapshot it does not have +func TestVarsArePublishedBeforeTheFirstTick(t *testing.T) { + t.Cleanup(frankenphp.Shutdown) + + err := frankenphp.Init(bgWorker("silent", "basic.php", nil, nil, frankenphp.WithWorkerMaxFailures(1)), frankenphp.WithNumThreads(1)) + require.Error(t, err) +} + +func TestVarsRejectInvalidPublications(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "bad.txt") + initServers(t, bgWorker("bad", "bad-vars.php", map[string]string{"BG_SENTINEL": sentinel}, nil), frankenphp.WithNumThreads(2)) + + result := requireFileContentEventually(t, sentinel) + assert.Contains(t, result, "ValueError") + assert.Contains(t, result, "must be null, scalars, arrays or enums") + assert.Contains(t, result, `LogicException: FrankenPHP\WorkerHandle::setVars(): this worker does not publish the vars namespace "undeclared"`) +} + +// a reboot restarts the PHP engine, and readers keep getting the snapshot +// published before it until the worker publishes again: it must not point +// into the memory of the engine that is gone +func TestVarsSurviveAReboot(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + env := map[string]string{"BG_SENTINEL": filepath.Join(t.TempDir(), "published"), "BG_REPUBLISH_DELAY_MS": "2000"} + initServers(t, frankenphp.WithServer(server), bgWorker("once", "republish.php", env, server), frankenphp.WithNumThreads(2)) + + before := serverGet(t, server, "http://example.com/vars.php?ns=once") + require.JSONEq(t, `{"answer":42,"nested":{"a":1,"list":[true,null,1.5,"x"]}}`, before) + + rebooted := make(chan struct{}) + go func() { + frankenphp.RestartWorkers() + close(rebooted) + }() + + // the request threads come back while the worker still waits to publish + time.Sleep(500 * time.Millisecond) + assert.JSONEq(t, before, serverGet(t, server, "http://example.com/vars.php?ns=once")) + + <-rebooted + assert.JSONEq(t, before, serverGet(t, server, "http://example.com/vars.php?ns=once")) +} + +func TestValidateVarsNamespaces(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + publisher := func(name string, num int, extra ...frankenphp.WorkerOption) frankenphp.Option { + return frankenphp.WithWorkers(name, "testdata/bgworker/publisher.php", num, append([]frankenphp.WorkerOption{frankenphp.WithWorkerBackground()}, extra...)...) + } + + assert.ErrorContains(t, frankenphp.Validate( + publisher("one", 1, frankenphp.WithWorkerVars("cfg")), + publisher("two", 1, frankenphp.WithWorkerVars("other", "cfg")), + ), `two global workers cannot publish the same vars namespace: "cfg"`) + + // one publisher per server + other, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + assert.NoError(t, frankenphp.Validate( + frankenphp.WithServer(server), + frankenphp.WithServer(other), + publisher("one", 1, frankenphp.WithWorkerVars("cfg"), frankenphp.WithWorkerServerScope(server)), + publisher("two", 1, frankenphp.WithWorkerVars("cfg"), frankenphp.WithWorkerServerScope(other)), + )) + + // the same namespace in a server and globally: the server's comes first for its requests + assert.NoError(t, frankenphp.Validate( + frankenphp.WithServer(server), + publisher("one", 1, frankenphp.WithWorkerVars("cfg")), + publisher("two", 1, frankenphp.WithWorkerVars("cfg"), frankenphp.WithWorkerServerScope(server)), + )) + + assert.ErrorContains(t, frankenphp.Validate( + publisher("one", 1, frankenphp.WithWorkerVars("cfg", "cfg")), + ), `declares the vars namespace "cfg" twice`) + + assert.ErrorContains(t, frankenphp.Validate( + publisher("pool", 2, frankenphp.WithWorkerVars("cfg")), + ), "publishes vars, it runs a single thread") + + assert.ErrorContains(t, frankenphp.Validate( + frankenphp.WithWorkers("http", "testdata/worker.php", 1, frankenphp.WithWorkerVars("cfg")), + ), "only background workers publish them") +} + +// the rules are the ones Init() applies +func TestInitRefusesASharedVarsNamespace(t *testing.T) { + t.Cleanup(frankenphp.Shutdown) + + err := frankenphp.Init( + bgWorker("cfg", "publisher.php", nil, nil), + frankenphp.WithWorkers("other", "testdata/bgworker/publisher.php", 1, frankenphp.WithWorkerBackground(), frankenphp.WithWorkerVars("cfg")), + frankenphp.WithNumThreads(1), + ) + require.ErrorContains(t, err, `two global workers cannot publish the same vars namespace: "cfg"`) +} + +// a bailout while copying a snapshot, here in an autoloader its enum +// triggers, lets go of it: the next readers are served, by a publisher that +// keeps publishing. memory_limit would do too, but not where the Zend +// allocator is off, as under the sanitizers. +func TestVarsBailoutInTheCopy(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("enum", "enum-vars.php", nil, server), frankenphp.WithNumThreads(2)) + + assert.Contains(t, serverGet(t, server, "http://example.com/vars-fatal.php?fatal=1"), "Cannot redeclare") + + first := serverGet(t, server, "http://example.com/vars-fatal.php") + require.Eventually(t, func() bool { + return serverGet(t, server, "http://example.com/vars-fatal.php") != first + }, 5*time.Second, 25*time.Millisecond, "the publisher is stuck") +} + +// an autoloader may keep the class name it is given: a request string, not +// the one of the snapshot, which the next publication frees +func TestVarsEnumClassNameOutlivesTheSnapshot(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("enum", "enum-vars.php", nil, server), frankenphp.WithNumThreads(2)) + + assert.Equal(t, "Hearts kept:EnumSuit", serverGet(t, server, "http://example.com/vars-enum.php")) +} + +// an HTTP worker reading vars in its bootstrap gives up once the start +// failed, here because the publisher never publishes, instead of keeping +// Init() waiting +func TestVarsWaitEndsWhenTheStartFails(t *testing.T) { + t.Cleanup(frankenphp.Shutdown) + done := make(chan error, 1) + go func() { + done <- frankenphp.Init( + bgWorker("silent", "basic.php", nil, nil, frankenphp.WithWorkerMaxFailures(1)), + frankenphp.WithWorkers("reader", "testdata/vars-at-boot.php", 1), + frankenphp.WithNumThreads(3), + ) + }() + + select { + case err := <-done: + require.ErrorContains(t, err, "keeps crashing") + case <-time.After(10 * time.Second): + t.Fatal("Init() did not return") + } +} + +// the namespaces of a run go with it, even when the next run has no workers +func TestVarsOwnersEndWithTheRun(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("publisher", "publisher.php", nil, server), frankenphp.WithNumThreads(2)) + require.Contains(t, serverGet(t, server, "http://example.com/vars.php?ns=publisher"), `"answer":42`) + frankenphp.Shutdown() + + initServers(t, frankenphp.WithServer(server), frankenphp.WithNumThreads(1)) + assert.Contains(t, serverGet(t, server, "http://example.com/vars.php?ns=publisher"), `no background worker publishes the vars namespace "publisher"`) +} + +// a worker of a server falls back to the global namespace like a request does +func TestVarsServerWorkerReadsAGlobalNamespace(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "consumer.json") + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, + frankenphp.WithServer(server), + bgWorker("publisher", "publisher.php", map[string]string{"BG_PUBLISH_VALUE": "global"}, nil), + bgWorker("consumer", "consumer.php", map[string]string{"BG_CONSUME": "publisher", "BG_SENTINEL": sentinel}, server), + frankenphp.WithNumThreads(3), + ) + + assert.Contains(t, requireFileContentEventually(t, sentinel), `"value":"global"`) +} diff --git a/zval.h b/zval.h index f75cccdc60..5c62cab573 100644 --- a/zval.h +++ b/zval.h @@ -11,9 +11,9 @@ * depth and a native stack overflow (crashes the whole process, not just * one request). * - * Fast paths: - * - Interned strings: shared memory, no copy. - * - Opcache-immutable arrays: shared pointer, no copy, no free. + * A persistent tree owns all its memory, interned strings and + * opcache-immutable arrays included: it can outlive an engine reboot, which + * frees the former, and an opcache restart, which rewinds the latter. * * Included by frankenphp.c; not a standalone compilation unit. */ @@ -58,12 +58,6 @@ static bool persistent_zval_validate_depth(zval *z, int depth) { case IS_OBJECT: return (Z_OBJCE_P(z)->ce_flags & ZEND_ACC_ENUM) != 0; case IS_ARRAY: { - /* Opcache-immutable arrays are compile-time constants in shared - * memory; their leaves are guaranteed scalars or further immutable - * arrays. The copy/free paths below already trust this flag, so a - * recursive walk here would just be cycles. */ - if ((GC_FLAGS(Z_ARRVAL_P(z)) & IS_ARRAY_IMMUTABLE) != 0) - return true; zval *val; ZEND_HASH_FOREACH_VAL(Z_ARRVAL_P(z), val) { if (!persistent_zval_validate_depth(val, depth + 1)) @@ -103,38 +97,22 @@ static void persistent_zval_persist(zval *dst, zval *src) { break; case IS_STRING: { zend_string *s = Z_STR_P(src); - if (ZSTR_IS_INTERNED(s)) { - ZVAL_STR(dst, s); /* interned strings live process-wide */ - } else { - ZVAL_NEW_STR(dst, zend_string_init(ZSTR_VAL(s), ZSTR_LEN(s), 1)); - } + ZVAL_NEW_STR(dst, zend_string_init(ZSTR_VAL(s), ZSTR_LEN(s), 1)); break; } case IS_OBJECT: { /* Must be an enum (validated earlier in persistent_zval_validate). */ zend_class_entry *ce = Z_OBJCE_P(src); persistent_zval_enum_t *e = pemalloc(sizeof(*e), 1); - e->class_name = - ZSTR_IS_INTERNED(ce->name) - ? ce->name - : zend_string_init(ZSTR_VAL(ce->name), ZSTR_LEN(ce->name), 1); + e->class_name = zend_string_init(ZSTR_VAL(ce->name), ZSTR_LEN(ce->name), 1); zval *case_name_zval = zend_enum_fetch_case_name(Z_OBJ_P(src)); zend_string *case_str = Z_STR_P(case_name_zval); - e->case_name = - ZSTR_IS_INTERNED(case_str) - ? case_str - : zend_string_init(ZSTR_VAL(case_str), ZSTR_LEN(case_str), 1); + e->case_name = zend_string_init(ZSTR_VAL(case_str), ZSTR_LEN(case_str), 1); ZVAL_PTR(dst, e); break; } case IS_ARRAY: { HashTable *src_ht = Z_ARRVAL_P(src); - if ((GC_FLAGS(src_ht) & IS_ARRAY_IMMUTABLE) != 0) { - /* Opcache-immutable arrays live for the process lifetime and are - * safe to share across threads by pointer. Zero-copy, zero-free. */ - ZVAL_ARR(dst, src_ht); - break; - } HashTable *dst_ht = pemalloc(sizeof(HashTable), 1); zend_hash_init(dst_ht, zend_hash_num_elements(src_ht), NULL, NULL, 1); ZVAL_ARR(dst, dst_ht); @@ -146,16 +124,12 @@ static void persistent_zval_persist(zval *dst, zval *src) { zval pval; persistent_zval_persist(&pval, val); if (key) { - if (ZSTR_IS_INTERNED(key)) { - zend_hash_add_new(dst_ht, key, &pval); - } else { - zend_string *pkey = zend_string_init(ZSTR_VAL(key), ZSTR_LEN(key), 1); - /* Iteration guarantees the source key has its hash set. - * Propagating it lets zend_hash_add_new skip the re-hash. */ - ZSTR_H(pkey) = ZSTR_H(key); - zend_hash_add_new(dst_ht, pkey, &pval); - zend_string_release(pkey); - } + zend_string *pkey = zend_string_init(ZSTR_VAL(key), ZSTR_LEN(key), 1); + /* Iteration guarantees the source key has its hash set. + * Propagating it lets zend_hash_add_new skip the re-hash. */ + ZSTR_H(pkey) = ZSTR_H(key); + zend_hash_add_new(dst_ht, pkey, &pval); + zend_string_release(pkey); } else { zend_hash_index_add_new(dst_ht, idx, &pval); } @@ -169,30 +143,21 @@ static void persistent_zval_persist(zval *dst, zval *src) { } } -/* Deep-free a persistent zval tree. Idempotent on scalars. Skips - * interned strings and immutable arrays (they are borrowed, not owned). */ +/* Deep-free a persistent zval tree. Idempotent on scalars. */ static void persistent_zval_free(zval *z) { switch (Z_TYPE_P(z)) { case IS_STRING: - if (!ZSTR_IS_INTERNED(Z_STR_P(z))) { - zend_string_free(Z_STR_P(z)); - } + zend_string_free(Z_STR_P(z)); break; case IS_PTR: { persistent_zval_enum_t *e = (persistent_zval_enum_t *)Z_PTR_P(z); - if (!ZSTR_IS_INTERNED(e->class_name)) - zend_string_free(e->class_name); - if (!ZSTR_IS_INTERNED(e->case_name)) - zend_string_free(e->case_name); + zend_string_free(e->class_name); + zend_string_free(e->case_name); pefree(e, 1); break; } case IS_ARRAY: { HashTable *ht = Z_ARRVAL_P(z); - if ((GC_FLAGS(ht) & IS_ARRAY_IMMUTABLE) != 0) { - /* Borrowed from opcache, do not touch. */ - break; - } zval *val; ZEND_HASH_FOREACH_VAL(ht, val) { persistent_zval_free(val); } ZEND_HASH_FOREACH_END(); @@ -223,15 +188,16 @@ static void persistent_zval_to_request(zval *dst, zval *src) { ZVAL_DOUBLE(dst, Z_DVAL_P(src)); break; case IS_STRING: - if (ZSTR_IS_INTERNED(Z_STR_P(src))) { - ZVAL_STR(dst, Z_STR_P(src)); - } else { - ZVAL_STRINGL(dst, Z_STRVAL_P(src), Z_STRLEN_P(src)); - } + ZVAL_STRINGL(dst, Z_STRVAL_P(src), Z_STRLEN_P(src)); break; case IS_PTR: { persistent_zval_enum_t *e = (persistent_zval_enum_t *)Z_PTR_P(src); - zend_class_entry *ce = zend_lookup_class(e->class_name); + /* autoloaders get the name and may keep it: a request copy, since the + * persistent one is shared with other threads and freed with its tree */ + zend_string *class_name = + zend_string_init(ZSTR_VAL(e->class_name), ZSTR_LEN(e->class_name), 0); + zend_class_entry *ce = zend_lookup_class(class_name); + zend_string_release(class_name); if (EG(exception)) { /* Autoloader threw; let that exception propagate untouched. */ ZVAL_NULL(dst); @@ -257,11 +223,6 @@ static void persistent_zval_to_request(zval *dst, zval *src) { } case IS_ARRAY: { HashTable *src_ht = Z_ARRVAL_P(src); - if ((GC_FLAGS(src_ht) & IS_ARRAY_IMMUTABLE) != 0) { - /* Zero-copy: immutable arrays are safe to expose directly. */ - ZVAL_ARR(dst, src_ht); - break; - } array_init_size(dst, zend_hash_num_elements(src_ht)); HashTable *dst_ht = Z_ARRVAL_P(dst); @@ -276,14 +237,10 @@ static void persistent_zval_to_request(zval *dst, zval *src) { break; } if (key) { - if (ZSTR_IS_INTERNED(key)) { - zend_hash_add_new(dst_ht, key, &rval); - } else { - zend_string *rkey = zend_string_init(ZSTR_VAL(key), ZSTR_LEN(key), 0); - ZSTR_H(rkey) = ZSTR_H(key); - zend_hash_add_new(dst_ht, rkey, &rval); - zend_string_release(rkey); - } + zend_string *rkey = zend_string_init(ZSTR_VAL(key), ZSTR_LEN(key), 0); + ZSTR_H(rkey) = ZSTR_H(key); + zend_hash_add_new(dst_ht, rkey, &rval); + zend_string_release(rkey); } else { zend_hash_index_add_new(dst_ht, idx, &rval); } diff --git a/zval_test.go b/zval_test.go index 80aad7f16f..e25903b0b7 100644 --- a/zval_test.go +++ b/zval_test.go @@ -47,4 +47,5 @@ func TestPersistentZvalRoundtrip(t *testing.T) { require.Contains(t, out, "OK stdClass rejected") require.Contains(t, out, "OK resource rejected") require.Contains(t, out, "OK nested stdClass rejected") + require.Contains(t, out, "OK immutable literal exposed repeatedly") }