Send application logs to stderr across Server deployment paths

.env.example

@@ -4,7 +4,7 @@ APP_DEBUG=true
 APP_URL=http://localhost:8080
 # APP_VERSION defaults to the checked-in source release identity.
 
-LOG_CHANNEL=stack
+LOG_CHANNEL=stderr
 LOG_LEVEL=debug
 
 DB_CONNECTION=mysql

.github/workflows/published-release-recovery.yml

@@ -66,6 +66,30 @@ jobs:
           TLS_TEST_ROOT: ${{ runner.temp }}/dw-server-tls-release-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
         run: bash scripts/ci/qualify-database-tls.sh
 
+      - name: Verify existing image first-run readiness and container logging
+        env:
+          DW_SERVER_IMAGE: ghcr.io/durable-workflow/server@${{ steps.release_verify.outputs.image_digest }}
+          DW_FIRST_RUN_SCOPE: recovery-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
+          RELEASE_SOURCE_COMMIT: ${{ inputs.release_commit }}
+        run: |
+          if git rev-parse --verify "${RELEASE_SOURCE_COMMIT}:scripts/ci/verify-container-logging.mjs" >/dev/null 2>&1; then
+            scripts/smoke-published-first-run.sh
+          else
+            echo "This immutable release predates container logging acceptance; historical verification remains unchanged."
+          fi
+
+      - name: Clean up existing image first-run resources
+        if: always()
+        env:
+          DW_FIRST_RUN_SCOPE: recovery-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
+          LOGGING_TEST_PROJECT: recovery-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-logs
+        run: |
+          docker ps -aq --filter "label=com.docker.compose.project=$LOGGING_TEST_PROJECT" | xargs -r docker rm -f >/dev/null 2>&1 || true
+          docker network ls -q --filter "label=com.docker.compose.project=$LOGGING_TEST_PROJECT" | xargs -r docker network rm >/dev/null 2>&1 || true
+          docker volume ls -q --filter "label=com.docker.compose.project=$LOGGING_TEST_PROJECT" | xargs -r docker volume rm -f >/dev/null 2>&1 || true
+          docker rm -f "${DW_FIRST_RUN_SCOPE}-server" "${DW_FIRST_RUN_SCOPE}-bootstrap" "${DW_FIRST_RUN_SCOPE}-unbootstrapped" >/dev/null 2>&1 || true
+          docker volume rm "${DW_FIRST_RUN_SCOPE}-database" >/dev/null 2>&1 || true
+
       - name: Clean up published image database TLS fixtures
         if: always()
         env:
@@ -93,6 +117,7 @@ jobs:
     needs: verify
     if: >-
       always() &&
+      needs.verify.result == 'success' &&
       needs.verify.outputs.release_verification_outcome == 'success' &&
       needs.verify.outputs.published_database_tls_outcome == 'success' &&
       github.repository == 'durable-workflow/server' &&

.github/workflows/release.yml

@@ -296,9 +296,14 @@ jobs:
         if: always()
         env:
           DW_FIRST_RUN_SCOPE: release-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
+          LOGGING_TEST_PROJECT: release-first-run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}-logs
         run: |
+          docker ps -aq --filter "label=com.docker.compose.project=$LOGGING_TEST_PROJECT" | xargs -r docker rm -f >/dev/null 2>&1 || true
+          docker network ls -q --filter "label=com.docker.compose.project=$LOGGING_TEST_PROJECT" | xargs -r docker network rm >/dev/null 2>&1 || true
+          docker volume ls -q --filter "label=com.docker.compose.project=$LOGGING_TEST_PROJECT" | xargs -r docker volume rm -f >/dev/null 2>&1 || true
           docker rm -f \
             "${DW_FIRST_RUN_SCOPE}-server" \
+            "${DW_FIRST_RUN_SCOPE}-unbootstrapped" \
             "${DW_FIRST_RUN_SCOPE}-bootstrap" >/dev/null 2>&1 || true
           docker volume rm -f \
             "${DW_FIRST_RUN_SCOPE}-database" >/dev/null 2>&1 || true

Dockerfile

@@ -164,7 +164,8 @@ ENV DW_WORKER_LONG_POLL_MAX_CONCURRENT=2 \
     DB_CONNECTION=sqlite \
     DB_DATABASE=/app/database/database.sqlite \
     QUEUE_CONNECTION=database \
-    CACHE_STORE=file
+    CACHE_STORE=file \
+    LOG_CHANNEL=stderr
 
 ENTRYPOINT ["server-entrypoint"]
 

config/dw-contract.php

@@ -1101,7 +1101,7 @@
         'APP_NAME', 'APP_ENV', 'APP_KEY', 'APP_DEBUG', 'APP_URL', 'APP_VERSION',
         'APP_LOCALE', 'APP_FALLBACK_LOCALE', 'APP_FAKER_LOCALE', 'APP_MAINTENANCE_DRIVER',
         'APP_MAINTENANCE_STORE', 'APP_PREVIOUS_KEYS', 'APP_CIPHER', 'APP_TIMEZONE',
-        'LOG_CHANNEL', 'LOG_LEVEL', 'LOG_DEPRECATIONS_CHANNEL', 'LOG_STACK',
+        'LOG_CHANNEL', 'LOG_LEVEL', 'LOG_STDERR_FORMATTER', 'LOG_DEPRECATIONS_CHANNEL', 'LOG_STACK',
         'LOG_DAILY_DAYS', 'LOG_SLACK_WEBHOOK_URL', 'LOG_PAPERTRAIL_URL',
         'LOG_PAPERTRAIL_PORT',
         'DB_CONNECTION', 'DB_HOST', 'DB_PORT', 'DB_DATABASE', 'DB_USERNAME',

k8s/README.md

@@ -16,6 +16,27 @@ release and pin its Docker Hub tag:
 durableworkflow/server:2.10.1
 ```
 
+## Application logs
+
+The raw ConfigMap sets `LOG_CHANNEL=stderr` for the API, worker and scheduler.
+Read application logs with `kubectl logs` or your cluster's log collector. This
+also works with existing published images when the setting is supplied explicitly.
+
+Set `LOG_LEVEL` to select the minimum severity. For newline-delimited JSON, add
+this ConfigMap entry:
+
+```yaml
+LOG_STDERR_FORMATTER: 'Monolog\Formatter\JsonFormatter'
+```
+
+An explicit `LOG_CHANNEL` override selects another Laravel channel. File channels
+need their own persistent storage and collection; Kubernetes does not collect
+`storage/logs/laravel.log` automatically. Restart persistent roles after changing
+logging settings because configuration is cached at startup. `DW_LOG_FORMAT` is
+not a supported setting.
+
+## Image selection
+
 Before production use, patch every workload image to the exact published tag or
 digest you intend to run:
 

k8s/helm/durable-workflow/Chart.yaml

@@ -5,7 +5,7 @@ type: application
 # The chart's own semver version. Bumped on every chart release; treated as
 # independent of the server image version (appVersion). Breaking-change rules
 # for this version live in docs/UPGRADING.md alongside the chart.
-version: 0.2.3
+version: 0.2.4
 # The immutable Durable Workflow Server identity this chart release packages.
 # The onboarding default in values.yaml and appVersion are generated from the
 # checked-in source release record.

k8s/helm/durable-workflow/README.md

@@ -132,6 +132,26 @@ Driver, or any GitOps secret bridge that produces a Kubernetes `Secret`.
 | `ServiceAccount` (`*`) | Namespace-scoped SA for every workload. | `serviceAccount.create: false` |
 | `NetworkPolicy` (`*-server`) | Optional ingress allow-list to the API. | `networkPolicy.enabled: false` (default) |
 
+## Application logs
+
+The chart defaults `config.extraEnv.LOG_CHANNEL` to `stderr` for the API, worker,
+scheduler and bootstrap job. Use `kubectl logs` or the cluster's log collector.
+To choose the minimum severity and emit newline-delimited JSON:
+
+```yaml
+config:
+  extraEnv:
+    LOG_CHANNEL: "stderr"
+    LOG_LEVEL: "info"
+    LOG_STDERR_FORMATTER: 'Monolog\Formatter\JsonFormatter'
+```
+
+These explicit settings also work with existing published Server images. Override
+`LOG_CHANNEL` to select a different Laravel channel. File channels require their
+own storage and collection. Restart persistent roles after changing logging
+settings because configuration is cached at startup. `DW_LOG_FORMAT` is not a
+supported setting.
+
 ## Database TLS
 
 For an authenticated encrypted database connection, pre-create a CA bundle

k8s/helm/durable-workflow/values.yaml

@@ -77,10 +77,11 @@ config:
   # Free-form ConfigMap entries appended verbatim. Use this for environment
   # tuning that the chart does not expose as first-class settings (e.g.
   # DW_V2_MATCHING_ROLE_QUEUE_WAKE for dedicated matching rollouts).
-  extraEnv: {}
+  extraEnv:
+    LOG_CHANNEL: "stderr"
   # extraEnv:
   #   DW_V2_MATCHING_ROLE_QUEUE_WAKE: "0"
-  #   DW_LOG_FORMAT: "json"
+  #   LOG_STDERR_FORMATTER: 'Monolog\Formatter\JsonFormatter'
 
 # ---------------------------------------------------------------------------
 # External persistence (REQUIRED — no in-cluster database is bundled)

k8s/secret.yaml

@@ -15,6 +15,7 @@ data:
   APP_VERSION: "2.10.1"
   APP_ENV: production
   APP_DEBUG: "false"
+  LOG_CHANNEL: "stderr"
   DB_CONNECTION: mysql
   DB_HOST: mysql.durable-workflow.svc.cluster.local
   DB_PORT: "3306"

resources/release/source-release.json

@@ -4,6 +4,6 @@
     "version": "2.10.1"
   },
   "helm_chart": {
-    "version": "0.2.3"
+    "version": "0.2.4"
   }
 }

scripts/ci/test_helm_chart_release.py

@@ -91,6 +91,7 @@ def evaluate_recovery_condition(
 ) -> bool:
     expression = job_condition(workflow_source(workflow), job)
     replacements = {
+        "needs.verify.result": context.get("verify_result", "success"),
         "needs.publish.outputs.exact_publish_outcome": context.get(
             "exact_publish_outcome", ""
         ),
@@ -888,6 +889,22 @@ def test_chart_publication_requires_the_installed_database_tls_gate(self) -> Non
                         ref="refs/heads/main",
                     ))
 
+    def test_recovery_requires_the_complete_image_verification_job(self) -> None:
+        for result in ["success", "failure", "cancelled", "skipped", ""]:
+            with self.subTest(result=result):
+                self.assertEqual(
+                    result == "success",
+                    evaluate_recovery_condition(
+                        "published-release-recovery.yml",
+                        "publish-chart",
+                        verify_result=result,
+                        release_verification_outcome="success",
+                        database_tls_outcome="success",
+                        repository="durable-workflow/server",
+                        ref="refs/heads/main",
+                    ),
+                )
+
     def test_duplicate_publication_reuses_and_verifies_the_existing_package(
         self,
     ) -> None:

scripts/ci/verify-container-logging.mjs

@@ -0,0 +1,172 @@
+import assert from 'node:assert/strict';
+import {randomBytes} from 'node:crypto';
+import {execFileSync, spawnSync} from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+const image = process.env.DW_SERVER_IMAGE;
+assert.ok(image, 'DW_SERVER_IMAGE must identify the image under qualification');
+const scope = process.env.DW_LOGGING_SMOKE_SCOPE || `dw-log-${randomBytes(8).toString('hex')}`;
+assert.match(scope, /^[A-Za-z0-9][A-Za-z0-9_.-]{0,95}$/);
+const label = 'dev.durable-workflow.container-logging';
+const fixture = fileURLToPath(new URL('../../tests/Fixtures/ContainerLogging/probe.php', import.meta.url));
+const root = fs.mkdtempSync(path.join(process.env.RUNNER_TEMP || os.tmpdir(), `${scope}-`));
+const database = `${scope}-database`;
+const ini = path.join(root, 'logging-probe.ini');
+fs.writeFileSync(ini, 'auto_prepend_file=/opt/container-logging-probe.php\n');
+const names = new Set();
+const failed = [];
+let volumeCreated = false;
+const pause = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds));
+
+function docker(args, timeout = 30000) {
+  return execFileSync('/usr/bin/docker', args, {encoding: 'utf8', timeout, maxBuffer: 524288}).trim();
+}
+
+function owned(name) {
+  return docker(['inspect', '--format', `{{index .Config.Labels "${label}"}}`, name]) === scope;
+}
+
+function remove(name) {
+  if (!names.has(name)) return;
+  assert.ok(owned(name), `refuse cleanup of foreign container ${name}`);
+  docker(['stop', '--time', '3', name]);
+  docker(['rm', name]);
+  names.delete(name);
+}
+
+const common = [
+  '--user', '1000:1000', '--cpus', '1', '--memory', '512m', '--memory-swap', '512m',
+  '--label', `${label}=${scope}`,
+  '--label', `com.docker.compose.project=${scope}`,
+  '--mount', `type=volume,source=${database},target=/app/database`,
+  '--env', 'DB_CONNECTION=sqlite', '--env', 'DB_DATABASE=/app/database/database.sqlite',
+  '--env', 'DB_SQLITE_JOURNAL_MODE=DELETE', '--env', 'CACHE_STORE=file',
+  '--env', 'QUEUE_CONNECTION=database', '--env', 'DW_AUTH_DRIVER=token',
+  '--env', 'DW_AUTH_TOKEN=container-logging-synthetic-token',
+];
+
+function copy(name, source, target) {
+  const result = spawnSync('/usr/bin/docker', ['cp', `${name}:${source}`, target], {
+    encoding: 'utf8', timeout: 10000, maxBuffer: 524288,
+  });
+  return result.status === 0 ? fs.readFileSync(target, 'utf8') : '';
+}
+
+async function waitForApi(name) {
+  const port = docker(['port', name, '8080/tcp']).split(':').at(-1);
+  // Poll serially so startup cannot accumulate concurrent health requests.
+  for (let attempt = 0; attempt < 100; attempt += 1) {
+    try {
+      const response = await fetch(`http://127.0.0.1:${port}/api/health`, {signal: AbortSignal.timeout(1000)});
+      await response.text();
+      if (response.status === 200) return;
+    } catch { /* The real API may still be starting. */ }
+    await pause(100);
+  }
+  assert.fail('API did not become live within its original probe budget');
+}
+
+function assertOutput(name, nonce, role, mode) {
+  const proof = copy(name, '/app/storage/framework/testing/dw-container-logging-probe', path.join(root, `${mode}-${role}.proof`));
+  assert.equal(proof, nonce, `${role}: actual role must invoke the default Laravel logger`);
+  const logs = spawnSync('/usr/bin/docker', ['logs', name], {encoding: 'utf8', timeout: 10000, maxBuffer: 524288});
+  assert.equal(logs.status, 0);
+  const file = copy(name, '/app/storage/logs/laravel.log', path.join(root, `${mode}-${role}.log`));
+  const stderr = logs.stderr.split('\n').filter(line => line.includes(nonce));
+  assert.ok(!logs.stdout.includes(nonce), `${role}: application error must not go to stdout`);
+  if (mode === 'file') {
+    assert.ok(file.includes(nonce), 'operator-selected file channel must be honored');
+    assert.equal(stderr.length, 0);
+  } else if (mode === 'filtered') {
+    assert.equal(stderr.length, 0, 'LOG_LEVEL must filter the emitted error');
+    assert.ok(!file.includes(nonce));
+  } else {
+    assert.ok(stderr.length > 0, `${role}/${mode}: application error missing from container stderr`);
+    assert.ok(!file.includes(nonce), `${role}: default stderr must not duplicate the error into a local file`);
+    if (mode === 'json') {
+      for (const line of stderr) {
+        const record = JSON.parse(line);
+        assert.equal(record.message, 'container-logging-probe');
+        assert.equal(record.context.role, role);
+        assert.equal(record.context.nonce, nonce);
+      }
+    }
+  }
+  console.log(JSON.stringify({role, mode, outcome: 'PASS'}));
+}
+
+async function check(role, mode) {
+  const name = `${scope}-${mode}-${role}`;
+  const nonce = randomBytes(16).toString('hex');
+  const args = ['create', '--name', name, ...common,
+    '--mount', `type=bind,source=${fixture},target=/opt/container-logging-probe.php,readonly`,
+    '--mount', `type=bind,source=${ini},target=/usr/local/etc/php/conf.d/zz-container-logging-probe.ini,readonly`,
+    '--env', `CONTAINER_LOGGING_PROBE_ROLE=${role}`,
+    '--env', `CONTAINER_LOGGING_PROBE_NONCE=${nonce}`];
+  if (mode === 'json') args.push('--env', String.raw`LOG_STDERR_FORMATTER=Monolog\Formatter\JsonFormatter`);
+  if (mode === 'file') args.push('--env', 'LOG_CHANNEL=single');
+  if (mode === 'filtered') args.push('--env', 'LOG_LEVEL=critical');
+  if (role === 'api') args.push('--publish', '127.0.0.1::8080');
+  args.push(image);
+  if (role === 'worker') args.push('php', 'artisan', 'queue:work', '--once', '--stop-when-empty', '--max-time=2');
+  if (role === 'scheduler') args.push('php', 'artisan', 'schedule:evaluate', '--limit=100', '--json');
+
+  assert.notEqual(spawnSync('/usr/bin/docker', ['inspect', name], {stdio: 'ignore'}).status, 0,
+    `container name already exists: ${name}`);
+  docker(args);
+  names.add(name);
+  try {
+    docker(['start', name]);
+    if (role === 'api') await waitForApi(name);
+    else assert.equal(docker(['wait', name]), '0', `${role} command must exit successfully`);
+    assertOutput(name, nonce, role, mode);
+  } finally {
+    remove(name);
+  }
+}
+
+async function main() {
+  try {
+    assert.notEqual(spawnSync('/usr/bin/docker', ['volume', 'inspect', database], {stdio: 'ignore'}).status, 0,
+      `volume already exists: ${database}`);
+    docker(['volume', 'create', '--label', `${label}=${scope}`, '--label', `com.docker.compose.project=${scope}`, database]);
+    assert.equal(docker(['volume', 'inspect', '--format', `{{index .Labels "${label}"}}`, database]), scope,
+      'refuse use of a foreign volume');
+    volumeCreated = true;
+    const bootstrap = `${scope}-bootstrap`;
+    docker(['create', '--name', bootstrap, ...common, image, 'server-bootstrap']);
+    names.add(bootstrap);
+    docker(['start', bootstrap]);
+    assert.equal(docker(['wait', bootstrap], 60000), '0', 'isolated database bootstrap must succeed');
+    remove(bootstrap);
+    // Roles share one SQLite fixture and must not overlap or multiply resource use.
+    for (const mode of ['default', 'json']) {
+      for (const role of ['api', 'worker', 'scheduler']) {
+        try {await check(role, mode);} catch (error) {failed.push(`${role}/${mode}: ${error.message}`);}
+      }
+    }
+    for (const mode of ['file', 'filtered']) {
+      try {await check('worker', mode);} catch (error) {failed.push(`worker/${mode}: ${error.message}`);}
+    }
+    assert.deepEqual(failed, [], 'container logging acceptance failed');
+    console.log('Container logging acceptance: 8 PASS');
+  } finally {
+    for (const name of names) remove(name);
+    if (volumeCreated) {
+      assert.equal(docker(['volume', 'inspect', '--format', `{{index .Labels "${label}"}}`, database]), scope,
+        'refuse cleanup of a foreign volume');
+      docker(['volume', 'rm', database]);
+    }
+    fs.rmSync(root, {recursive: true});
+  }
+}
+
+try {
+  await main();
+} catch (error) {
+  console.error(error.message.slice(0, 4000));
+  process.exitCode = 1;
+}