From 46a241fb29b66eae7e6e0894a76d3b09b085d67b Mon Sep 17 00:00:00 2001 From: Sylvester Damgaard Date: Thu, 8 Oct 2026 21:12:41 +0200 Subject: [PATCH] feat(service): secrets by reference, web probes, and a release job Three things a production service needs that a spec could not say. Secrets by reference. ServiceSpec::$secretRefs (SecretReference: variable, secret, key) compiles to valueFrom.secretKeyRef on every container of the service, at a Secret the operator created. Unlike $envSecret, the value never passes through the spec or anything compiled. Never optional: a missing Secret or key holds the pod in CreateContainerConfigError, which is the right failure. Names Kubernetes would refuse, a variable claimed twice, and a reference to a Secret this compiler writes are refused here. Probes. ServiceSpec::$health (HealthCheck) compiles startup + liveness (httpGet on the liveness path) and readiness for the web Deployment only. A readiness token compiles to an exec probe that reads the variable from the container's own environment and pipes the header to curl -H @-: an httpGet probe can only carry static headers, which would put the token in the pod spec. The release. ServiceSpec::$release and ReleaseJobCompiler compile the service's release command as a run Job named for its pod template's hash, kept OUT of the service's manifest set so an applier can run it to completion before the Deployments roll. The sequence is documented in docs/core-concepts/secrets-probes-releases.md. Existing goldens are byte-for-byte unchanged; service-production is new. --- CHANGELOG.md | 37 ++ README.md | 1 + docs/core-concepts/_index.md | 1 + docs/core-concepts/public-api.md | 4 +- docs/core-concepts/secrets-probes-releases.md | 167 ++++++++ docs/getting-started/installation.md | 4 +- docs/getting-started/testing.md | 1 + src/Compile/ReleaseJobCompiler.php | 101 +++++ src/Compile/RunJobCompiler.php | 6 +- src/Compile/ServiceCompiler.php | 175 +++++++- src/Contracts/ReleaseCompiler.php | 19 + src/Service/HealthCheck.php | 91 +++++ src/Service/SecretReference.php | 94 +++++ src/Service/ServiceSpec.php | 54 +++ src/Testing/CompilesPlatformIntent.php | 9 +- tests/Compile/CredentialBoundaryTest.php | 23 ++ tests/Compile/HealthProbeTest.php | 144 +++++++ tests/Compile/ReleaseJobTest.php | 128 ++++++ tests/Compile/SecretReferenceTest.php | 130 ++++++ tests/Golden/service-production.golden.yaml | 381 ++++++++++++++++++ 20 files changed, 1563 insertions(+), 7 deletions(-) create mode 100644 docs/core-concepts/secrets-probes-releases.md create mode 100644 src/Compile/ReleaseJobCompiler.php create mode 100644 src/Contracts/ReleaseCompiler.php create mode 100644 src/Service/HealthCheck.php create mode 100644 src/Service/SecretReference.php create mode 100644 tests/Compile/HealthProbeTest.php create mode 100644 tests/Compile/ReleaseJobTest.php create mode 100644 tests/Compile/SecretReferenceTest.php create mode 100644 tests/Golden/service-production.golden.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f8ccd6..42d584a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,43 @@ the package is `0.x` the public API may change in a minor release; see ## [Unreleased] +### Added + +- **Secrets by reference.** `ServiceSpec::$secretRefs` — a list of + `SecretReference(variable, secret, key)` — compiles to `valueFrom.secretKeyRef` + on every container of the service (web, processes, runs, the release) at a + Secret the operator created. The value is never in the spec or anything + compiled; a missing Secret or key fails the pod with + `CreateContainerConfigError`, never `optional`. Refused at compile time: names + Kubernetes would refuse, a variable claimed by a reference and by anything + else, and a reference to `-env` or `-registry`. +- **Probes for the web process.** `ServiceSpec::$health` — a + `HealthCheck(readinessPath, livenessPath, tokenVariable)` — compiles a + `startupProbe` and `livenessProbe` (`httpGet` on the liveness path) and a + `readinessProbe`. A readiness token compiles to an `exec` probe that reads the + variable from the container's own environment and sends it through `curl -H @-`, + because an `httpGet` probe can only carry static headers and the token would + otherwise be in the pod spec. Process Deployments, runs and releases carry no + probes. Null compiles none, as before. +- **The release.** `ServiceSpec::$release` (argv) and `ReleaseJobCompiler` + (`Contracts\ReleaseCompiler`): the run Job for the service's release command, + named `-release-` so one release is one Job. + Not part of the service's manifest set, so an applier can run it to completion + before the Deployments roll — see + [the sequence](docs/core-concepts/secrets-probes-releases.md#the-release). +- `CompilesPlatformIntent::compileRelease()`. + +### Changed + +- `RunJobCompiler` and process Deployments also strip `startupProbe` from the + container they derive from the web Deployment. +- `CompilesPlatformIntent::compileRun()` passes the held target to + `RunJobCompiler`, so a run's labels follow the target's identity like + everything else compiled in the test. + +No compiled output changes for a spec that sets none of the new fields; the +existing golden files are byte-for-byte unchanged. + ## [0.8.0] — 2026-08-09 ### Fixed diff --git a/README.md b/README.md index af49134..8dac2e4 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,7 @@ the application's. - [Labels on compiled objects](docs/core-concepts/labels.md) - [Kubernetes API versions](docs/core-concepts/api-versions.md) - [Plan and diff](docs/core-concepts/plan-and-diff.md) +- [Secrets by reference, probes and the release](docs/core-concepts/secrets-probes-releases.md) - [Public API and what `0.x` means](docs/core-concepts/public-api.md) - [Compiling without a framework](docs/cookbook/compile-without-a-framework.md) - [Writing your own mapper](docs/cookbook/writing-a-mapper.md) diff --git a/docs/core-concepts/_index.md b/docs/core-concepts/_index.md index de43e8a..ff9ebab 100644 --- a/docs/core-concepts/_index.md +++ b/docs/core-concepts/_index.md @@ -20,3 +20,4 @@ description: How the package is put together, why the compiler is pure, and what `0.x` means for you. - [Databases and the node that dies](databases.md) — why the engines compile down different paths, and what has to exist before one can have replicas. +- [Secrets by reference, probes and the release](secrets-probes-releases.md) — environment from an operator's Secret, readiness and liveness for the web process, and a command that finishes before anything rolls. diff --git a/docs/core-concepts/public-api.md b/docs/core-concepts/public-api.md index 5e8ad48..8a993f4 100644 --- a/docs/core-concepts/public-api.md +++ b/docs/core-concepts/public-api.md @@ -20,8 +20,8 @@ called out in the [changelog](../../CHANGELOG.md) before then. | Surface | What is promised | |---|---| -| `Cbox\Platform\Contracts\*` | the interfaces: `Compiler`, `DatabaseCompiler`, `BackupCompiler`, `GatewayCompiler`, `RunCompiler`, `Planner`, `SnapshotRuntime` | -| The spec value objects | `ServiceSpec`, `ProcessSpec`, `VolumeSpec`, `RuntimeSettings`, `RegistrySpec`, `DatabaseSpec`, `BackupSpec`, `BackupStorage`, `RestoreSpec`, `RunSpec`, `EnvironmentGatewaySpec`, `BindingSpec`, `ConnectionSource`, `ResourceRequirements` — construct them | +| `Cbox\Platform\Contracts\*` | the interfaces: `Compiler`, `DatabaseCompiler`, `BackupCompiler`, `GatewayCompiler`, `RunCompiler`, `ReleaseCompiler`, `Planner`, `SnapshotRuntime` | +| The spec value objects | `ServiceSpec`, `ProcessSpec`, `VolumeSpec`, `RuntimeSettings`, `RegistrySpec`, `DatabaseSpec`, `BackupSpec`, `BackupStorage`, `RestoreSpec`, `RunSpec`, `EnvironmentGatewaySpec`, `BindingSpec`, `ConnectionSource`, `ResourceRequirements`, `SecretReference`, `HealthCheck` — construct them | | `Cbox\Platform\Capability\*` | `PlatformTarget`, `HttpAutoscaler`, `BackupCatalog`, `CustomerAccess`, `Certificates`, `CertificateSource`, `Placement`, `GatewayImplementation`, `GatewayOwnership`, `CustomResourcePolicy` | | `Cbox\Platform\Manifest\*` | `Manifest`, `ManifestSet` — including `hash()`, `hashes()`, `key()`, `find()`, `toYaml()`, `toArray()`, `fromArray()` | | `Cbox\Platform\Plan\*` | `Plan`, `PlanEntry`, `PlanAction`, `HashPlanner` (including `planAgainst()`), `ManifestDiff` (including `ManifestDiff::REDACTED`), `FieldChange`, `FieldChangeKind` | diff --git a/docs/core-concepts/secrets-probes-releases.md b/docs/core-concepts/secrets-probes-releases.md new file mode 100644 index 0000000..52f9f5c --- /dev/null +++ b/docs/core-concepts/secrets-probes-releases.md @@ -0,0 +1,167 @@ +--- +title: Secrets by reference, probes and the release +weight: 27 +description: Environment from a Secret an operator created, readiness and liveness for the web process, and a command that has to finish before the workloads roll. +--- + +# Secrets by reference, probes and the release + +Three things a production service needs that a spec could not say before +`0.12`: a credential that never passes through whoever builds the spec, a +readiness probe that holds a rollout until the new pod can serve, and a +migration that runs once, before any of that. + +## Secrets by reference + +`ServiceSpec::$envSecret` carries **values**: the compiler writes them into +`-env` and references that. Whoever builds the spec holds the plaintext. + +`ServiceSpec::$secretRefs` carries **names**: + +```php +use Cbox\Platform\Service\SecretReference; + +new ServiceSpec( + // … + secretRefs: [ + new SecretReference(variable: 'APP_KEY', secret: 'cbox-id-app', key: 'APP_KEY'), + new SecretReference(variable: 'MAIL_PASSWORD', secret: 'cbox-id-mail', key: 'MAIL_PASSWORD'), + ], +); +``` + +Each one compiles to `valueFrom.secretKeyRef` on **every container of the +service** — the web Deployment, every process Deployment, every run Job and the +release Job — because they all take the web container and change its command. +The Secret is the operator's: nothing here creates, updates or prunes it, and its +value is never in the spec, the compiled set, a plan, a diff or a golden file. + +**A missing Secret, or a missing key in it, fails the pod — on purpose.** The +reference is never `optional`, so Kubernetes holds the container in +`CreateContainerConfigError` with an event naming the Secret or the key, and the +pod does not start. That is the right failure: a process that booted without its +`APP_KEY` would be serving, and wrong. Create the Secret in the service's +namespace before the first deploy. + +Refused at compile time, with a sentence rather than an API-server error: + +| Refused | Why | +|---|---| +| a variable that is not `[A-Za-z_][A-Za-z0-9_]*` | the only names every shell and runtime can read | +| a Secret name that is not an RFC 1123 subdomain (≤ 253) | Kubernetes would refuse it | +| a key outside `[-._a-zA-Z0-9]`, over 253, or `.`/`..` | Kubernetes would refuse it | +| a variable also in `env`, `envSecret`, a binding, or another reference | a container takes the last entry for a name, so one source would be silently ignored | +| `-env` or `-registry` | those are Secrets this compiler writes; the next deploy would overwrite the operator's | + +## Probes + +`ServiceSpec::$health` takes a `HealthCheck`. Null compiles no probes, which is +what every service compiled before this. + +```php +use Cbox\Platform\Service\HealthCheck; + +health: new HealthCheck( + readinessPath: '/health/ready', + livenessPath: '/up', + tokenVariable: 'HEALTH_TOKEN', // optional +), +``` + +Web Deployment only. A worker listens on nothing, so a process Deployment, a run +and the release carry no probes at all. + +| Probe | From | Shape | +|---|---|---| +| `startupProbe` | `livenessPath` | `httpGet`, every 2 s, 150 failures — five minutes to boot | +| `livenessProbe` | `livenessPath` | `httpGet`, every 10 s, timeout 3 s, restart after 3 misses | +| `readinessProbe` | `readinessPath` | `httpGet` — or `exec` with a token, below — every 10 s, timeout 5 s, out of rotation after 3 misses | + +**Startup gates liveness.** Until it passes, neither liveness nor readiness +runs, so a slow first boot is not killed for being slow. No liveness path, no +startup probe: it exists only to hold liveness back. + +**Liveness is shallow.** Point it at something that answers whenever the process +is up — Laravel's `/up`. A liveness probe on a deep check restarts every +healthy pod the moment the database is slow. + +### Readiness that needs a token + +An application that refuses to describe its dependencies to an anonymous caller +is right to. But a Kubernetes `httpGet` probe sends **static headers only** — +there is no `secretKeyRef` for a probe header — so the token would have to be +written into the pod spec, readable by anyone who can `kubectl get deploy -o +yaml`. That is not offered. + +Instead a readiness check with a `tokenVariable` compiles to an `exec` probe that +asks the container itself, with the token from its own environment: + +```sh +printf 'Authorization: Bearer %s\n' "${HEALTH_TOKEN}" \ + | curl -fsS -o /dev/null --max-time 4 -H @- 'http://127.0.0.1:80/health/ready' +``` + +- The value never leaves the Secret → env path: the probe names the variable, + the container's shell reads it. +- The header goes through stdin (`-H @-`), not argv, so the token is not in the + process list either. +- It needs `sh` and `curl` in the image. Every Cbox base image has both; an image + without curl fails readiness visibly (`curl: not found` in the pod's events) + rather than serving unprobed. +- The variable must be set by the spec — normally a `SecretReference` — or the + compile is refused: an empty token is a probe that is refused forever and a + rollout that hangs. +- Refused on the checkpointing (CRIU) scale-to-zero tier, where an idle + container is a checkpoint and has nothing to run a command in. Use a readiness + path that needs no token there. + +The path is restricted to `[A-Za-z0-9._~%/+:@=&?,-]`, because it is part of a +shell line: a quote, a dollar sign or a space is refused rather than escaped. + +The alternative — letting the readiness path answer anyone on the pod network +without a token — was rejected: it exposes which dependency is unhappy to every +pod in the cluster, which is exactly what the token is there to prevent. + +## The release + +`ServiceSpec::$release` is an argv list, `['php', 'artisan', 'migrate', '--force']`. +`ServiceCompiler` ignores it. `ReleaseJobCompiler` compiles it: + +```php +use Cbox\Platform\Compile\ReleaseJobCompiler; + +$release = new ReleaseJobCompiler($services, $target)->compile($spec); // ManifestSet: 0 or 1 Job +``` + +The Job is a [run](../getting-started/installation.md): the web container with +the command on it — same image, env, Secret references, bindings and pull +credential — no probes, `restartPolicy: Never`, `backoffLimit: 0`, an hour's +deadline, kept a day after it finishes so its logs can be read. + +**Its name is the release.** `-release-<10 hex>`, where the hash is of +the Job's pod template. The same spec compiles to the same Job, so an applier +deploying it twice finds it already `Complete` and does not migrate twice; a new +image, variable or command is a new Job. (A Job's template is immutable, so a +fixed name could only ever run the first release.) Empty while the service is +suspended. + +**It is deliberately not in the service's manifest set.** A set is applied at +once and has no order; the release must finish before the Deployments change. +The compiler keeps the two apart so the applier can sequence them: + +1. Apply the service's set **without** its `Deployment`s and the scalers that + act on them (`ScaledObject`, `HTTPScaledObject`) — the namespace, Secrets, + Services and databases the release needs. Wait for the databases to accept + connections. +2. If a Job of the release's name exists and has `Failed`, delete it (the same + release, retried). Apply the release Job and wait for its `Complete` or + `Failed` condition. Watch its pod: `CreateContainerConfigError` (a missing + Secret) or `ImagePullBackOff` will never finish on their own, so treat them as + a failure rather than waiting out the deadline. +3. On `Failed`, **stop**: report the Job's logs and apply nothing else. The + running release — the old Deployments — is untouched and still serving. +4. On `Complete`, apply the full set. The workloads roll behind their readiness + probes. + +Migrations therefore run while the previous release still serves, so they must +be safe for the old code: add, never drop or rename what it still reads. diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index d3d7f08..142cc3b 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -24,6 +24,7 @@ use Cbox\Platform\Compile\BackupCompiler; use Cbox\Platform\Compile\CnpgDatabaseCompiler; use Cbox\Platform\Compile\EngineDatabaseCompiler; use Cbox\Platform\Compile\EnvironmentGatewayCompiler; +use Cbox\Platform\Compile\ReleaseJobCompiler; use Cbox\Platform\Compile\RunJobCompiler; use Cbox\Platform\Compile\ServiceCompiler; use Cbox\Platform\Compile\StatefulDatabaseCompiler; @@ -37,7 +38,8 @@ $databases = new EngineDatabaseCompiler( new StatefulDatabaseCompiler($target, $backups), ); $gateways = new EnvironmentGatewayCompiler(); -$runs = new RunJobCompiler($services); +$runs = new RunJobCompiler($services, $target); +$releases = new ReleaseJobCompiler($services, $target); // run BEFORE the service's Deployments ``` ## Wiring in a Laravel container diff --git a/docs/getting-started/testing.md b/docs/getting-started/testing.md index 39ca34b..596cdd1 100644 --- a/docs/getting-started/testing.md +++ b/docs/getting-started/testing.md @@ -40,6 +40,7 @@ class DeployTest extends TestCase | `compileBackup(BackupSpec)` | | | `compileGateway(EnvironmentGatewaySpec)` | | | `compileRun(RunSpec)` | | +| `compileRelease(ServiceSpec)` | the release Job; an empty set when there is none | ## Fakes, for testing the layer above the compiler diff --git a/src/Compile/ReleaseJobCompiler.php b/src/Compile/ReleaseJobCompiler.php new file mode 100644 index 0000000..0e18b82 --- /dev/null +++ b/src/Compile/ReleaseJobCompiler.php @@ -0,0 +1,101 @@ +-release-`, where the hash is of the + * Job's pod template — image, command, environment, everything. So: + * + * - the same release deployed twice is the SAME Job, and an applier finds it + * already complete instead of migrating twice; + * - a new image, a changed variable or a different command is a NEW Job. + * A Job's pod template is immutable once created, so re-applying one name + * with a new image is refused by the API server — a fixed name could only + * ever run the first release; + * - it is a pure function of the spec, like everything else here: no clock, + * no counter, no release number the compiler would have to be told. + * + * NOT PART OF THE SERVICE'S MANIFEST SET, and that is the point rather than an + * omission. A set is applied at once and has no order; this has to finish, and + * succeed, before the Deployments in that set are allowed to change. Keeping it + * apart is what lets an applier sequence it: + * + * 1. apply the service's set WITHOUT its Deployments and scalers + * (the namespace, Secrets, Services, databases — what the release needs) + * 2. apply this Job and wait for `Complete` — or `Failed`, and stop there, + * with the running release untouched + * 3. apply the full set, and the workloads roll + * + * NOTHING WHILE SUSPENDED. A service that is put away runs nothing, and a + * migration nobody is going to serve is a migration nobody is watching. + */ +class ReleaseJobCompiler implements ReleaseCompiler +{ + /** The longest a Job name can be and still label its pods (`job-name`). */ + private const MAX_NAME = 63; + + public function __construct( + private readonly Compiler $services, + private readonly PlatformTarget $target = new PlatformTarget, + ) {} + + public function compile(ServiceSpec $spec): ManifestSet + { + if ($spec->release === [] || $spec->suspended) { + return new ManifestSet([]); + } + + $job = new RunJobCompiler($this->services, $this->target)->compile(new RunSpec( + runId: 'release', + jobName: $spec->name.'-release', + command: $spec->release, + service: $spec, + ))->manifests[0]; + + // The template's canonical hash — the same canonicalisation plan/diff + // uses, so two specs that compile to the same pod are the same release. + /** @var array $template */ + $template = $job->map('spec', 'template'); + $name = self::jobName($spec->name, new Manifest('batch/v1', 'Job', '', '', $template)->hash()); + + $body = $job->body; + $body['metadata'] = ['name' => $name] + $job->map('metadata'); + + return new ManifestSet([new Manifest( + apiVersion: $job->apiVersion, + kind: $job->kind, + name: $name, + namespace: $job->namespace, + body: $body, + )]); + } + + /** + * `-release-<10 hex>`, shortened from the service's end so the + * whole thing stays a valid pod label. + */ + public static function jobName(string $service, string $hash): string + { + $suffix = '-release-'.substr($hash, 0, 10); + + return rtrim(substr($service, 0, self::MAX_NAME - strlen($suffix)), '-').$suffix; + } +} diff --git a/src/Compile/RunJobCompiler.php b/src/Compile/RunJobCompiler.php index 3036a58..37a888b 100644 --- a/src/Compile/RunJobCompiler.php +++ b/src/Compile/RunJobCompiler.php @@ -58,8 +58,10 @@ public function compile(RunSpec $spec): ManifestSet $container['command'] = $spec->command; // A run does not serve. Ports on a Job are inert at best, and a - // readiness probe on a container that exits is a contradiction. - unset($container['ports'], $container['readinessProbe'], $container['livenessProbe']); + // readiness probe on a container that exits is a contradiction — a + // liveness or startup probe on one would kill a long migration for not + // answering HTTP. + unset($container['ports'], $container['readinessProbe'], $container['livenessProbe'], $container['startupProbe']); $podSpec['containers'] = [$container]; $podSpec['restartPolicy'] = 'Never'; diff --git a/src/Compile/ServiceCompiler.php b/src/Compile/ServiceCompiler.php index 4e30cd0..caf3966 100644 --- a/src/Compile/ServiceCompiler.php +++ b/src/Compile/ServiceCompiler.php @@ -39,6 +39,8 @@ public function compile(ServiceSpec $spec): ManifestSet $this->guardVolumeReplicas($spec); $this->guardAutoscaleHasCpuRequest($spec); $this->guardApplicationSource($spec); + $this->guardSecretReferences($spec); + $this->guardHealth($spec); $manifests = [$this->namespace($spec)]; @@ -460,7 +462,11 @@ private function processDeployment(ServiceSpec $spec, ProcessSpec $process): Man $container['name'] = $process->name; $container['command'] = $process->command; - unset($container['ports'], $container['readinessProbe'], $container['livenessProbe']); + // The probes are the WEB process's: they ask an HTTP server whether it + // can serve, and a queue worker listens on nothing — every one of them + // would fail, and the liveness probe would restart a healthy worker + // every thirty seconds. + unset($container['ports'], $container['readinessProbe'], $container['livenessProbe'], $container['startupProbe']); $mounted = $this->mountsFor($spec, $process->name); @@ -834,6 +840,162 @@ private function guardApplicationSource(ServiceSpec $spec): void } } + /** + * A referenced Secret has to be the operator's, and its variable has to be + * the only one of that name. + * + * ONE NAME, ONE SOURCE. A container given the same variable twice takes the + * LAST entry — so `APP_KEY` both as a plain value and as a reference is one + * of the two silently ignored, and which one depends on an order nobody + * reading the spec can see. + * + * NOT A SECRET THIS COMPILER WRITES. `-env` and `-registry` + * are compiled, applied and pruned with the service; an operator's Secret of + * the same name would be overwritten by the next deploy, or deleted with the + * service, and the reference would quietly start reading the platform's own. + */ + private function guardSecretReferences(ServiceSpec $spec): void + { + if ($spec->secretRefs === []) { + return; + } + + $declared = array_count_values($spec->declaredVariables()); + $owned = [$this->secretName($spec), $this->pullSecretName($spec)]; + + foreach ($spec->secretRefs as $reference) { + if (($declared[$reference->variable] ?? 0) > 1) { + throw new \LogicException( + "[{$reference->variable}] is declared more than once in service [{$spec->name}] — " + ."as a reference to Secret [{$reference->secret}] and again elsewhere. A container " + .'takes the last entry for a name, so one of them would be silently ignored. Keep one.' + ); + } + + if (in_array($reference->secret, $owned, true)) { + throw new \LogicException( + "[{$reference->secret}] is a Secret this platform writes for service [{$spec->name}], " + .'so it cannot also be one an operator creates: the next deploy would overwrite it. ' + .'Name the Secret something else.' + ); + } + } + } + + /** + * A token probe needs a token, and a container it can run in. + * + * Without the variable the probe sends `Authorization: Bearer ` with nothing + * after it, every check answers 403, and the pod is never ready — a rollout + * that hangs with nothing anywhere saying why. + * + * NOT ON THE SNAPSHOT TIER. A token probe is a command run INSIDE the + * container, and on that tier an idle container is a checkpoint, not a + * process: every probe would either fail or restore it, which is the + * opposite of idling. An unauthenticated readiness path, or no scale to + * zero, are the two shapes that work. + */ + private function guardHealth(ServiceSpec $spec): void + { + $health = $spec->health; + + if ($health === null || ! $health->authenticated()) { + return; + } + + if (! in_array($health->tokenVariable, $spec->declaredVariables(), true)) { + throw new \LogicException( + "Service [{$spec->name}] presents [{$health->tokenVariable}] to its readiness probe, and " + .'nothing sets it. The probe would send an empty token, be refused, and the pod would ' + .'never become ready. Reference it from a Secret.' + ); + } + + if ($this->criuTier($spec)) { + throw new \LogicException( + "Service [{$spec->name}] scales to zero by checkpointing, and its readiness probe needs a " + .'token — which means running a command inside a container that is not running between ' + .'requests. Give it a readiness path that answers without a token, or turn off scale to zero.' + ); + } + } + + /** + * The web process's probes, from its health check. + * + * STARTUP GATES LIVENESS, and it is what makes a liveness probe safe to + * have. A Laravel container caches its config, routes and views at start, + * and on a cold node with a slow volume that is tens of seconds; a liveness + * probe running from the first second kills it for being slow and the pod + * never finishes booting. Until the startup probe passes, liveness and + * readiness do not run at all — and it gets five minutes, probing every two + * seconds, so a fast boot is not made slow by waiting for the next period. + * + * LIVENESS IS SHALLOW AND PATIENT: three misses ten seconds apart before a + * restart, against a path that should not depend on anything. + * + * READINESS WITH A TOKEN IS AN EXEC PROBE. `httpGet` takes static headers + * only — writing the token there would put its value in the pod spec, which + * is exactly where secrets are kept out of. So the container asks itself, + * with the token from its own environment: + * + * printf 'Authorization: Bearer %s\n' "${HEALTH_TOKEN}" \ + * | curl -fsS -o /dev/null --max-time 4 -H @- 'http://127.0.0.1:80/health/ready' + * + * The header goes through stdin, never argv, so the token is not in the + * process list either. It needs `sh` and `curl` in the image; every Cbox + * base image has both, and an image without curl fails readiness visibly + * (`curl: not found` in the pod's events) rather than serving unprobed. + * + * @return array> + */ + private function probes(ServiceSpec $spec): array + { + $health = $spec->health; + + if ($health === null) { + return []; + } + + $probes = []; + + if ($health->livenessPath !== '') { + $get = ['httpGet' => ['path' => $health->livenessPath, 'port' => 'http']]; + + $probes['startupProbe'] = $get + [ + 'periodSeconds' => 2, + 'timeoutSeconds' => 3, + 'failureThreshold' => 150, + ]; + + $probes['livenessProbe'] = $get + [ + 'periodSeconds' => 10, + 'timeoutSeconds' => 3, + 'failureThreshold' => 3, + ]; + } + + if ($health->readinessPath !== '') { + $check = $health->authenticated() + ? ['exec' => ['command' => [ + 'sh', '-c', + 'printf \'Authorization: Bearer %s\n\' "${'.$health->tokenVariable.'}"' + .' | curl -fsS -o /dev/null --max-time 4 -H @-' + ." 'http://127.0.0.1:{$spec->port}{$health->readinessPath}'", + ]]] + : ['httpGet' => ['path' => $health->readinessPath, 'port' => 'http']]; + + $probes['readinessProbe'] = $check + [ + 'periodSeconds' => 10, + 'timeoutSeconds' => 5, + 'failureThreshold' => 3, + 'successThreshold' => 1, + ]; + } + + return $probes; + } + private function guardName(string $name): void { if (preg_match('/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/', $name) !== 1 || strlen($name) > 63) { @@ -906,6 +1068,15 @@ private function deployment(ServiceSpec $spec): Manifest ]; } + // AND FROM SECRETS THIS COMPILER DID NOT WRITE. The operator created + // them; the spec carries only where they are. Never `optional`: a + // missing Secret or key holds the pod in CreateContainerConfigError, + // which is the right failure — a process that booted with an empty + // APP_KEY would be serving, and wrong. + foreach ($spec->secretRefs as $reference) { + $env[] = $reference->toEnv(); + } + $container = [ 'name' => $spec->name, // THE BASE IMAGE RUNS, and the application is mounted onto it. @@ -931,6 +1102,8 @@ private function deployment(ServiceSpec $spec): Manifest $container['env'] = $env; } + $container = [...$container, ...$this->probes($spec)]; + $mounted = $this->mountsFor($spec, 'web'); if ($mounted['mounts'] !== []) { diff --git a/src/Contracts/ReleaseCompiler.php b/src/Contracts/ReleaseCompiler.php new file mode 100644 index 0000000..40997d5 --- /dev/null +++ b/src/Contracts/ReleaseCompiler.php @@ -0,0 +1,19 @@ + $readinessPath, 'liveness' => $livenessPath] as $what => $path) { + if ($path !== '' && ! self::isPath($path)) { + throw new LogicException( + "[{$path}] cannot be the {$what} path. Write an absolute path such as /health/ready — " + .'letters, digits and . _ ~ % / + : @ = & ? , - only, with no spaces or quotes.' + ); + } + } + + if ($tokenVariable !== '' && ! SecretReference::isVariableName($tokenVariable)) { + throw new LogicException( + "[{$tokenVariable}] cannot be the health token's variable. Name the environment variable " + .'that holds it — letters, digits and underscores.' + ); + } + + if ($tokenVariable !== '' && $readinessPath === '') { + throw new LogicException( + 'A health token is presented to the readiness path, and there is none. Liveness never ' + .'takes one: it has to answer even when the application cannot.' + ); + } + } + + /** + * A path a probe can request, and that is safe inside a shell's single quotes. + * + * Narrower than RFC 3986 on purpose. The token probe runs through `sh -c`, + * and a path is part of that line — so a quote, a backslash, a dollar sign + * or a space is refused rather than escaped. No real health endpoint needs one. + */ + public static function isPath(string $path): bool + { + return preg_match('#^/[A-Za-z0-9._~%/+:@=&?,-]*$#', $path) === 1; + } + + public function authenticated(): bool + { + return $this->tokenVariable !== ''; + } +} diff --git a/src/Service/SecretReference.php b/src/Service/SecretReference.php new file mode 100644 index 0000000..f8e71d1 --- /dev/null +++ b/src/Service/SecretReference.php @@ -0,0 +1,94 @@ + $this->variable, + 'valueFrom' => ['secretKeyRef' => ['name' => $this->secret, 'key' => $this->key]], + ]; + } +} diff --git a/src/Service/ServiceSpec.php b/src/Service/ServiceSpec.php index 003ac82..5bd3e03 100644 --- a/src/Service/ServiceSpec.php +++ b/src/Service/ServiceSpec.php @@ -146,8 +146,62 @@ public function __construct( * @var list */ public array $mounts = [], + /** + * Environment variables taken from Secrets the operator created. + * + * The counterpart of `$envSecret`: that one carries values and the + * compiler writes the Secret; this one carries only names, so the value + * is never in the spec at all. Compiled to `secretKeyRef` on every + * container of the service — web, every process, a run and the release. + * + * @var list + */ + public array $secretRefs = [], + /** + * Readiness, liveness and startup probes for the web process. + * + * Null compiles no probes, which is exactly what was compiled before this + * was expressible: a pod counted as ready the moment its container + * started. + */ + public ?HealthCheck $health = null, + /** + * A command to run to completion, once per release, before the + * workloads roll — `php artisan migrate --force`. + * + * Compiled by ReleaseJobCompiler, never by ServiceCompiler: the Job has + * to finish BEFORE the Deployments change, and one manifest set applied + * at once has no order. Sequencing it is the applier's job; the compiler + * makes it possible by keeping the two apart. + * + * @var list argv, never a shell string + */ + public array $release = [], ) {} + /** + * Every environment variable name the spec itself declares, whatever its + * source — so two sources can be refused from claiming one name. + * + * @return list + */ + public function declaredVariables(): array + { + $names = [...array_keys($this->env), ...array_keys($this->envSecret)]; + + foreach ($this->bindings as $binding) { + foreach ($binding->map as $entry) { + $names[] = $entry['name']; + } + } + + foreach ($this->secretRefs as $reference) { + $names[] = $reference->variable; + } + + return array_map(strval(...), $names); + } + /** What this service asks of a node, defaults included. */ public function resources(): ResourceRequirements { diff --git a/src/Testing/CompilesPlatformIntent.php b/src/Testing/CompilesPlatformIntent.php index d2cfa58..b2e4ed3 100644 --- a/src/Testing/CompilesPlatformIntent.php +++ b/src/Testing/CompilesPlatformIntent.php @@ -9,6 +9,7 @@ use Cbox\Platform\Compile\CnpgDatabaseCompiler; use Cbox\Platform\Compile\EngineDatabaseCompiler; use Cbox\Platform\Compile\EnvironmentGatewayCompiler; +use Cbox\Platform\Compile\ReleaseJobCompiler; use Cbox\Platform\Compile\RunJobCompiler; use Cbox\Platform\Compile\ServiceCompiler; use Cbox\Platform\Compile\StatefulDatabaseCompiler; @@ -88,6 +89,12 @@ protected function compileGateway(EnvironmentGatewaySpec $spec): ManifestSet protected function compileRun(RunSpec $spec): ManifestSet { - return new RunJobCompiler(new ServiceCompiler($this->target()))->compile($spec); + return new RunJobCompiler(new ServiceCompiler($this->target()), $this->target())->compile($spec); + } + + /** The service's release Job — empty when it has no release command. */ + protected function compileRelease(ServiceSpec $spec): ManifestSet + { + return new ReleaseJobCompiler(new ServiceCompiler($this->target()), $this->target())->compile($spec); } } diff --git a/tests/Compile/CredentialBoundaryTest.php b/tests/Compile/CredentialBoundaryTest.php index 0303c13..385e580 100644 --- a/tests/Compile/CredentialBoundaryTest.php +++ b/tests/Compile/CredentialBoundaryTest.php @@ -7,6 +7,7 @@ use Cbox\Platform\Database\DatabaseSpec; use Cbox\Platform\Run\RunSpec; use Cbox\Platform\Service\RegistrySpec; +use Cbox\Platform\Service\SecretReference; use Cbox\Platform\Service\ServiceSpec; /** @@ -103,6 +104,7 @@ function bodiesOutsideSecrets(iterable $manifests): array 'StatefulDatabaseCompiler', 'BackupCompiler', // reached through the database sweep 'RunJobCompiler', // derives its container from the Deployment + 'ReleaseJobCompiler', // a run, named for its release — below 'EnvironmentGatewayCompiler', // holds no credential at all 'CustomerAccessCompiler', // RBAC only ]; @@ -127,3 +129,24 @@ function bodiesOutsideSecrets(iterable $manifests): array expect($body)->not->toContain(SECRETS['env'], $key); } }); + +it('carries a release no credential, and a referenced Secret as a name only', function (): void { + // The release is a run, so it inherits the Deployment's references — and + // an operator's Secret is never anything but its name and key. + $service = new ServiceSpec( + serviceId: 'svc', organizationId: 'org', namespace: 'ns', name: 'web', + image: 'img', port: 80, replicas: 1, + envSecret: ['APP_KEY' => SECRETS['env']], + secretRefs: [new SecretReference('MAIL_PASSWORD', 'web-mail', 'MAIL_PASSWORD')], + release: ['php', 'artisan', 'migrate', '--force'], + ); + + $set = test()->compileRelease($service); + + expect($set->manifests)->toHaveCount(1); + + foreach (bodiesOutsideSecrets($set->manifests) as $key => $body) { + expect($body)->not->toContain(SECRETS['env'], $key) + ->and($body)->toContain('web-mail'); + } +}); diff --git a/tests/Compile/HealthProbeTest.php b/tests/Compile/HealthProbeTest.php new file mode 100644 index 0000000..2b48fff --- /dev/null +++ b/tests/Compile/HealthProbeTest.php @@ -0,0 +1,144 @@ + '01J0000000000000000000SVC1', + 'organizationId' => '01J0000000000000000000ORG1', + 'namespace' => 'cx-production-db9k2', + 'name' => 'web', + 'image' => 'acme/app:1.4.0', + 'port' => 80, + 'replicas' => 2, + 'secretRefs' => [new SecretReference('HEALTH_TOKEN', 'app', 'HEALTH_TOKEN')], + 'processes' => [new ProcessSpec('queue', ['php', 'artisan', 'queue:work'], 1)], + 'health' => $health, + ...$extra, + ]); +} + +/** @return array */ +function probedContainer(ServiceSpec $spec, string $deployment = 'web'): array +{ + return test()->compileService($spec)->find('Deployment/'.$deployment)?->body['spec']['template']['spec']['containers'][0] ?? []; +} + +it('compiles no probes when the service declares no health check', function (): void { + expect(probedContainer(probedSpec(null))) + ->not->toHaveKey('readinessProbe') + ->not->toHaveKey('livenessProbe') + ->not->toHaveKey('startupProbe'); +}); + +it('probes readiness with the token from the container\'s own environment', function (): void { + $container = probedContainer(probedSpec(new HealthCheck('/health/ready', '/up', 'HEALTH_TOKEN'))); + + expect($container['readinessProbe'])->toBe([ + 'exec' => ['command' => [ + 'sh', '-c', + 'printf \'Authorization: Bearer %s\n\' "${HEALTH_TOKEN}"' + .' | curl -fsS -o /dev/null --max-time 4 -H @- \'http://127.0.0.1:80/health/ready\'', + ]], + 'periodSeconds' => 10, + 'timeoutSeconds' => 5, + 'failureThreshold' => 3, + 'successThreshold' => 1, + ]); + + // THE VALUE IS NOWHERE IN THE SPEC. A static httpGet header would have put + // it there; the exec probe names the variable and the shell reads it. + expect(json_encode($container['readinessProbe'], JSON_THROW_ON_ERROR))->not->toContain('secretKeyRef'); +}); + +it('probes liveness shallowly, and gates it behind a patient startup probe', function (): void { + $container = probedContainer(probedSpec(new HealthCheck('/health/ready', '/up', 'HEALTH_TOKEN'))); + + expect($container['livenessProbe'])->toBe([ + 'httpGet' => ['path' => '/up', 'port' => 'http'], + 'periodSeconds' => 10, + 'timeoutSeconds' => 3, + 'failureThreshold' => 3, + ]); + + // Five minutes to boot, asked every two seconds: a slow first start is not + // killed, and a fast one is not made to wait for the next period. + expect($container['startupProbe']['httpGet'])->toBe(['path' => '/up', 'port' => 'http']) + ->and($container['startupProbe']['periodSeconds'] * $container['startupProbe']['failureThreshold'])->toBe(300); +}); + +it('probes readiness over plain HTTP when it needs no token', function (): void { + $container = probedContainer(probedSpec(new HealthCheck('/ready'))); + + expect($container['readinessProbe']['httpGet'])->toBe(['path' => '/ready', 'port' => 'http']) + // No liveness path, no liveness probe — and no startup probe, which only + // exists to hold liveness back. + ->and($container)->not->toHaveKey('livenessProbe') + ->and($container)->not->toHaveKey('startupProbe'); +}); + +it('probes only the web process', function (): void { + $spec = probedSpec(new HealthCheck('/health/ready', '/up', 'HEALTH_TOKEN')); + + // A queue worker listens on nothing: every probe would fail, and liveness + // would restart a healthy worker every thirty seconds. + expect(probedContainer($spec, 'web-queue')) + ->not->toHaveKey('readinessProbe') + ->not->toHaveKey('livenessProbe') + ->not->toHaveKey('startupProbe'); + + $run = test()->compileRun(new RunSpec('run-1', 'web-run-1', ['php', 'artisan', 'migrate'], $spec)); + $container = $run->manifests[0]->body['spec']['template']['spec']['containers'][0]; + + expect($container)->not->toHaveKey('readinessProbe') + ->not->toHaveKey('livenessProbe') + ->not->toHaveKey('startupProbe'); +}); + +it('refuses a token nothing in the service sets', function (): void { + expect(fn () => test()->compileService(probedSpec( + new HealthCheck('/health/ready', '/up', 'MISSING_TOKEN'), + )))->toThrow(LogicException::class, 'nothing sets it'); +}); + +it('refuses a token probe on the checkpointing tier', function (): void { + test()->compilingFor(new PlatformTarget(snapshotRuntime: new ZeropodSnapshotRuntime('zeropod'))); + + expect(fn () => test()->compileService(probedSpec( + new HealthCheck('/health/ready', '/up', 'HEALTH_TOKEN'), + ['scaleToZero' => true, 'domains' => ['app.acme.example']], + )))->toThrow(LogicException::class, 'checkpointing'); + + // Without a token the probe is an HTTP request the runtime handles like any other. + expect(probedContainer(probedSpec( + new HealthCheck('/ready', '/up'), + ['scaleToZero' => true, 'domains' => ['app.acme.example']], + )))->toHaveKey('readinessProbe'); +}); + +it('refuses a health check it could not compile safely', function (string $readiness, string $liveness, string $token, string $message): void { + expect(fn () => new HealthCheck($readiness, $liveness, $token))->toThrow(LogicException::class, $message); +})->with([ + 'nothing to probe' => ['', '', '', 'readiness path, a liveness path, or both'], + 'relative path' => ['health', '', '', 'cannot be the readiness path'], + 'quote in the path' => ["/ready'; rm -rf /", '', '', 'cannot be the readiness path'], + 'dollar in the path' => ['/ready$(id)', '', '', 'cannot be the readiness path'], + 'space in the liveness path' => ['', '/u p', '', 'cannot be the liveness path'], + 'token variable with a hyphen' => ['/ready', '', 'HEALTH-TOKEN', 'health token'], + 'token without readiness' => ['', '/up', 'HEALTH_TOKEN', 'presented to the readiness path'], +]); diff --git a/tests/Compile/ReleaseJobTest.php b/tests/Compile/ReleaseJobTest.php new file mode 100644 index 0000000..6a45a05 --- /dev/null +++ b/tests/Compile/ReleaseJobTest.php @@ -0,0 +1,128 @@ + 'cbox-id', + 'organizationId' => 'local', + 'namespace' => 'cbox-cbox-id', + 'name' => 'cbox-id', + 'image' => 'ghcr.io/cboxdk/php-baseimages/php-fpm-nginx:8.5-bookworm-v1', + 'port' => 80, + 'replicas' => 2, + 'env' => ['APP_ENV' => 'production', 'LOG_CHANNEL' => 'stderr'], + 'bindings' => [new BindingSpec( + databaseName: 'database', + engine: 'postgres', + map: [ + ['field' => ConnectionField::Host, 'name' => 'DB_HOST'], + ['field' => ConnectionField::Password, 'name' => 'DB_PASSWORD'], + ], + source: new ConnectionSource('database-app', ['password' => 'password'], ['host' => 'database-rw']), + )], + 'processes' => [ + new ProcessSpec('queue', ['php', 'artisan', 'queue:autoscale'], 1), + new ProcessSpec('scheduler', ['php', 'artisan', 'schedule:work'], 1), + ], + 'secretRefs' => [ + new SecretReference('APP_KEY', 'cbox-id-app', 'APP_KEY'), + new SecretReference('HEALTH_TOKEN', 'cbox-id-app', 'HEALTH_TOKEN'), + ], + 'health' => new HealthCheck('/health/ready', '/up', 'HEALTH_TOKEN'), + 'release' => ['php', 'artisan', 'migrate', '--force'], + ...$override, + ]); +} + +it('compiles nothing to run first when there is no release command', function (): void { + expect(test()->compileRelease(releasedSpec(['release' => []]))->manifests)->toBe([]); +}); + +it('runs nothing for a service that is put away', function (): void { + // A migration nobody is going to serve is a migration nobody is watching. + expect(test()->compileRelease(releasedSpec(['suspended' => true]))->manifests)->toBe([]); +}); + +it('runs the release in exactly what the new pods will see', function (): void { + $spec = releasedSpec(); + $job = test()->compileRelease($spec)->manifests[0]; + $web = test()->compileService($spec)->find('Deployment/cbox-id'); + + $container = $job->body['spec']['template']['spec']['containers'][0]; + $served = $web?->body['spec']['template']['spec']['containers'][0] ?? []; + + expect($job->kind)->toBe('Job') + ->and($container['command'])->toBe(['php', 'artisan', 'migrate', '--force']) + // The same image, env, references and binding — it IS the web + // container, with a command on it. + ->and($container['image'])->toBe($served['image']) + ->and($container['env'])->toBe($served['env']) + ->and($container)->not->toHaveKeys(['ports', 'readinessProbe', 'livenessProbe', 'startupProbe']) + ->and($job->body['spec']['template']['spec']['restartPolicy'])->toBe('Never') + // A migration that failed halfway is not improved by running again + // unattended. + ->and($job->body['spec']['backoffLimit'])->toBe(0); +}); + +it('names the Job for what it runs', function (): void { + $first = test()->compileRelease(releasedSpec())->manifests[0]; + $again = test()->compileRelease(releasedSpec())->manifests[0]; + $newImage = test()->compileRelease(releasedSpec(['image' => 'ghcr.io/cboxdk/php-baseimages/php-fpm-nginx:8.5-bookworm-v2']))->manifests[0]; + $newCommand = test()->compileRelease(releasedSpec(['release' => ['php', 'artisan', 'migrate', '--force', '--step']]))->manifests[0]; + $newSecret = test()->compileRelease(releasedSpec(['secretRefs' => [ + new SecretReference('APP_KEY', 'cbox-id-app', 'APP_KEY'), + new SecretReference('HEALTH_TOKEN', 'cbox-id-app', 'HEALTH_TOKEN'), + new SecretReference('MAIL_PASSWORD', 'cbox-id-mail', 'MAIL_PASSWORD'), + ]]))->manifests[0]; + + // The same release twice is the same Job — an applier finds it complete + // instead of migrating twice. + expect($again->name)->toBe($first->name) + ->and($first->name)->toMatch('/^cbox-id-release-[0-9a-f]{10}$/') + ->and($first->body['metadata']['name'])->toBe($first->name); + + // Anything that changes the pod is a new Job. A Job's template is immutable, + // so a fixed name could only ever run the first release. + expect([$newImage->name, $newCommand->name, $newSecret->name])->each->not->toBe($first->name); + expect(array_unique([$newImage->name, $newCommand->name, $newSecret->name]))->toHaveCount(3); +}); + +it('keeps the name a valid pod label for a long service name', function (): void { + $name = ReleaseJobCompiler::jobName(str_repeat('a', 50).'-'.str_repeat('b', 12), str_repeat('f', 64)); + + expect(strlen($name))->toBeLessThanOrEqual(63) + ->and($name)->toMatch('/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/') + ->and($name)->toEndWith('-release-ffffffffff'); +}); + +it('matches the production-shaped golden, service and release, byte for byte', function (): void { + $yaml = test()->compileService(releasedSpec())->toYaml() + ."---\n".test()->compileRelease(releasedSpec())->toYaml(); + + $golden = test()->golden('service-production'); + + if (getenv('UPDATE_GOLDEN') === '1' || ! file_exists($golden)) { + file_put_contents($golden, $yaml); + } + + expect($yaml)->toBe(file_get_contents($golden)); +}); diff --git a/tests/Compile/SecretReferenceTest.php b/tests/Compile/SecretReferenceTest.php new file mode 100644 index 0000000..c9bb227 --- /dev/null +++ b/tests/Compile/SecretReferenceTest.php @@ -0,0 +1,130 @@ + '01J0000000000000000000SVC1', + 'organizationId' => '01J0000000000000000000ORG1', + 'namespace' => 'cx-production-db9k2', + 'name' => 'web', + 'image' => 'acme/app:1.4.0', + 'port' => 80, + 'replicas' => 2, + 'env' => ['APP_ENV' => 'production'], + 'secretRefs' => $references, + ...$extra, + ]); +} + +/** @return array> */ +function referencedEnvOf(?Manifest $manifest): array +{ + return collect($manifest->body['spec']['template']['spec']['containers'][0]['env'])->keyBy('name')->all(); +} + +it('compiles a reference to a secretKeyRef at the operator\'s Secret', function (): void { + $set = test()->compileService(referencingSpec([ + new SecretReference('APP_KEY', 'cbox-id-app', 'APP_KEY'), + new SecretReference('MAIL_PASSWORD', 'cbox-id-mail', 'password'), + ])); + + $env = referencedEnvOf($set->find('Deployment/web')); + + expect($env['APP_KEY'])->toBe([ + 'name' => 'APP_KEY', + 'valueFrom' => ['secretKeyRef' => ['name' => 'cbox-id-app', 'key' => 'APP_KEY']], + ]) + ->and($env['MAIL_PASSWORD']['valueFrom']['secretKeyRef'])->toBe(['name' => 'cbox-id-mail', 'key' => 'password']) + // Never optional. A missing Secret holds the pod in + // CreateContainerConfigError, which is the honest failure — a pod that + // booted without its APP_KEY would be serving, and wrong. + ->and($env['APP_KEY']['valueFrom']['secretKeyRef'])->not->toHaveKey('optional'); + + // The operator's Secret is THEIRS: nothing here creates, overwrites or + // prunes it. + expect(collect($set->manifests)->where('kind', 'Secret')->pluck('name')->all())->toBe([]); +}); + +it('gives every process and every run the same references as the web process', function (): void { + $spec = referencingSpec( + [new SecretReference('APP_KEY', 'cbox-id-app', 'APP_KEY')], + ['processes' => [ + new ProcessSpec('queue', ['php', 'artisan', 'queue:work'], 1), + new ProcessSpec('scheduler', ['php', 'artisan', 'schedule:work'], 1), + ]], + ); + + $set = test()->compileService($spec); + $expected = ['secretKeyRef' => ['name' => 'cbox-id-app', 'key' => 'APP_KEY']]; + + foreach (['web', 'web-queue', 'web-scheduler'] as $deployment) { + expect(referencedEnvOf($set->find('Deployment/'.$deployment))['APP_KEY']['valueFrom'])->toBe($expected); + } + + $run = test()->compileRun(new RunSpec('run-1', 'web-run-1', ['php', 'artisan', 'tinker'], $spec)); + + expect(referencedEnvOf($run->manifests[0])['APP_KEY']['valueFrom'])->toBe($expected); +}); + +it('refuses a variable claimed by a reference and by anything else', function (array $extra): void { + expect(fn () => test()->compileService(referencingSpec( + [new SecretReference('APP_KEY', 'cbox-id-app', 'APP_KEY')], + $extra, + )))->toThrow(LogicException::class, 'declared more than once'); +})->with([ + 'a plain value' => [['env' => ['APP_KEY' => 'base64:inline']]], + 'a compiled secret' => [['envSecret' => ['APP_KEY' => 'base64:inline']]], + 'a binding' => [['bindings' => [new BindingSpec( + databaseName: 'db', + engine: 'postgres', + map: [['field' => ConnectionField::Password, 'name' => 'APP_KEY']], + source: new ConnectionSource('db-app', ['password' => 'password'], []), + )]]], + 'another reference' => [['secretRefs' => [ + new SecretReference('APP_KEY', 'cbox-id-app', 'APP_KEY'), + new SecretReference('APP_KEY', 'cbox-id-other', 'APP_KEY'), + ]]], +]); + +it('refuses to reference a Secret this platform writes itself', function (string $secret): void { + expect(fn () => test()->compileService(referencingSpec([new SecretReference('X', $secret, 'X')]))) + ->toThrow(LogicException::class, 'a Secret this platform writes'); +})->with(['web-env', 'web-registry']); + +it('refuses names Kubernetes would refuse, before anything is compiled', function (string $variable, string $secret, string $key, string $message): void { + expect(fn () => new SecretReference($variable, $secret, $key))->toThrow(LogicException::class, $message); +})->with([ + 'variable with a hyphen' => ['APP-KEY', 'app', 'k', 'environment variable name'], + 'variable starting with a digit' => ['1KEY', 'app', 'k', 'environment variable name'], + 'empty variable' => ['', 'app', 'k', 'environment variable name'], + 'upper-case secret' => ['K', 'App', 'k', 'Secret name'], + 'secret ending in a hyphen' => ['K', 'app-', 'k', 'Secret name'], + 'secret too long' => ['K', str_repeat('a', 254), 'k', 'Secret name'], + 'key with a slash' => ['K', 'app', 'a/b', 'key in a Secret'], + 'key that is a dot' => ['K', 'app', '.', 'key in a Secret'], + 'key starting with two dots' => ['K', 'app', '..x', 'key in a Secret'], +]); + +it('accepts the names Kubernetes accepts', function (): void { + $reference = new SecretReference('_lower_ok_1', 'cbox-id.app-2', 'tls.crt_KEY-1'); + + expect($reference->toEnv()['valueFrom']['secretKeyRef'])->toBe(['name' => 'cbox-id.app-2', 'key' => 'tls.crt_KEY-1']); +}); diff --git a/tests/Golden/service-production.golden.yaml b/tests/Golden/service-production.golden.yaml new file mode 100644 index 0000000..02e9ef2 --- /dev/null +++ b/tests/Golden/service-production.golden.yaml @@ -0,0 +1,381 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cbox-cbox-id + labels: + platform.cbox.dk/managed: 'true' + platform.cbox.dk/organization: local + platform.cbox.dk/service: cbox-id + app.kubernetes.io/name: cbox-id + app.kubernetes.io/instance: cbox-id + app.kubernetes.io/managed-by: cbox-platform +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cbox-id + namespace: cbox-cbox-id + labels: + platform.cbox.dk/managed: 'true' + platform.cbox.dk/organization: local + platform.cbox.dk/service: cbox-id + app.kubernetes.io/name: cbox-id + app.kubernetes.io/instance: cbox-id + app.kubernetes.io/component: web + app.kubernetes.io/managed-by: cbox-platform + app.kubernetes.io/version: 8.5-bookworm-v1 +spec: + replicas: 2 + selector: + matchLabels: + platform.cbox.dk/service: cbox-id + platform.cbox.dk/process: web + template: + metadata: + labels: + platform.cbox.dk/managed: 'true' + platform.cbox.dk/organization: local + platform.cbox.dk/service: cbox-id + app.kubernetes.io/name: cbox-id + app.kubernetes.io/instance: cbox-id + app.kubernetes.io/component: web + app.kubernetes.io/managed-by: cbox-platform + app.kubernetes.io/version: 8.5-bookworm-v1 + platform.cbox.dk/process: web + spec: + containers: + - + name: cbox-id + image: 'ghcr.io/cboxdk/php-baseimages/php-fpm-nginx:8.5-bookworm-v1' + ports: + - containerPort: 80 + name: http + resources: + requests: + cpu: 100m + memory: 128Mi + env: + - name: APP_ENV + value: production + - name: LOG_CHANNEL + value: stderr + - name: DB_HOST + value: database-rw + - + name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: database-app + key: password + - + name: APP_KEY + valueFrom: + secretKeyRef: + name: cbox-id-app + key: APP_KEY + - + name: HEALTH_TOKEN + valueFrom: + secretKeyRef: + name: cbox-id-app + key: HEALTH_TOKEN + startupProbe: + httpGet: + path: /up + port: http + periodSeconds: 2 + timeoutSeconds: 3 + failureThreshold: 150 + livenessProbe: + httpGet: + path: /up + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + exec: + command: + - sh + - '-c' + - "printf 'Authorization: Bearer %s\\n' \"${HEALTH_TOKEN}\" | curl -fsS -o /dev/null --max-time 4 -H @- 'http://127.0.0.1:80/health/ready'" + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + topologySpreadConstraints: + - + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: cbox-id + platform.cbox.dk/process: web +--- +apiVersion: v1 +kind: Service +metadata: + name: cbox-id + namespace: cbox-cbox-id + labels: + platform.cbox.dk/managed: 'true' + platform.cbox.dk/organization: local + platform.cbox.dk/service: cbox-id + app.kubernetes.io/name: cbox-id + app.kubernetes.io/instance: cbox-id + app.kubernetes.io/managed-by: cbox-platform +spec: + selector: + platform.cbox.dk/service: cbox-id + platform.cbox.dk/process: web + ports: + - name: http + port: 80 + targetPort: 80 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: cbox-id + namespace: cbox-cbox-id + labels: + platform.cbox.dk/managed: 'true' + platform.cbox.dk/organization: local + platform.cbox.dk/service: cbox-id + app.kubernetes.io/name: cbox-id + app.kubernetes.io/instance: cbox-id + app.kubernetes.io/managed-by: cbox-platform +spec: + maxUnavailable: 1 + selector: + matchLabels: + platform.cbox.dk/service: cbox-id + platform.cbox.dk/process: web +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cbox-id-queue + namespace: cbox-cbox-id + labels: + platform.cbox.dk/managed: 'true' + platform.cbox.dk/organization: local + platform.cbox.dk/service: cbox-id + app.kubernetes.io/name: cbox-id + app.kubernetes.io/instance: cbox-id + app.kubernetes.io/managed-by: cbox-platform + platform.cbox.dk/process: queue +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: cbox-id + platform.cbox.dk/process: queue + template: + metadata: + labels: + platform.cbox.dk/managed: 'true' + platform.cbox.dk/organization: local + platform.cbox.dk/service: cbox-id + app.kubernetes.io/name: cbox-id + app.kubernetes.io/instance: cbox-id + app.kubernetes.io/managed-by: cbox-platform + platform.cbox.dk/process: queue + spec: + containers: + - + name: queue + image: 'ghcr.io/cboxdk/php-baseimages/php-fpm-nginx:8.5-bookworm-v1' + resources: + requests: + cpu: 100m + memory: 128Mi + env: + - name: APP_ENV + value: production + - name: LOG_CHANNEL + value: stderr + - name: DB_HOST + value: database-rw + - + name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: database-app + key: password + - + name: APP_KEY + valueFrom: + secretKeyRef: + name: cbox-id-app + key: APP_KEY + - + name: HEALTH_TOKEN + valueFrom: + secretKeyRef: + name: cbox-id-app + key: HEALTH_TOKEN + command: + - php + - artisan + - 'queue:autoscale' + topologySpreadConstraints: + - + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: cbox-id + platform.cbox.dk/process: queue +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cbox-id-scheduler + namespace: cbox-cbox-id + labels: + platform.cbox.dk/managed: 'true' + platform.cbox.dk/organization: local + platform.cbox.dk/service: cbox-id + app.kubernetes.io/name: cbox-id + app.kubernetes.io/instance: cbox-id + app.kubernetes.io/managed-by: cbox-platform + platform.cbox.dk/process: scheduler +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: cbox-id + platform.cbox.dk/process: scheduler + template: + metadata: + labels: + platform.cbox.dk/managed: 'true' + platform.cbox.dk/organization: local + platform.cbox.dk/service: cbox-id + app.kubernetes.io/name: cbox-id + app.kubernetes.io/instance: cbox-id + app.kubernetes.io/managed-by: cbox-platform + platform.cbox.dk/process: scheduler + spec: + containers: + - + name: scheduler + image: 'ghcr.io/cboxdk/php-baseimages/php-fpm-nginx:8.5-bookworm-v1' + resources: + requests: + cpu: 100m + memory: 128Mi + env: + - name: APP_ENV + value: production + - name: LOG_CHANNEL + value: stderr + - name: DB_HOST + value: database-rw + - + name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: database-app + key: password + - + name: APP_KEY + valueFrom: + secretKeyRef: + name: cbox-id-app + key: APP_KEY + - + name: HEALTH_TOKEN + valueFrom: + secretKeyRef: + name: cbox-id-app + key: HEALTH_TOKEN + command: + - php + - artisan + - 'schedule:work' + topologySpreadConstraints: + - + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: cbox-id + platform.cbox.dk/process: scheduler +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: cbox-id-release-78a3eb33bc + namespace: cbox-cbox-id + labels: + platform.cbox.dk/managed: 'true' + platform.cbox.dk/organization: local + platform.cbox.dk/service: cbox-id + platform.cbox.dk/run: release + app.kubernetes.io/managed-by: cbox-platform +spec: + backoffLimit: 0 + activeDeadlineSeconds: 3600 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + platform.cbox.dk/managed: 'true' + platform.cbox.dk/organization: local + platform.cbox.dk/service: cbox-id + platform.cbox.dk/run: release + app.kubernetes.io/managed-by: cbox-platform + spec: + containers: + - + name: run + image: 'ghcr.io/cboxdk/php-baseimages/php-fpm-nginx:8.5-bookworm-v1' + resources: + requests: + cpu: 100m + memory: 128Mi + env: + - name: APP_ENV + value: production + - name: LOG_CHANNEL + value: stderr + - name: DB_HOST + value: database-rw + - + name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: database-app + key: password + - + name: APP_KEY + valueFrom: + secretKeyRef: + name: cbox-id-app + key: APP_KEY + - + name: HEALTH_TOKEN + valueFrom: + secretKeyRef: + name: cbox-id-app + key: HEALTH_TOKEN + command: + - php + - artisan + - migrate + - '--force' + topologySpreadConstraints: + - + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: cbox-id + platform.cbox.dk/process: web + restartPolicy: Never