fix(backup): trust store, failed-enable revert, cluster-scoped repositories — with a server E2E tier

.github/workflows/e2e-server.yml

@@ -0,0 +1,136 @@
+# The server end-to-end suite.
+#
+# It is deliberately outside PR CI: the Lima/SSH run takes long enough to make
+# review feedback drag. Release calls it before publishing, while nightly and
+# manual runs keep the external dependencies exercised between releases.
+name: E2E (Server)
+
+on:
+  workflow_call:
+  # Nightly, because most of what this suite depends on is outside the
+  # repository and rots on its own schedule: the cloud image, the wal-g
+  # release, the postgres tag, and the certificates in between.
+  schedule:
+    - cron: "0 4 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: e2e-server-${{ github.sha }}
+  cancel-in-progress: false
+
+jobs:
+  server:
+    name: End-to-end (Server / ${{ matrix.suite.name }})
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        suite:
+          - name: lifecycle
+            test: TestServerLifecycle
+          - name: probes
+            test: TestServerProbes
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Enable KVM for the runner
+        run: |
+          set -euo pipefail
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
+            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq qemu-system-x86
+          # Without KVM, qemu falls back to software emulation and this job
+          # takes tens of minutes instead of a few. That is a finding about the
+          # runner, not a reason to wait: fail here rather than let a job that
+          # silently degraded read as a slow but healthy one.
+          test -e /dev/kvm
+
+      - name: Install Lima
+        run: |
+          set -euo pipefail
+          version=2.2.0
+          archive=lima-${version}-Linux-x86_64.tar.gz
+          sha=a0ea1ccf6b7335a900adb5f8d2b8384457965fecb1ba72f09b4e3e46d12f424a
+          # Pinned and checksummed for the same reason `just` is in ci.yml:
+          # what runs is exactly the artifact this line names, or the job stops.
+          # `releases/latest` would resolve at run time, which is the failure
+          # mode that pinning exists to remove.
+          curl -fsSL --retry 3 -o "$archive" \
+            "https://github.com/lima-vm/lima/releases/download/v${version}/${archive}"
+          echo "${sha}  ${archive}" | sha256sum --check --strict -
+          sudo tar Cxzf /usr/local "$archive"
+          rm -f "$archive"
+          limactl --version
+
+      - name: Install sops
+        run: |
+          set -euo pipefail
+          version=3.13.3
+          sha=e5bec3346a873ae91d871550f3e698c1aad962aff462a080e40f25fde17fef6b
+          # The fixture's backup credentials are sops-encrypted, and ob shells
+          # out to `sops -d` to read them: without this the suite fails at the
+          # credential check rather than at anything it means to test.
+          curl -fsSL --retry 3 -o sops \
+            "https://github.com/getsops/sops/releases/download/v${version}/sops-v${version}.linux.amd64"
+          echo "${sha}  sops" | sha256sum --check --strict -
+          sudo install -m 0755 sops /usr/local/bin/sops
+          rm -f sops
+          sops --version --disable-version-check
+
+      # The guest image is ~600MB and is pinned by digest in e2e/lima.yaml, so
+      # the cache key can be the file itself: a new image means a new key.
+      - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/lima
+          key: lima-image-${{ hashFiles('e2e/lima.yaml') }}
+
+      - name: Boot the server
+        run: timeout 300 bash scripts/lima.sh up
+
+      - name: Run the server end-to-end suite
+        run: bash scripts/lima.sh test -run '^${{ matrix.suite.test }}$'
+
+      # Curated diagnostics only.
+      #
+      # /var/lib/ob holds the mode-0600 backup credential file and decrypted
+      # environment files, so archiving that tree would publish the repository
+      # key and the object-store keys as a downloadable artifact on a public
+      # run. Names and permissions answer the questions a failure asks;
+      # contents answer questions nobody asked.
+      - name: Collect diagnostics
+        if: always()
+        run: |
+          set -uo pipefail
+          mkdir -p diagnostics
+          ssh_guest() {
+            limactl shell onebox-e2e -- sudo "$@" 2>&1 || true
+          }
+          ssh_guest systemctl list-units --type=timer --all --no-pager > diagnostics/timers.txt
+          ssh_guest journalctl -u 'ob-backup-*' --no-pager --lines=500 > diagnostics/journal.txt
+          ssh_guest docker ps -a > diagnostics/containers.txt
+          ssh_guest docker logs --tail=200 minio > diagnostics/minio.txt
+          # Layout and permissions, never contents.
+          ssh_guest find /var/lib/ob -maxdepth 4 -printf '%M %u:%g %10s %p\n' > diagnostics/ob-tree.txt
+          cp ~/.lima/onebox-e2e/serial.log diagnostics/ 2>/dev/null || true
+          cp ~/.lima/onebox-e2e/ha.stderr.log diagnostics/ 2>/dev/null || true
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: always()
+        with:
+          name: server-diagnostics-${{ matrix.suite.name }}
+          path: diagnostics/
+          retention-days: 7

.github/workflows/release.yml

@@ -23,9 +23,15 @@ jobs:
       contents: read
     uses: ./.github/workflows/ci.yml
 
+  server-e2e:
+    name: Verify release on a real server
+    permissions:
+      contents: read
+    uses: ./.github/workflows/e2e-server.yml
+
   release:
     name: Publish signed release, Homebrew, and Scoop
-    needs: verify
+    needs: [verify, server-e2e]
     runs-on: ubuntu-24.04
     timeout-minutes: 40
     permissions:

Justfile

@@ -157,6 +157,30 @@ workflow-check:
 e2e:
     OB_E2E=1 go test ./e2e/ -count=1 -timeout 20m
 
+# Boot the throwaway server the `server-e2e` suite deploys to.
+#
+# Separate from the suite because the guest outlives a test run: booting takes
+# about a minute, and iterating on a failing case should not pay for it again.
+lima-up:
+    bash scripts/lima.sh up
+
+lima-down:
+    bash scripts/lima.sh down
+
+# The server end-to-end suite: the tests run here and reach a real machine over
+# SSH, which is the transport every operator uses and the one the Docker suite
+# substitutes local docker for.
+#
+# It boots the guest first rather than failing on a missing one, because the
+# common case is not having booted it, and `lima-up` reuses a running instance.
+server-e2e: lima-up
+    bash scripts/lima.sh test
+
+# Print the connection the suite uses, for running a single case by hand:
+#   eval "$(just server-env | sed 's/^/export /')"
+server-env:
+    @bash scripts/lima.sh env
+
 # Regenerate the parts of the documentation site that are derived from Go.
 #
 # The project-file field reference, the error-code catalogue and the CLI

cmd/ob/backup.go

@@ -148,7 +148,7 @@ func addBackupCommands(root *cobra.Command, g *globalFlags) {
 	}
 	backupCmd.AddCommand(verifyCmd)
 
-	var restoreTo, restoreConfirm string
+	var restoreTo, restoreConfirm, restoreGeneration string
 	var restoreBreakLock bool
 	restoreCmd := &cobra.Command{
 		Use:   "restore <service>",
@@ -177,16 +177,17 @@ func addBackupCommands(root *cobra.Command, g *globalFlags) {
 			}
 			return runMutation(cmd, g, onebox.ExecuteRequest{
 				Kind: onebox.KindRestoreCutover, Service: args[0],
-				RecoveryTarget: restoreTo, BreakLock: restoreBreakLock,
+				RecoveryTarget: restoreTo, RecoveryGeneration: restoreGeneration, BreakLock: restoreBreakLock,
 			}, "backup restore")
 		},
 	}
 	restoreCmd.Flags().StringVar(&restoreTo, "to", "", "RFC 3339 point in time to recover to (default: the newest recoverable point)")
+	restoreCmd.Flags().StringVar(&restoreGeneration, "generation", "", "repository generation to recover (PostgreSQL system identifier or legacy; default: current)")
 	restoreCmd.Flags().StringVar(&restoreConfirm, "confirm", "", "name of the service whose live data may be replaced")
 	restoreCmd.Flags().BoolVar(&restoreBreakLock, "break-lock", false, "break a stale operation lock after inspecting its holder")
 	backupCmd.AddCommand(restoreCmd)
 
-	var drillTo string
+	var drillTo, drillGeneration string
 	drillCmd := &cobra.Command{
 		Use:   "drill <service>",
 		Short: "prove the repository recovers, without touching anything",
@@ -199,13 +200,15 @@ func addBackupCommands(root *cobra.Command, g *globalFlags) {
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return runMutation(cmd, g, onebox.ExecuteRequest{
-				Kind: onebox.KindRestoreTest, Service: args[0], RecoveryTarget: drillTo,
+				Kind: onebox.KindRestoreTest, Service: args[0], RecoveryTarget: drillTo, RecoveryGeneration: drillGeneration,
 			}, "backup drill")
 		},
 	}
 	drillCmd.Flags().StringVar(&drillTo, "to", "", "RFC 3339 point in time to prove recoverable (default: the newest recoverable point)")
+	drillCmd.Flags().StringVar(&drillGeneration, "generation", "", "repository generation to prove (PostgreSQL system identifier or legacy; default: current)")
 	backupCmd.AddCommand(drillCmd)
 
+	var statusGeneration string
 	statusCmd := &cobra.Command{
 		Use:   "status <service>",
 		Short: "what the repository can recover, read from the repository",
@@ -215,7 +218,7 @@ func addBackupCommands(root *cobra.Command, g *globalFlags) {
 			"policy is declared but never enabled has no repository to ask, and says so.",
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			status, err := operationsService(cmd, g).BackupStatus(cmd.Context(), args[0])
+			status, err := operationsService(cmd, g).BackupStatusGeneration(cmd.Context(), args[0], statusGeneration)
 			if err != nil {
 				return err
 			}
@@ -226,6 +229,9 @@ func addBackupCommands(root *cobra.Command, g *globalFlags) {
 			}
 			out := cmd.OutOrStdout()
 			fmt.Fprintf(out, "service     %s\nrepository  %s\n", status.Service, status.Repository)
+			for _, generation := range status.AvailableRepositoryGenerations {
+				fmt.Fprintf(out, "generation  %s\n", generation)
+			}
 			for _, issue := range status.RuntimeIssues {
 				fmt.Fprintf(out, "drift       %s\n", issue)
 			}
@@ -260,6 +266,7 @@ func addBackupCommands(root *cobra.Command, g *globalFlags) {
 			return w.Flush()
 		},
 	}
+	statusCmd.Flags().StringVar(&statusGeneration, "generation", "", "repository generation to inspect (PostgreSQL system identifier or legacy; default: current)")
 	backupCmd.AddCommand(statusCmd)
 
 	root.AddCommand(backupCmd)

docs/product.md

@@ -38,12 +38,12 @@ identity, not user configuration.
 
 The broader managed-operations goal is direction, not an inventory. Owned today:
 host bootstrap, the container runtime check, the proxy and its TLS, the host
-ingress network, release
-staging and retention, the supporting data services and their credentials, and
-scheduled jobs. **Not owned today: backups, restore proof, and log rotation.**
-Onebox says so rather than implying otherwise — `ob doctor` reports the absence
-of backups for every workload and service holding durable data, because silence
-there would read as approval.
+ingress network, release staging and retention, the supporting data services and
+their credentials, scheduled jobs, and PostgreSQL backup, point-in-time recovery,
+and on-demand restore proof. **Not owned today: workload-volume backups,
+unattended full restore drills, and log rotation.** Onebox says so rather than
+implying otherwise — `ob doctor` reports every durable workload or service that
+has no executable backup contract, because silence there would read as approval.
 
 The distinction matters more than it looks. A product direction that reads as a
 capability list is how an operator ends up believing their database is backed

e2e/lima.yaml

@@ -0,0 +1,75 @@
+# The machine the server suite deploys to.
+#
+# It stands in for the box an operator rents: a bare Ubuntu with sshd and
+# nothing else. Nothing is provisioned here beyond root's key, because what
+# ob does to an unprepared machine is the thing under test — a guest that
+# arrives with a container runtime already installed would answer the
+# bootstrap contract by assumption instead of by observation.
+#
+# Lima chooses the hypervisor: vz on Apple Silicon, qemu on the CI runner.
+# vmType is deliberately unset, and rosetta deliberately absent — nothing in
+# this repository is amd64-only (internal/app/backup_walg.go pins wal-g for
+# both architectures and postgres:18 is multi-arch), so translating x86_64
+# binaries on an arm64 guest would only make the Mac run something CI never
+# runs.
+minimumLimaVersion: 2.2.0
+
+# Pinned to an immutable dated release, by digest, per architecture. The
+# undated `release/` directory is a moving symlink: pinning to it would let
+# the guest change under a run that changed nothing, and floating URLs rot on
+# each architecture independently, which breaks exactly one of the two
+# environments this suite exists to keep identical.
+images:
+  - location: "https://cloud-images.ubuntu.com/releases/noble/release-20260814/ubuntu-24.04-server-cloudimg-amd64.img"
+    arch: "x86_64"
+    digest: "sha256:6e40c07ae715f744f84af0bec76415cc1987dd115b4b8de437818561f01a3733"
+  - location: "https://cloud-images.ubuntu.com/releases/noble/release-20260814/ubuntu-24.04-server-cloudimg-arm64.img"
+    arch: "aarch64"
+    digest: "sha256:4a281a921b8d7db952895ab619736f10efe9f63e111fa5b5779ed18f023818aa"
+
+cpus: 2
+memory: "4GiB"
+disk: "20GiB"
+
+# Lima installs containerd by default. A guest that arrives with a container
+# runtime cannot test the contract ce7e41b established, which is that Docker
+# is the operator's to provide through the bootstrap hook.
+containerd:
+  system: false
+  user: false
+
+# No shared filesystem, for two reasons. It is the one place the two
+# hypervisors genuinely differ — virtiofs under vz, 9p under qemu — and
+# removing it removes that divergence from the harness. And ob's own
+# tar-over-SSH upload is what should be carrying payloads to a server: it is
+# the code path under test, and a mount would route around it.
+mounts: []
+
+# Only Lima's generated key reaches the guest. The developer's personal keys
+# are irrelevant to a throwaway machine and would make the harness behave
+# differently on a laptop than on a runner.
+ssh:
+  loadDotSSHPubKeys: false
+
+provision:
+  # ob connects as root and never elevates: there is no `sudo` anywhere in
+  # internal/engine, internal/transport or internal/onebox, and it writes unit
+  # files directly under /etc/systemd/system. A guest reachable only as an
+  # unprivileged user would fail at the first schedule sync.
+  #
+  # Overwriting rather than appending is deliberate: Ubuntu's cloud images
+  # install a forced-command in root's authorized_keys that prints a refusal
+  # and exits.
+  - mode: system
+    script: |
+      #!/bin/sh
+      set -eu
+      install -d -m 0700 /root/.ssh
+      cat /home/*/.ssh/authorized_keys > /root/.ssh/authorized_keys
+      chmod 0600 /root/.ssh/authorized_keys
+      # A bare machine is the premise of this guest. If an image ever ships
+      # one, fail here rather than let the bootstrap tests pass vacuously.
+      if command -v docker >/dev/null 2>&1; then
+        echo "guest arrived with docker installed; bootstrap coverage would be vacuous" >&2
+        exit 1
+      fi